diff --git a/.circleci/AGENTS.md b/.circleci/AGENTS.md deleted file mode 100644 index b3591145a3e..00000000000 --- a/.circleci/AGENTS.md +++ /dev/null @@ -1,7 +0,0 @@ -# .circleci - -The `integration` workflow on `main` runs from a scheduled pipeline in the CircleCI project settings, cron `9 0-8,14-23 * * *` (UTC) on `main` with no pipeline parameters. It has no `triggers:` block in this file because a workflow with one runs only on its schedule, so the API pipelines that set `run_migration_tests` or `routing_parity_base` would stop running it. If main stops getting integration runs, check that the schedule still exists and recreate it with these values - -Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because CircleCI falls back to the project's SSH checkout key without its GitHub App access, GitHub rejects that key, and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base - -A step that checks out files from another commit fetches that commit with `--no-filter`, as `routing_parity` does, because the clone has no file contents and git would otherwise download each missing file in its own request, which took 210 to 896 seconds for the routing parity base diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 95ee0a6412c..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,1780 +0,0 @@ -version: 2.1 -parameters: - run_migration_tests: - type: boolean - default: false - migration_candidate_image: - type: string - default: "" - migration_baseline_image: - type: string - default: "ghcr.io/berriai/litellm-database:v1.102.0" - migration_source_sha: - type: string - default: "" - routing_parity_base: - type: string - default: "" -orbs: - codecov: codecov/codecov@4.0.1 - node: circleci/node@5.1.0 # Add this line to declare the node orb - win: circleci/windows@5.0 # Add Windows orb - -commands: - checkout_over_https: - parameters: - revision: - type: string - default: "" - shell: - type: string - default: /bin/bash -eo pipefail - steps: - - run: - name: Check out the public repository over HTTPS - shell: << parameters.shell >> - environment: - REQUESTED_REVISION: << parameters.revision >> - command: | - revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}" - [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1 - git init - git remote add origin https://github.com/BerriAI/litellm.git - for attempt in 1 2 3; do - git fetch --filter=blob:none origin "$revision" && git checkout --detach FETCH_HEAD && break - [[ $attempt -lt 3 ]] || exit 1 - sleep 5 - done - skip_if_unrelated_changes: - parameters: - category: - type: enum - enum: ["backend", "client"] - default: "backend" - steps: - - run: - name: "Skip job when no << parameters.category >>-relevant files changed" - command: bash .circleci/scripts/path_filter.sh << parameters.category >> - setup_google_dns: - steps: - - run: - name: "Configure Google DNS" - command: | - # Backup original resolv.conf - sudo cp /etc/resolv.conf /etc/resolv.conf.backup - # Add both local and Google DNS servers - echo "nameserver 127.0.0.11" | sudo tee /etc/resolv.conf - echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf - echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf - wait_for_service: - description: "Poll a TCP or HTTP endpoint until it responds (replaces dockerize -wait)" - parameters: - url: - type: string - timeout: - type: string - default: "60" - steps: - - run: - name: "Wait for << parameters.url >>" - command: | - TIMEOUT=<< parameters.timeout >> - URL="<< parameters.url >>" - ELAPSED=0 - echo "Waiting up to ${TIMEOUT}s for ${URL} ..." - if echo "$URL" | grep -q '^tcp://'; then - HOST=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f1) - PORT=$(echo "$URL" | sed 's|tcp://||' | cut -d: -f2) - while ! bash -c "echo > /dev/tcp/$HOST/$PORT" 2>/dev/null; do - sleep 2; ELAPSED=$((ELAPSED+2)) - if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi - done - else - while ! curl -sf --max-time 5 "$URL" > /dev/null 2>&1; do - sleep 2; ELAPSED=$((ELAPSED+2)) - if [ "$ELAPSED" -ge "$TIMEOUT" ]; then echo "Timed out"; exit 1; fi - done - fi - echo "Service ready after ${ELAPSED}s" - install_helm: - steps: - - run: - name: Install Helm v3.17.3 - command: | - curl -sSLf -o /tmp/helm.tar.gz \ - https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz - echo "ee88b3c851ae6466a3de507f7be73fe94d54cbf2987cbaa3d1a3832ea331f2cd /tmp/helm.tar.gz" | sha256sum -c - - sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/helm.tar.gz linux-amd64/helm - rm -f /tmp/helm.tar.gz - install_kind: - steps: - - run: - name: Install Kind v0.20.0 - command: | - curl -sSLf -o /tmp/kind \ - https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - echo "513a7213d6d3332dd9ef27c24dab35e5ef10a04fa27274fe1c14d8a246493ded /tmp/kind" | sha256sum -c - - chmod +x /tmp/kind - sudo mv /tmp/kind /usr/local/bin/kind - install_uv: - description: "Install pinned uv (0.10.9) with checksum verification. Adds ~/.local/bin to PATH." - steps: - - run: - name: Install uv (pinned 0.10.9) - command: | - curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh - echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c - - env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh - rm -f /tmp/uv-install.sh - echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" - export PATH="$HOME/.local/bin:$PATH" - install_node: - description: "Install the Node.js version pinned in ui/litellm-dashboard/.nvmrc (24.19.0, which bundles npm 11.17.0) with checksum verification, and prepend it to PATH. Run this on any executor whose image does not already ship that version, or `npm ci` in ui/litellm-dashboard fails EBADENGINE against the engines floor. Installs into /opt/node rather than over /usr/local on purpose: cimg/python:*-browsers ships its own node there, and unpacking the tarball on top of it leaves npm 11.17 files merged with the image's npm 11.9 tree, which reports the new version and then exits 1 on `npm ci` with no error text at all. Requires checkout, which the .nvmrc drift check reads." - steps: - - run: - name: Install Node.js 24.19.0 - command: | - NODE_VERSION="24.19.0" - NODE_TARBALL="node-v${NODE_VERSION}-linux-x64.tar.xz" - NODE_EXPECTED_SHA="14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647" - NVMRC_VERSION="$(tr -d '[:space:]' < ui/litellm-dashboard/.nvmrc)" - if [ "$NVMRC_VERSION" != "$NODE_VERSION" ]; then - echo "install_node: ui/litellm-dashboard/.nvmrc pins ${NVMRC_VERSION} but this command pins ${NODE_VERSION}; update NODE_VERSION and NODE_EXPECTED_SHA together" >&2 - exit 1 - fi - curl -sSLf -o "/tmp/${NODE_TARBALL}" "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TARBALL}" - echo "${NODE_EXPECTED_SHA} /tmp/${NODE_TARBALL}" | sha256sum -c - - sudo mkdir -p /opt/node - sudo tar -xJf "/tmp/${NODE_TARBALL}" -C /opt/node --strip-components=1 - rm -f "/tmp/${NODE_TARBALL}" - echo 'export PATH="/opt/node/bin:$PATH"' >> "$BASH_ENV" - export PATH="/opt/node/bin:$PATH" - node --version - npm --version - install_rust: - description: "Install pinned rustup (1.28.2) and Rust toolchain (1.98.0) with checksum verification. Adds ~/.cargo/bin to PATH. Run this before any `uv sync` or `uv build` of the workspace: the root package builds litellm-rust through maturin, and on an image without cargo maturin fetches an unpinned rustup and a floating toolchain by itself. Also restores the dev-profile cargo cache that save_cargo_target writes on main, minus the workspace crates' fingerprints so those always rebuild from the checked-out source." - steps: - - run: - name: Install Rust (rustup 1.28.2, toolchain 1.98.0) - command: | - case "$(uname -m)" in - x86_64) - RUSTUP_TRIPLE=x86_64-unknown-linux-gnu - RUSTUP_SHA256=20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c - ;; - aarch64) - RUSTUP_TRIPLE=aarch64-unknown-linux-gnu - RUSTUP_SHA256=e3853c5a252fca15252d07cb23a1bdd9377a8c6f3efa01531109281ae47f841c - ;; - *) - echo "install_rust: unsupported architecture $(uname -m)" >&2 - exit 1 - ;; - esac - curl -sSLf --retry 3 --retry-all-errors --retry-max-time 120 --connect-timeout 15 --max-time 60 -o /tmp/rustup-init \ - "https://static.rust-lang.org/rustup/archive/1.28.2/${RUSTUP_TRIPLE}/rustup-init" - echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c - - chmod +x /tmp/rustup-init - /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain 1.98.0 - rm -f /tmp/rustup-init - echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> "$BASH_ENV" - echo 'export CARGO_INCREMENTAL=0' >> "$BASH_ENV" - export PATH="$HOME/.cargo/bin:$PATH" - rustc --version - cargo --version - { rustc -vV; cc --version; cat /etc/os-release; } > /tmp/cargo-build-env - - restore_cache: - keys: - - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} - - v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}- - - run: - name: Force a rebuild of the workspace crates restored from the cargo cache - command: rm -rf litellm-rust/target/debug/.fingerprint/litellm-* - save_cargo_target: - steps: - - when: - condition: - equal: [main, << pipeline.git.branch >>] - steps: - - save_cache: - key: v1-cargo-dev-{{ checksum "/tmp/cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} - paths: - - ~/.cargo/registry - - ~/project/litellm-rust/target/debug - start_postgres: - description: "Start a postgres-db container on port 5432 and wait until it accepts connections." - parameters: - db_name: - type: string - default: circle_test - image: - type: string - default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26 - server_args: - type: string - default: "" - steps: - - run: - name: Start PostgreSQL - command: | - docker run -d \ - --name postgres-db \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=<< parameters.db_name >> \ - -p 5432:5432 \ - << parameters.image >> << parameters.server_args >> - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - start_redis: - description: "Start a redis container on port 6379 and wait until it accepts connections. Use this to isolate a job from the shared remote Redis so concurrent CI pipelines don't contend for pod locks or buffer keys." - steps: - - run: - name: Start Redis - command: | - docker run -d \ - --name redis-cache \ - -p 6379:6379 \ - redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6 - - wait_for_service: - url: tcp://localhost:6379 - timeout: "60" - install_codecov_cli: - steps: - - run: - name: Install Codecov CLI (pinned v11.3.1) - when: always - command: | - curl -sSLf -o /tmp/codecov https://cli.codecov.io/v11.3.1/linux/codecov - curl -sSLf -o /tmp/codecov.SHA256SUM https://cli.codecov.io/v11.3.1/linux/codecov.SHA256SUM - [ "$(cat /tmp/codecov.SHA256SUM)" = "ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d codecov" ] - (cd /tmp && sha256sum -c codecov.SHA256SUM) - chmod +x /tmp/codecov - mkdir -p "$HOME/.local/bin" - mv /tmp/codecov "$HOME/.local/bin/codecov" - start_openai_record_replay_proxy: - description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay Redis cache hits. Cache misses call the live provider and store its response, so these legacy E2E jobs still require provider availability and may incur provider charges. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream// so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced." - steps: - - run: - name: Start record/replay proxy - background: true - command: | - CASSETTE_REDIS_URL="$CASSETTE_REDIS_URL" \ - RECORDER_UPSTREAM_BASE_URL="https://api.openai.com" \ - uv run --no-sync python tests/_openai_record_replay_proxy.py --host 0.0.0.0 --port 8090 - - run: - name: Wait for record/replay proxy - command: | - for i in $(seq 1 30); do - if curl -sf http://localhost:8090/__recorder_health >/dev/null 2>&1; then - echo "record/replay proxy is up" - exit 0 - fi - sleep 1 - done - echo "record/replay proxy did not become ready" >&2 - exit 1 - start_fake_openai_endpoint: - description: "Start the canned OpenAI mock (tests/_fake_openai_endpoint_server.py) on host port 8190 and wait until healthy. Models whose api_base points here (via FAKE_OPENAI_API_BASE) get well-formed chat/text/embedding responses with realistic usage, so the E2E run neither pays for nor depends on the live provider. A request whose model is '429' returns HTTP 429 for rate-limit/cooldown tests. Run after uv deps are synced." - steps: - - run: - name: Start fake OpenAI endpoint - background: true - command: | - uv run --no-sync python tests/_fake_openai_endpoint_server.py --host 0.0.0.0 --port 8190 - - run: - name: Wait for fake OpenAI endpoint - command: | - for i in $(seq 1 30); do - if curl -sf http://localhost:8190/health >/dev/null 2>&1; then - echo "fake OpenAI endpoint is up" - exit 0 - fi - sleep 1 - done - echo "fake OpenAI endpoint did not become ready" >&2 - exit 1 - setup_litellm_enterprise_pip: - steps: - - run: - name: "Install local version of litellm-enterprise" - command: | - # litellm-enterprise is a uv workspace member and is already installed - # by the main `uv sync --all-groups --all-extras`. Do NOT run - # `uv sync --package litellm-enterprise` here — that overwrites the - # shared .venv and strips out dev/test deps (pytest, prisma, etc.). - uv run --no-sync python -c "import litellm_enterprise; print('litellm-enterprise OK:', litellm_enterprise.__file__)" - install_windows_toolchain: - steps: - - run: - name: Install Rust and uv - no_output_timeout: 30m - command: | - $rustupInit = Join-Path $env:TEMP "rustup-init.exe" - $rustupVersion = "1.28.2" - $rustupUrl = "https://static.rust-lang.org/rustup/archive/$rustupVersion/x86_64-pc-windows-msvc/rustup-init.exe" - Invoke-WebRequest -Uri $rustupUrl -OutFile $rustupInit - $rustupExpected = "88d8258dcf6ae4f7a80c7d1088e1f36fa7025a1cfd1343731b4ee6f385121fc0" - $rustupActual = (Get-FileHash -Path $rustupInit -Algorithm SHA256).Hash.ToLower() - if ($rustupActual -ne $rustupExpected) { - throw "rustup installer hash mismatch: expected $rustupExpected got $rustupActual" - } - & $rustupInit -y --profile minimal --default-toolchain 1.98.0 - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - Remove-Item $rustupInit - $cargoBin = Join-Path $HOME ".cargo\bin" - $env:Path = "$cargoBin;$env:Path" - rustc --version - cargo --version - $installer = Join-Path $env:TEMP "uv-install.ps1" - Invoke-WebRequest -Uri https://astral.sh/uv/0.10.9/install.ps1 -OutFile $installer - $expected = "d43ffff8d28e7d1e7d1831a212465f12b24a43c7f87f386e95e2a5915aee5d7d" - $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower() - if ($actual -ne $expected) { - throw "uv installer hash mismatch: expected $expected got $actual" - } - & $installer - Remove-Item $installer - $uvBin = Join-Path $HOME ".local\bin" - $env:Path = "$uvBin;$env:Path" - if (!(Test-Path $PROFILE)) { - New-Item -ItemType File -Force -Path $PROFILE | Out-Null - } - if (-not (Select-String -Path $PROFILE -SimpleMatch $uvBin -Quiet)) { - Add-Content -Path $PROFILE -Value "`$env:Path = `"$uvBin;`$env:Path`"" - } - if (-not (Select-String -Path $PROFILE -SimpleMatch $cargoBin -Quiet)) { - Add-Content -Path $PROFILE -Value "`$env:Path = `"$cargoBin;`$env:Path`"" - } - setup_litellm_test_deps: - parameters: - checkout: - type: boolean - default: true - steps: - - when: - condition: << parameters.checkout >> - steps: - - checkout_over_https - - setup_google_dns - - install_uv - - install_rust - - restore_cache: - keys: - - v3-integration-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - setup_litellm_enterprise_pip - - save_cache: - paths: - - ~/.cache/uv - key: v3-integration-uv-cache-{{ checksum "uv.lock" }} - - save_cargo_target - -jobs: - windows_release_wheel: - executor: - name: win/default - shell: powershell.exe - size: xlarge - working_directory: ~/project - environment: - UV_PYTHON: "3.11" - CARGO_HTTP_MULTIPLEXING: "false" - CARGO_NET_RETRY: "5" - steps: - - checkout_over_https: - shell: bash.exe - - run: - name: Skip job when no windows-release-relevant files changed - shell: bash.exe - command: bash .circleci/scripts/path_filter.sh windows-release - - run: - name: Install Python 3.11.9 from python.org - no_output_timeout: 6m - command: | - $ProgressPreference = "SilentlyContinue" - $installer = Join-Path $env:TEMP "python-3.11.9-amd64.exe" - Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.11.9/python-3.11.9-amd64.exe -OutFile $installer -TimeoutSec 120 - $expected = "5ee42c4eee1e6b4464bb23722f90b45303f79442df63083f05322f1785f5fdde" - $actual = (Get-FileHash -Path $installer -Algorithm SHA256).Hash.ToLower() - if ($actual -ne $expected) { - throw "Python installer hash mismatch: expected $expected got $actual" - } - $arguments = "/quiet InstallAllUsers=1 PrependPath=1 Include_test=0 Include_doc=0 Include_launcher=0 TargetDir=C:\Python311" - $process = Start-Process -FilePath $installer -ArgumentList $arguments -PassThru - if (-not $process.WaitForExit(300000)) { - $process.Kill() - throw "Python installer did not finish within 5 minutes" - } - if ($process.ExitCode -ne 0) { - throw "Python installer exited with $($process.ExitCode)" - } - Remove-Item $installer - & C:\Python311\python.exe --version - - install_windows_toolchain - - run: - name: Record the Rust build environment for the release cargo cache key - command: | - & "$HOME\.cargo\bin\rustc.exe" -vV | Out-File -Encoding ascii .cargo-build-env - - restore_cache: - keys: - - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} - - v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}- - - run: - name: Force a rebuild of the workspace crates restored from the cargo cache - command: | - $fingerprints = "litellm-rust/target/release/.fingerprint" - if (Test-Path $fingerprints) { - Get-ChildItem -Path $fingerprints -Filter "litellm-*" | Remove-Item -Recurse -Force - } - - run: - name: Build the release wheel and install it under a worst-case MAX_PATH prefix - no_output_timeout: 30m - environment: - UV_HTTP_TIMEOUT: "300" - command: | - $env:Path = "C:\Python311;C:\Python311\Scripts;$HOME\.cargo\bin;$HOME\.local\bin;$env:Path" - uv build --wheel --out-dir dist - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - python tests/windows_tests/check_windows_wheel_install.py - - when: - condition: - equal: [main, << pipeline.git.branch >>] - steps: - - save_cache: - key: v1-cargo-release-windows-{{ checksum ".cargo-build-env" }}-{{ checksum "litellm-rust/Cargo.lock" }} - paths: - - ~/.cargo/registry - - ~/project/litellm-rust/target/release - - base_sdk_install: - docker: - - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - working_directory: ~/project - steps: - - checkout_over_https - - setup_google_dns - - install_uv - - install_rust - - run: - name: Build the wheel - no_output_timeout: 30m - environment: - UV_HTTP_TIMEOUT: "300" - command: | - uv build --wheel --out-dir dist - - run: - name: Install the wheel with no extras and smoke-check it - environment: - UV_HTTP_TIMEOUT: "300" - command: | - uv venv /tmp/base-sdk --python 3.12 - VIRTUAL_ENV=/tmp/base-sdk uv pip install dist/*.whl - /tmp/base-sdk/bin/python tests/base_sdk_tests/check_base_sdk_install.py - - run: - name: Guard against MAX_PATH-busting packaged wheel paths - command: | - python3 tests/windows_tests/check_windows_wheel_install.py --lengths-only - - helm_chart_testing: - machine: - image: ubuntu-2204:2024.04.1 # Use machine executor instead of docker - resource_class: medium - working_directory: ~/project - - steps: - - checkout_over_https - - skip_if_unrelated_changes - - attach_workspace: - at: ~/project - - setup_google_dns - - install_helm - - install_kind - - # Install kubectl (pinned version with hardcoded checksum) - - run: - name: Install kubectl v1.31.4 - command: | - curl -sSLf -o /tmp/kubectl \ - https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl - echo "298e19e9c6c17199011404278f0ff8168a7eca4217edad9097af577023a5620f /tmp/kubectl" | sha256sum -c - - chmod +x /tmp/kubectl - sudo mv /tmp/kubectl /usr/local/bin/ - - # Create kind cluster - - run: - name: Create Kind Cluster - command: | - kind create cluster --name litellm-test - - - run: - name: Load Docker Database Image for helm tests - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - IMAGE_TAG=${CIRCLE_SHA1:-ci} - docker tag litellm-docker-database:ci litellm-ci:${IMAGE_TAG} - - - run: - name: Load Docker image into Kind - command: | - IMAGE_TAG=${CIRCLE_SHA1:-ci} - kind load docker-image litellm-ci:${IMAGE_TAG} --name litellm-test - - # Run helm lint - - run: - name: Run helm lint - command: | - helm lint ./helm/litellm-helm - - # Run helm tests - - run: - name: Run helm tests - command: | - IMAGE_TAG=${CIRCLE_SHA1:-ci} - helm install litellm ./helm/litellm-helm -f ./helm/litellm-helm/ci/test-values.yaml \ - --set image.repository=litellm-ci \ - --set image.tag=${IMAGE_TAG} \ - --set image.pullPolicy=Never - if ! kubectl rollout status deployment/litellm --timeout=180s; then - kubectl describe pods -l app.kubernetes.io/name=litellm - kubectl logs -l app.kubernetes.io/name=litellm --tail=200 - exit 1 - fi - - echo "Printing pod logs..." - kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}") - - # Run the helm tests - helm test litellm --logs - - # Cleanup - - run: - name: Cleanup - command: | - kind delete cluster --name litellm-test - when: always # This ensures cleanup runs even if previous steps fail - - db_migration_disable_update_check: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: medium - working_directory: ~/project - steps: - - checkout_over_https - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes - - setup_google_dns - - install_uv - - install_rust - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_fake_openai_endpoint - - start_postgres: - db_name: litellm_test - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Seed database with real schema - command: | - docker run -d \ - -p 4001:4000 \ - -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ - -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \ - --name schema-seed \ - --add-host=host.docker.internal:host-gateway \ - -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --use_prisma_db_push - - wait_for_service: - url: http://localhost:4001 - timeout: "300" - - run: - name: Stop schema seed container - command: docker stop schema-seed && docker rm schema-seed - - run: - name: Run Docker container with bad schema and disabled updates - command: | - docker run -d \ - -p 4000:4000 \ - -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \ - -e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_test" \ - -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ - -e DISABLE_SCHEMA_UPDATE="True" \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - --name my-app \ - --add-host=host.docker.internal:host-gateway \ - -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \ - -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \ - -v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 - - wait_for_service: - url: http://localhost:4000 - timeout: "60" - - run: - name: Check container logs for expected message - command: | - echo "=== Printing Full Container Startup Logs ===" - LOG_OUTPUT="$(docker logs my-app 2>&1)" - printf '%s\n' "$LOG_OUTPUT" - echo "=== End of Full Container Startup Logs ===" - - if printf '%s\n' "$LOG_OUTPUT" | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then - echo "Expected message found in logs. Test passed." - else - echo "Expected message not found in logs. Test failed." - exit 1 - fi - - run: - name: Run Basic Proxy Startup Tests (Health Readiness and Chat Completion) - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/integration/configuration/test_basic_proxy_startup.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ - -v \ - --junitxml=test-results/junit-2.xml \ - --durations=5" - no_output_timeout: 15m - - store_test_results: - path: test-results - - proxy_logging_guardrails_model_info_tests: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout_over_https - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes - - setup_google_dns - - install_uv - - install_rust - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_postgres - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - start_openai_record_replay_proxy - - start_fake_openai_endpoint - # Second Docker Container Run with Different Config - # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license - - run: - name: Run Second Docker container - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e REDIS_HOST=$REDIS_HOST \ - -e REDIS_PASSWORD=$REDIS_PASSWORD \ - -e REDIS_PORT=$REDIS_PORT \ - -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - -e LITELLM_LICENSE="bad-license" \ - -e LITELLM_LOG=ERROR \ - --add-host host.docker.internal:host-gateway \ - --name my-app-3 \ - -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 - - - run: - name: Start outputting logs for second container - command: docker logs -f my-app-3 - background: true - - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - - - run: - name: Run second round of tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/integration/configuration/test_basic_proxy_startup.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ - -v \ - --junitxml=test-results/junit-2.xml \ - --durations=5" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results - e2e_ui_build: - docker: - - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - resource_class: large - working_directory: ~/project - steps: - - checkout_over_https - - skip_if_unrelated_changes: - category: client - - setup_google_dns - - install_node - - restore_cache: - keys: - - ui-dashboard-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - - run: - name: Install dashboard dependencies - command: | - cd ui/litellm-dashboard - npm ci - - save_cache: - key: ui-dashboard-node-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }} - paths: - - ui/litellm-dashboard/node_modules - - run: - name: Build UI bundle - command: | - cd ui/litellm-dashboard - npm run build - mkdir -p ../../ui-bundle - mv out ../../ui-bundle/out - find ../../ui-bundle/out -name '*.html' ! -name 'index.html' | while read -r f; do - d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html" - done - - persist_to_workspace: - root: ~/project - paths: - - ui-bundle/out - - e2e_ui_testing: - docker: - - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: e2euser - POSTGRES_PASSWORD: e2epassword - POSTGRES_DB: litellm_e2e - resource_class: xlarge - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" - CI: "true" - # Boot the proxy with an external logout URL so proxyLogoutUrl.spec.ts can - # assert the redirect. Set at job level so both the proxy boot step and the - # Playwright step (whose skip guard reads this) see the same value. Safe for - # the rest of the suite: nothing else performs a logout. - PROXY_LOGOUT_URL: "https://www.example.com" - steps: - - checkout_over_https - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes: - category: client - - attach_workspace: - at: ~/project - - setup_google_dns - - install_node - - install_uv - - install_rust - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Python dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - - save_cache: - key: v1-uv-cache-{{ checksum "uv.lock" }} - paths: - - ~/.cache/uv - - restore_cache: - keys: - - ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - - run: - name: Install Node dependencies and Playwright - # The cimg/python:3.12-browsers image already ships the Chromium system - # libraries Playwright needs (libnss3, libatk-bridge2.0-0, libcups2, etc.). - # `--with-deps` triggers a redundant apt-get update + install that adds - # 5-10 minutes to the job and frequently stalls on flaky Ubuntu mirrors, - # so we install just the browser binary. - command: | - cd tests/e2e/ui - npm ci - npx playwright install chromium - - save_cache: - key: ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - paths: - - tests/e2e/ui/node_modules - - ~/.cache/ms-playwright - - run: - name: Use built UI bundle - command: | - rm -rf litellm/proxy/_experimental/out - cp -r ui-bundle/out litellm/proxy/_experimental/out - - wait_for_service: - url: tcp://localhost:5432 - timeout: "30" - - run: - name: Push Prisma schema - command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Seed database - command: | - PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ - -f tests/e2e/ui/fixtures/seed.sql - - run: - name: Start mock LLM server - command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py - background: true - - run: - name: Start mock Presidio server - command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py - background: true - - run: - name: Wait for mock Presidio server - command: | - for i in $(seq 1 30); do - if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi - sleep 1 - done - echo "Mock Presidio server never answered /health on port 8091" >&2 - exit 1 - - run: - name: Start LiteLLM proxy - environment: - MOCK_LLM_URL: "http://127.0.0.1:8090/v1" - DISABLE_SCHEMA_UPDATE: "true" - SERVER_ROOT_PATH: "" - # PROXY_LOGOUT_URL is inherited from the job-level environment so the - # proxy and proxyLogoutUrl.spec.ts agree on the logout target. - # LITELLM_LICENSE is forwarded from the project env so premium-gated - # UI flows can be exercised. license.spec.ts asserts the resulting - # JWT carries premium_user=true; if it ever stops being passed, that - # test fails loudly rather than silently regressing premium coverage. - command: | - LITELLM_LICENSE="$LITELLM_LICENSE" \ - uv run --no-sync python -m litellm.proxy.proxy_cli \ - --config tests/e2e/ui/fixtures/config.yml \ - --port 4000 - background: true - - run: - name: Wait for proxy to be ready - command: | - for i in $(seq 1 60); do - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true) - if [ "$HTTP_CODE" = "200" ]; then - echo "Proxy is ready" - exit 0 - fi - sleep 2 - done - echo "Proxy failed to start" - exit 1 - - run: - name: Run Playwright E2E tests - # Forward LITELLM_LICENSE so license.spec.ts can detect that the - # proxy was launched with a license and assert premium_user=true. - command: | - cd tests/e2e/ui - LITELLM_LICENSE="$LITELLM_LICENSE" \ - npx playwright test --config playwright.config.ts --workers=3 --retries=0 - no_output_timeout: 10m - - store_artifacts: - path: tests/e2e/ui/test-results - destination: e2e-test-results - - store_artifacts: - path: tests/e2e/ui/playwright-report - destination: e2e-playwright-report - - e2e_ui_testing_server_root_path: - docker: - - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2 - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 - environment: - POSTGRES_USER: e2euser - POSTGRES_PASSWORD: e2epassword - POSTGRES_DB: litellm_e2e - resource_class: large - working_directory: ~/project - environment: - DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e" - CI: "true" - # The whole job exercises the proxy mounted under a prefix. SERVER_ROOT_PATH - # is read both by the proxy at boot (to rewrite the built UI bundle in place) - # and by migration.serverRootPath.config.ts, which refuses to run without it. - SERVER_ROOT_PATH: "/litellm" - steps: - - checkout_over_https - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes: - category: client - - attach_workspace: - at: ~/project - - setup_google_dns - - install_node - - install_uv - - install_rust - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Python dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma - - save_cache: - key: v1-uv-cache-{{ checksum "uv.lock" }} - paths: - - ~/.cache/uv - - restore_cache: - keys: - - ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - - run: - name: Install Node dependencies and Playwright - command: | - cd tests/e2e/ui - npm ci - npx playwright install chromium - - save_cache: - key: ui-playwright-node-deps-v1-{{ checksum "tests/e2e/ui/package-lock.json" }} - paths: - - tests/e2e/ui/node_modules - - ~/.cache/ms-playwright - - run: - name: Use built UI bundle - command: | - rm -rf litellm/proxy/_experimental/out - cp -r ui-bundle/out litellm/proxy/_experimental/out - - wait_for_service: - url: tcp://localhost:5432 - timeout: "30" - - run: - name: Push Prisma schema - command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Seed database - command: | - PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \ - -f tests/e2e/ui/fixtures/seed.sql - - run: - name: Start mock LLM server - command: uv run --no-sync python tests/e2e/ui/fixtures/mock_llm_server/server.py - background: true - - run: - name: Start mock Presidio server - command: uv run --no-sync python tests/e2e/ui/fixtures/mock_presidio_server/server.py - background: true - - run: - name: Wait for mock Presidio server - command: | - for i in $(seq 1 30); do - if curl -sf http://127.0.0.1:8091/health >/dev/null 2>&1; then exit 0; fi - sleep 1 - done - echo "Mock Presidio server never answered /health on port 8091" >&2 - exit 1 - - run: - name: Start LiteLLM proxy under a server root path - environment: - MOCK_LLM_URL: "http://127.0.0.1:8090/v1" - DISABLE_SCHEMA_UPDATE: "true" - # Output flows to this step's own log, so a boot crash is visible here - # rather than swallowed by a downstream readiness probe. - command: | - LITELLM_LICENSE="$LITELLM_LICENSE" \ - uv run --no-sync python -m litellm.proxy.proxy_cli \ - --config tests/e2e/ui/fixtures/config.yml \ - --port 4000 - background: true - - run: - name: Wait for prefixed proxy to be ready - command: | - for i in $(seq 1 60); do - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer $LITELLM_MASTER_KEY" http://127.0.0.1:4000/litellm/health 2>/dev/null || true) - if [ "$HTTP_CODE" = "200" ]; then - echo "Prefixed proxy is ready" - exit 0 - fi - sleep 2 - done - echo "Prefixed proxy failed to start; see the 'Start LiteLLM proxy under a server root path' step for the boot log" - exit 1 - - run: - name: Run migration smoke under SERVER_ROOT_PATH - command: | - cd tests/e2e/ui - LITELLM_LICENSE="$LITELLM_LICENSE" \ - npx playwright test --config migration.serverRootPath.config.ts - no_output_timeout: 10m - - store_artifacts: - path: tests/e2e/ui/test-results - destination: e2e-server-root-path-test-results - - store_artifacts: - path: tests/e2e/ui/playwright-report - destination: e2e-server-root-path-playwright-report - - build_docker_database_image: - parameters: - migration_qualification: - type: boolean - default: false - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - when: - condition: << parameters.migration_qualification >> - steps: - - checkout_over_https: - revision: << pipeline.parameters.migration_source_sha >> - - unless: - condition: << parameters.migration_qualification >> - steps: - - checkout_over_https - - skip_if_unrelated_changes - - - run: - name: Build Docker image - environment: - MIGRATION_CANDIDATE_IMAGE: << pipeline.parameters.migration_candidate_image >> - command: | - if [ -n "$MIGRATION_CANDIDATE_IMAGE" ]; then - [[ "$MIGRATION_CANDIDATE_IMAGE" =~ ^ghcr.io/berriai/[a-z0-9._/-]+@sha256:[0-9a-f]{64}$ ]] || exit 1 - docker pull "$MIGRATION_CANDIDATE_IMAGE" - docker tag "$MIGRATION_CANDIDATE_IMAGE" litellm-docker-database:ci - else - docker build \ - --label org.opencontainers.image.revision="$(git rev-parse HEAD)" \ - -t litellm-docker-database:ci \ - -f docker/Dockerfile.database . - fi - python3 .circleci/scripts/run_migration_tests.py record-image - - - run: - name: Save Docker image to workspace root - command: | - docker save litellm-docker-database:ci | zstd -1 -T0 > litellm-docker-database.tar.zst - - - persist_to_workspace: - root: . - paths: - - litellm-docker-database.tar.zst - - migration-image.json - - migration_startup_tests: - parameters: - suite: - type: enum - enum: [startup, recovery, legacy, upgrade, shaped] - baseline: - type: boolean - default: false - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - environment: - LITELLM_MIGRATION_TESTS: "1" - LITELLM_MIGRATION_TEST_IMAGE: litellm-docker-database:ci - LITELLM_MIGRATION_BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >> - MIGRATION_TEST_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres - MIGRATION_TEST_CONTAINER_ADMIN_URL: postgresql://postgres:postgres@host.docker.internal:5432/postgres - MIGRATION_TEST_OUTPUT: /tmp/migration-results - PYTHONPATH: tests/e2e - steps: - - checkout_over_https: - revision: << pipeline.parameters.migration_source_sha >> - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - install_uv - - install_rust - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install test dependencies - command: uv sync --frozen --all-groups --all-extras --python 3.12 - - attach_workspace: - at: ~/project - - run: - name: Load the shared candidate and start PostgreSQL - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker run -d --name migration-postgres \ - -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres \ - -p 5432:5432 \ - postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - - wait_for_service: - url: tcp://localhost:5432 - timeout: "60" - - when: - condition: << parameters.baseline >> - steps: - - run: - name: Pull the baseline release the upgrade starts from - environment: - BASELINE_IMAGE: << pipeline.parameters.migration_baseline_image >> - command: | - [[ "$BASELINE_IMAGE" =~ ^ghcr.io/berriai/[a-z0-9._/-]+(@sha256:[0-9a-f]{64}|:v[0-9][0-9a-z.-]*)$ ]] || exit 1 - docker pull "$BASELINE_IMAGE" - - run: - name: Run migration startup regressions - environment: - MIGRATION_TEST_SUITE: << parameters.suite >> - MIGRATION_CANDIDATE_IMAGE: << pipeline.parameters.migration_candidate_image >> - command: | - mkdir -p /tmp/migration-results - uv run --no-sync python .circleci/scripts/run_migration_tests.py - no_output_timeout: 15m - - store_test_results: - path: /tmp/migration-results/junit - - run: - name: Package migration diagnostics - when: always - command: | - mkdir -p /tmp/migration-artifacts - if [ -d /tmp/migration-results ]; then - tar -czf /tmp/migration-artifacts/diagnostics.tar.gz -C /tmp/migration-results . - if [ -f /tmp/migration-results/verdict.json ]; then - cp /tmp/migration-results/verdict.json /tmp/migration-artifacts/verdict.json - fi - fi - - store_artifacts: - path: /tmp/migration-artifacts - destination: migration-results - - run: - name: Remove migration test containers - when: always - command: | - docker ps -aq --filter label=litellm-migration-test=true | xargs -r docker rm -f - docker rm -f migration-postgres || true - - test_bad_database_url: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: medium - working_directory: ~/project - steps: - - checkout_over_https - - skip_if_unrelated_changes - - attach_workspace: - at: ~/project - - setup_google_dns - - start_postgres - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker tag litellm-docker-database:ci myapp:latest - - run: - name: Run Docker container with bad DATABASE_URL - command: | - set +e - docker run --name my-app \ - -p 4000:4000 \ - -e LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY=true \ - -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \ - -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \ - myapp:latest \ - --port 4000 > docker_output.log 2>&1 - echo "$?" > docker_exit_code - set -e - - run: - name: Display Docker logs - command: cat docker_output.log - - run: - name: Proxy must refuse to serve on an unreachable database - command: | - fail() { echo "FAILED: $1"; cat docker_output.log; exit 1; } - exit_code="$(cat docker_exit_code)" - [ "$exit_code" -ne 0 ] || fail "proxy exited 0 with an unreachable database" - grep -q "P1001" docker_output.log || fail "log does not name the unreachable database server" - ! grep -q "Application startup complete" docker_output.log || fail "proxy reached serving state" - ! docker exec my-app true 2>/dev/null || fail "container is still running" - echo "Proxy refused to serve (exit $exit_code) and never reached startup. Test passed." - - integration_contracts: - parameters: - suite: - type: string - mode: - type: enum - enum: [standard, replica] - default: standard - parallelism: - type: integer - default: 1 - resource_class: - type: enum - enum: [large, xlarge] - default: large - machine: - image: ubuntu-2204:2024.04.1 - resource_class: << parameters.resource_class >> - working_directory: ~/project - parallelism: << parameters.parallelism >> - steps: - - setup_litellm_test_deps - - when: - condition: - equal: [browser, << parameters.suite >>] - steps: - - install_node - - restore_cache: - keys: - - integration-ui-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} - - run: - name: Install locked browser dependencies - command: | - cd ui/litellm-dashboard - npm ci - cd ../../tests/e2e/ui - npm ci - sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l \ - timeout --signal=TERM --kill-after=20s 6m node node_modules/@playwright/test/cli.js install-deps chromium - timeout --signal=TERM --kill-after=20s 3m node node_modules/@playwright/test/cli.js install chromium - - save_cache: - key: integration-ui-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }} - paths: - - ~/.npm - - ~/.cache/ms-playwright - - run: - name: Build the candidate dashboard - command: cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build - - when: - condition: - equal: [mcp, << parameters.suite >>] - steps: - - run: - name: Install MCP SDK1 peer - command: | - uv venv --python 3.12 .venv-mcp-peer - uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1' - echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV" - - when: - condition: - equal: [sdk, << parameters.suite >>] - steps: - - run: - name: Install redis-server 7.2.16 - command: | - mkdir -p /tmp/redis-bin - cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)" - docker cp "${cid}:/usr/local/bin/redis-server" /tmp/redis-bin/redis-server - docker rm "$cid" - /tmp/redis-bin/redis-server --version | grep -q 'v=7.2.16' - echo 'export PATH="/tmp/redis-bin:$PATH"' >> "$BASH_ENV" - - start_postgres: - image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - server_args: "-c max_connections=300 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000" - - start_redis - - run: - name: Run owned integration contracts - command: bash .circleci/scripts/run_integration.sh << parameters.suite >> << parameters.mode >> - no_output_timeout: 15m - - run: - name: Stop owned database and Redis - when: always - command: | - mkdir -p test-results/services-<< parameters.suite >>-<< parameters.mode >> - docker logs postgres-db > test-results/services-<< parameters.suite >>-<< parameters.mode >>/postgres.log 2>&1 || true - docker logs redis-cache > test-results/services-<< parameters.suite >>-<< parameters.mode >>/redis.log 2>&1 || true - docker rm -f postgres-db redis-cache - test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" - - when: - condition: - not: - equal: [1, << parameters.parallelism >>] - steps: - - run: - name: Prepare this shard's execution record - command: | - results="test-results/integration-<< parameters.suite >>" - if [ "<< parameters.mode >>" = replica ]; then - results="$results-replica" - fi - mkdir -p "/tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >>/$CIRCLE_NODE_INDEX" - cp "$results/execution.json" "/tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >>/$CIRCLE_NODE_INDEX/execution.json" - if [ "<< parameters.suite >>" = mcp ]; then - mkdir -p "/tmp/integration-shards/mcp-standard/$CIRCLE_NODE_INDEX/coverage" - cp test-results/integration-mcp/coverage/data "/tmp/integration-shards/mcp-standard/$CIRCLE_NODE_INDEX/coverage/data.$CIRCLE_NODE_INDEX" - fi - - persist_to_workspace: - root: /tmp/integration-shards - paths: - - << parameters.suite >>-<< parameters.mode >> - - store_test_results: - path: test-results - - store_artifacts: - path: test-results - - integration_shard_coverage: - parameters: - suite: - type: enum - enum: [management, accounting, database, extensions, providers, mcp] - mode: - type: enum - enum: [standard, replica] - default: standard - shards: - type: integer - docker: - - image: cimg/python:3.12@sha256:9c796c23c84e84a66a964acb508d39dc5433c81a47e07efd56dccbbc2427e07c - auth: - username: ${DOCKERHUB_USERNAME} - password: ${DOCKERHUB_PASSWORD} - resource_class: small - working_directory: ~/project - environment: - UV_PYTHON_DOWNLOADS: never - steps: - - checkout_over_https - - install_uv - - run: - name: Install the locked shard verifier and coverage dependencies - command: | - uv export --frozen --all-groups --all-extras --no-emit-project --no-emit-workspace --no-header --no-annotate \ - | awk '/^[^ ]/ { keep = $0 ~ /^(annotated-types|coverage|pydantic|pydantic-core|typing-extensions|typing-inspection)==/ } keep' \ - > /tmp/fan-in-requirements.txt - uv venv .venv --python 3.12 - uv pip install --no-config --python .venv --require-hashes -r /tmp/fan-in-requirements.txt - - attach_workspace: - at: /tmp/integration-shards - - run: - name: Verify every << parameters.suite >> case ran exactly once - command: | - .venv/bin/python -m tests.integration._support.sharding /tmp/integration-shards/<< parameters.suite >>-<< parameters.mode >> \ - --shards << parameters.shards >> - - when: - condition: - equal: [mcp, << parameters.suite >>] - steps: - - run: - name: Combine coverage from every MCP shard - command: | - mkdir -p test-results/integration-mcp/coverage - export COVERAGE_FILE="$PWD/test-results/integration-mcp/coverage/data" - .venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml /tmp/integration-shards/mcp-standard/*/coverage - .venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > test-results/integration-mcp/coverage/coverage.txt - .venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d test-results/integration-mcp/coverage/html - - store_artifacts: - path: test-results/integration-mcp/coverage - - routing_parity: - parameters: - suite: - type: string - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - setup_litellm_test_deps - - run: - name: Check out base product code - environment: - ROUTING_PARITY_BASE: << pipeline.parameters.routing_parity_base >> - command: | - [[ "$ROUTING_PARITY_BASE" =~ ^[0-9a-f]{40}$ ]] || exit 1 - git fetch --depth 1 --no-filter origin "$ROUTING_PARITY_BASE" - git rm -r -f --quiet litellm enterprise litellm-proxy-extras - git checkout "$ROUTING_PARITY_BASE" -- litellm enterprise litellm-proxy-extras - git reset --quiet - test -f litellm/rust_bridge/_native.abi3.so - - start_postgres: - image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000" - - start_redis - - run: - name: Run base side - command: bash .circleci/scripts/run_integration.sh << parameters.suite >> parity base - no_output_timeout: 15m - - run: - name: Stop base database and Redis - when: always - command: | - mkdir -p test-results/services-<< parameters.suite >>-parity-base - docker logs postgres-db > test-results/services-<< parameters.suite >>-parity-base/postgres.log 2>&1 || true - docker logs redis-cache > test-results/services-<< parameters.suite >>-parity-base/redis.log 2>&1 || true - docker rm -f postgres-db redis-cache - test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" - - run: - name: Check out head product code - command: | - git rm -r -f --quiet litellm enterprise litellm-proxy-extras - git checkout "$CIRCLE_SHA1" -- litellm enterprise litellm-proxy-extras - git reset --quiet - test -f litellm/rust_bridge/_native.abi3.so - - start_postgres: - image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - server_args: "-c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000" - - start_redis - - run: - name: Run head side - command: bash .circleci/scripts/run_integration.sh << parameters.suite >> parity head - no_output_timeout: 15m - - run: - name: Stop head database and Redis - when: always - command: | - mkdir -p test-results/services-<< parameters.suite >>-parity-head - docker logs postgres-db > test-results/services-<< parameters.suite >>-parity-head/postgres.log 2>&1 || true - docker logs redis-cache > test-results/services-<< parameters.suite >>-parity-head/redis.log 2>&1 || true - docker rm -f postgres-db redis-cache - test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)" - - run: - name: Compare routing parity - command: PYTHONPATH="$PWD/tests" .venv/bin/python -m integration._support.routing check test-results/parity-<< parameters.suite >>/base test-results/parity-<< parameters.suite >>/head - - store_test_results: - path: test-results - - store_artifacts: - path: test-results - - postgres_suite: - parameters: - test_path: - type: string - seed: - type: boolean - coverage_flag: - type: string - default: "" - timeout_minutes: - type: integer - workers: - type: integer - default: 1 - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - environment: - DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test - steps: - - checkout_over_https - - skip_if_unrelated_changes - - setup_litellm_test_deps: - checkout: false - - start_postgres: - db_name: litellm_test - image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - - run: - name: Generate Prisma client - command: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - - when: - condition: << parameters.seed >> - steps: - - run: - name: Seed database schema - command: uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - run: - name: Run << parameters.test_path >> - command: | - mkdir -p test-results - coverage_args=() - worker_args=() - if [ "<< parameters.workers >>" -gt 1 ]; then - worker_args=(-n << parameters.workers >> --dist=worksteal) - fi - if [ -n "<< parameters.coverage_flag >>" ]; then - coverage_args=(--cov=./litellm --cov-report=xml:coverage.xml) - fi - timeout --signal=TERM << parameters.timeout_minutes >>m uv run --no-sync pytest \ - << parameters.test_path >> -vv --tb=short --durations=10 -o junit_family=xunit1 \ - --junitxml=test-results/junit.xml "${coverage_args[@]}" "${worker_args[@]}" - - when: - condition: << parameters.coverage_flag >> - steps: - - install_codecov_cli - - run: - name: Upload coverage - when: always - command: | - [ -f coverage.xml ] || { echo "no coverage.xml produced"; exit 1; } - codecov upload-process --disable-search --fail-on-error -f coverage.xml \ - -F << parameters.coverage_flag >> -C "$CIRCLE_SHA1" \ - -n "<< parameters.coverage_flag >>-${CIRCLE_BUILD_NUM}" --git-service github - - store_test_results: - path: test-results - - store_artifacts: - path: test-results - -workflows: - migration_startup_scheduled: - triggers: - - schedule: - cron: "17 0,6,12,18 * * *" - filters: - branches: - only: main - jobs: - - build_docker_database_image: - migration_qualification: true - - migration_startup_tests: - name: migration-startup - suite: startup - requires: [build_docker_database_image] - - migration_startup_tests: - name: migration-recovery - suite: recovery - requires: [build_docker_database_image] - - migration_startup_tests: - name: migration-legacy-and-pooling - suite: legacy - requires: [build_docker_database_image] - - migration_startup_tests: - name: migration-upgrade - suite: upgrade - baseline: true - requires: [build_docker_database_image] - - migration_startup_tests: - name: migration-upgrade-shaped - suite: shaped - baseline: true - requires: [build_docker_database_image] - integration: - jobs: - - integration_contracts: - name: integration-<< matrix.suite >> - filters: &integration_default_run not pipeline.parameters.run_migration_tests and pipeline.parameters.routing_parity_base == "" - matrix: - parameters: - suite: [sdk, cost, browser] - parallelism: 1 - - integration_contracts: - name: integration-security - suite: security - parallelism: 1 - resource_class: xlarge - filters: *integration_default_run - - integration_contracts: - name: integration-management - suite: management - parallelism: 4 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-management-coverage - suite: management - shards: 4 - requires: [integration-management] - filters: *integration_default_run - - integration_contracts: - name: integration-accounting - suite: accounting - parallelism: 4 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-accounting-coverage - suite: accounting - shards: 4 - requires: [integration-accounting] - filters: *integration_default_run - - integration_contracts: - name: integration-providers - suite: providers - parallelism: 4 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-providers-coverage - suite: providers - shards: 4 - requires: - - integration-providers - filters: *integration_default_run - - integration_contracts: - name: integration-database - suite: database - parallelism: 2 - filters: *integration_default_run - - integration_shard_coverage: - name: integration-database-coverage - suite: database - shards: 2 - requires: [integration-database] - filters: *integration_default_run - - integration_contracts: - name: integration-mcp - suite: mcp - parallelism: 2 - filters: *integration_default_run - - integration_shard_coverage: - name: integration-mcp-coverage - suite: mcp - shards: 2 - requires: - - integration-mcp - filters: *integration_default_run - - integration_contracts: - name: integration-extensions - suite: extensions - parallelism: 8 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-extensions-coverage - suite: extensions - shards: 8 - requires: - - integration-extensions - filters: *integration_default_run - - integration_contracts: - name: integration-management-replica - suite: management - mode: replica - parallelism: 4 - resource_class: xlarge - filters: *integration_default_run - - integration_shard_coverage: - name: integration-management-replica-coverage - suite: management - mode: replica - shards: 4 - requires: [integration-management-replica] - filters: *integration_default_run - - integration_contracts: - name: integration-database-replica - suite: database - mode: replica - parallelism: 2 - filters: *integration_default_run - - integration_shard_coverage: - name: integration-database-replica-coverage - suite: database - mode: replica - shards: 2 - requires: [integration-database-replica] - filters: *integration_default_run - - postgres_suite: - name: roi-database - test_path: tests/integration/database/test_roi_observed.py - seed: false - coverage_flag: roi-postgres - timeout_minutes: 10 - filters: *integration_default_run - - windows_release_wheel: - filters: *integration_default_run - - base_sdk_install: - filters: *integration_default_run - - build_docker_database_image: - filters: *integration_default_run - - proxy_logging_guardrails_model_info_tests: - requires: - - build_docker_database_image - filters: *integration_default_run - - db_migration_disable_update_check: - requires: - - build_docker_database_image - filters: *integration_default_run - - test_bad_database_url: - requires: - - build_docker_database_image - filters: *integration_default_run - - helm_chart_testing: - requires: - - build_docker_database_image - filters: *integration_default_run - - e2e_ui_build: - filters: *integration_default_run - - e2e_ui_testing: - requires: [e2e_ui_build] - filters: *integration_default_run - - e2e_ui_testing_server_root_path: - requires: [e2e_ui_build] - filters: *integration_default_run - - build_docker_database_image: - name: build_docker_database_image-migration - migration_qualification: true - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-startup - suite: startup - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-recovery - suite: recovery - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-legacy-and-pooling - suite: legacy - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-upgrade - suite: upgrade - baseline: true - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - migration_startup_tests: - name: migration-upgrade-shaped - suite: shaped - baseline: true - requires: [build_docker_database_image-migration] - filters: pipeline.parameters.run_migration_tests - - routing_parity: - name: routing-parity-<< matrix.suite >> - matrix: - parameters: - suite: [management, accounting, database, providers, extensions, cost, mcp] - filters: pipeline.parameters.routing_parity_base != "" diff --git a/.circleci/scripts/run_migration_tests.py b/.circleci/scripts/run_migration_tests.py deleted file mode 100644 index 50d01f1d50b..00000000000 --- a/.circleci/scripts/run_migration_tests.py +++ /dev/null @@ -1,116 +0,0 @@ -from __future__ import annotations - -import json -import os -import re -import subprocess -import sys -from pathlib import Path -from typing import Final -from xml.etree import ElementTree - -SUITES: Final = { - "startup": (("test_startup.py",), 13), - "recovery": (("test_recovery.py",), 15), - "legacy": (("test_legacy.py", "test_pooling.py"), 11), - "upgrade": (("test_upgrade.py", "test_rolling_upgrade.py"), 5), - "shaped": (("test_shaped_database.py",), 1), -} - - -def successful_junit(path: Path, expected: int, exit_code: int) -> bool: - if exit_code != 0 or not path.is_file(): - return False - try: - root: Final = ElementTree.parse(path).getroot() - except ElementTree.ParseError: - return False - cases: Final = tuple(root.iter("testcase")) - identities: Final = frozenset((case.get("classname"), case.get("name")) for case in cases) - return len(cases) == len(identities) == expected and all( - not any(case.find(tag) is not None for tag in ("failure", "error", "skipped")) for case in cases - ) - - -def output(*command: str) -> str: - return subprocess.check_output(command, text=True, timeout=90).strip() - - -def record_image() -> None: - source: Final = output("git", "rev-parse", "HEAD") - image: Final = output("docker", "image", "inspect", "litellm-docker-database:ci", "--format", "{{.Id}}") - revision: Final = output( - "docker", - "image", - "inspect", - "litellm-docker-database:ci", - "--format", - '{{index .Config.Labels "org.opencontainers.image.revision"}}', - ) - assert re.fullmatch(r"[0-9a-f]{40}", source), "Invalid source revision" - assert revision == source, "Candidate image revision differs from the tested source" - Path("migration-image.json").write_text( - json.dumps( - { - "source_sha": source, - "image_id": image, - "candidate_image": os.environ.get("MIGRATION_CANDIDATE_IMAGE", ""), - } - ) - ) - - -def main() -> int: - suite: Final = os.environ["MIGRATION_TEST_SUITE"] - files, expected = SUITES[suite] - metadata: Final = json.loads(Path("migration-image.json").read_text()) - assert metadata["source_sha"] == output("git", "rev-parse", "HEAD"), "Image and test source revisions differ" - assert metadata["image_id"] == output( - "docker", "image", "inspect", os.environ["LITELLM_MIGRATION_TEST_IMAGE"], "--format", "{{.Id}}" - ), "Loaded image differs from the build output" - assert metadata["candidate_image"] == os.environ.get("MIGRATION_CANDIDATE_IMAGE", ""), "Wrong release candidate" - destination: Final = Path(os.environ["MIGRATION_TEST_OUTPUT"]) - junit: Final = destination / "junit" / "results.xml" - junit.parent.mkdir(parents=True, exist_ok=True) - result: Final = subprocess.run( - ( - sys.executable, - "-m", - "pytest", - *(f"tests/e2e/migrations/{name}" for name in files), - "-vv", - "--tb=short", - "--durations=10", - f"--junitxml={junit}", - "-o", - "addopts=", - "--reruns=0", - ), - check=False, - timeout=1200, - ) - passed: Final = successful_junit(junit, expected, result.returncode) - (destination / "verdict.json").write_text( - json.dumps( - { - **metadata, - "suite": suite, - "baseline_image": os.environ.get("LITELLM_MIGRATION_BASELINE_IMAGE", ""), - "expected_cases": expected, - "passed": passed, - "pytest_exit_code": result.returncode, - "test_revision": metadata["source_sha"], - "workflow_id": os.environ.get("CIRCLE_WORKFLOW_ID", ""), - "job_number": os.environ.get("CIRCLE_BUILD_NUM", ""), - }, - indent=2, - ) - ) - return 0 if passed else 1 - - -if __name__ == "__main__": - if sys.argv[1:] == ["record-image"]: - record_image() - else: - raise SystemExit(main()) diff --git a/.dockerignore b/.dockerignore index f3a80fee3e4..d8eb72a057f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,13 +1,11 @@ docs -cookbook -.circleci +litellm/proxy/discovery_endpoints/skill_scanner_adapter .github tests .git .devcontainer *.tgz log.txt -docker/Dockerfile.* # Claude Flow generated files (must be excluded from Docker build) .claude/ diff --git a/.github/AGENTS.md b/.github/AGENTS.md new file mode 100644 index 00000000000..14bd461a8a0 --- /dev/null +++ b/.github/AGENTS.md @@ -0,0 +1,5 @@ +# CI + +Integration groups and replica runs remain in `workflows/test-integration.yml`. Do not duplicate them in another workflow. + +CI helpers used by existing workflows live in `scripts/ci/`. Jobs that need merge bases check out full history. Removing a CI job does not count its tests as covered. List tests that no remaining job invokes explicitly in `ci-coverage-allowlist.yml`. diff --git a/.github/PULL_REQUEST_TEMPLATE/general.md b/.github/PULL_REQUEST_TEMPLATE/general.md index 4f1a18e925e..f815601a29c 100644 --- a/.github/PULL_REQUEST_TEMPLATE/general.md +++ b/.github/PULL_REQUEST_TEMPLATE/general.md @@ -6,19 +6,12 @@ ## Handwritten 1-3 sentence description - -`` - -`` +[insert handwritten description here]
Details @@ -100,6 +93,7 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac Otherwise proof must be fully e2e, no mocks, with real provider calls where applicable. Pytest alone is insufficient. The following format applies to live proof Show only the latest proof. Before uses the merge base and After the tested tip. Rerun affected QA after behavior changes. Verified neutral diffs may carry proof with the tested SHA, current tip, equivalence reason and focused checks, following AGENTS.md and /qa Use matching Before/After case headings with numbered commands and observed output. Put shared setup above them. For one case, omit case headings + A reviewer must be able to rerun every case from the body alone, so the shared setup gives the proxy start command, the config.yaml, env var names (never values) and the client code or curl used, all exactly as run. Wrap any block longer than about 20 lines in a collapsed
with a one-line summary ### Before () diff --git a/.github/actions/detect-changes/action.yml b/.github/actions/detect-changes/action.yml index b0f9b72f0ee..59c6ab28c30 100644 --- a/.github/actions/detect-changes/action.yml +++ b/.github/actions/detect-changes/action.yml @@ -1,6 +1,6 @@ name: "Detect relevant changes" description: >- - Classify the pull request's changed files with .circleci/scripts/classify_changes.sh + Classify the pull request's changed files with .github/scripts/ci/classify_changes.sh and expose decision=run|skip for one category. backend means anything outside ui/, docs/ and markdown; ui means the dashboard sources alone. decision=skip lets callers short-circuit expensive steps while the job still completes successfully and satisfies diff --git a/.github/ci-coverage-allowlist.yml b/.github/ci-coverage-allowlist.yml index 5dffa78776d..3a714187325 100644 --- a/.github/ci-coverage-allowlist.yml +++ b/.github/ci-coverage-allowlist.yml @@ -4,6 +4,16 @@ description: >- by a job nor listed here, so every entry below is a decision on the record. test_paths: + - reason: ran only in CircleCI, removed + paths: + - tests/e2e/migrations/test_legacy.py + - tests/e2e/migrations/test_pooling.py + - tests/e2e/migrations/test_recovery.py + - tests/e2e/migrations/test_rolling_upgrade.py + - tests/e2e/migrations/test_shaped_database.py + - tests/e2e/migrations/test_startup.py + - tests/e2e/migrations/test_upgrade.py + - tests/integration/database/test_roi_observed.py - reason: >- litellm.agent() end-to-end suite. It drives the real claude, codex, opencode and pi CLIs and deepagents against a live LiteLLM AI Gateway, so it needs those binaries on PATH plus @@ -19,7 +29,7 @@ test_paths: - tests/rust-python-harness - reason: >- Owned by the open pull request that moves the Rust/Python parity checks (#45135). It ran in - the CircleCI build_and_test job that globbed tests/test_*.py, which this migration deletes, + the retired build_and_test job that globbed tests/test_*.py, and that pull request gives it a new home paths: - tests/test_rust_python_harness.py @@ -57,15 +67,6 @@ test_paths: - tests/integration/test_oci_proxy_integration.py dockerfiles: - - reason: >- - The dashboard container is a static Next.js export served by nginx, and the dashboard build - and lint workflows already exercise that output, so building the image adds no signal about it - paths: - - ui/Dockerfile - - reason: >- - An example image under cookbook/ that is documentation rather than a shipped artifact - paths: - - cookbook/litellm-ollama-docker-image/Dockerfile - reason: >- The Rust gateway image compiles the whole workspace in release mode, which is too slow for a per-pull-request job while the gateway binary is still being assembled; the Rust lint, diff --git a/.github/scripts/assert_ci_coverage.py b/.github/scripts/assert_ci_coverage.py index b442948c3ca..b443fd9e0d9 100644 --- a/.github/scripts/assert_ci_coverage.py +++ b/.github/scripts/assert_ci_coverage.py @@ -10,6 +10,7 @@ import warnings from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from functools import lru_cache +from itertools import chain from types import MappingProxyType from typing import Final @@ -17,7 +18,6 @@ import yaml REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] WORKFLOW_DIR = REPO_ROOT / ".github" / "workflows" -CIRCLECI_CONFIG = REPO_ROOT / ".circleci" / "config.yml" ALLOWLIST_FILE = REPO_ROOT / ".github" / "ci-coverage-allowlist.yml" TESTS_ROOT = REPO_ROOT / "tests" @@ -25,14 +25,18 @@ ALLOWLIST_KEYS = frozenset({"description", "test_paths", "dockerfiles"}) PATH_FILTER_KEYS = frozenset({"paths", "paths-ignore"}) TEST_PATH_KEYS = frozenset({"test-path", "test-paths", "test_path"}) DOCKERFILE_INPUT_KEYS = frozenset({"file", "dockerfile"}) -TEST_RUNNER_RE = re.compile(r"\bpytest\b|\bcircleci tests\b|\bhelm unittest\b|\bplaywright test\b|\bpython[0-9.]*\s") +TEST_RUNNER_RE = re.compile(r"\bpytest\b|\bhelm unittest\b|\bplaywright test\b|\bpython[0-9.]*\s") IMAGE_BUILD_RE = re.compile(r"\bdocker\s+(?:buildx\s+)?build\b") TEST_TOKEN_RE = re.compile(r"tests/[A-Za-z0-9_./*?-]+") IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)") DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*") COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE) GLOB_CHARS = frozenset("*?") -DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType({"image-scan.yml": "tests/integration/database/image/"}) +DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType( + { + "image-scan.yml": "tests/integration/database/image/", + } +) # Trees whose jobs are sharded with no catch-all bucket, so every child that holds # tests has to be named by some shard or it runs nowhere. A child listed here is @@ -115,9 +119,7 @@ def _scalars(node: object, key: str) -> tuple[Scalar, ...]: def _config_files() -> tuple[pathlib.Path, ...]: - workflows = tuple(sorted(path for path in WORKFLOW_DIR.iterdir() if path.suffix in (".yml", ".yaml"))) - circleci = (CIRCLECI_CONFIG,) if CIRCLECI_CONFIG.is_file() else () - return workflows + circleci + return tuple(sorted(path for path in WORKFLOW_DIR.iterdir() if path.suffix in (".yml", ".yaml"))) def _all_scalars() -> tuple[Scalar, ...]: @@ -243,7 +245,6 @@ def _describe(paths: tuple[str, ...]) -> str: return f"{len(paths)} test file(s) invoked by no job: {names}{suffix}" -GLOB_CALL_RE = re.compile(r'circleci tests glob "([^"]+)"') KEYWORD_RE = re.compile(r"-k\s+\\?[\"']([^\"'\\]+)") @@ -316,16 +317,14 @@ def _keyword_terms( def _slices() -> tuple[Slice, ...]: - if not CIRCLECI_CONFIG.exists(): - return () test_files: Final = frozenset(_test_files()) - jobs: Final = yaml.safe_load(CIRCLECI_CONFIG.read_text()).get("jobs", {}) return tuple( - Slice(job=job, globs=globs, named=named, required=required, excluded=excluded, understood=understood) - for job, body in jobs.items() + Slice(job=f"{path.name}:{job}", globs=globs, named=named, required=required, excluded=excluded, understood=understood) + for path in _config_files() + for job, body in yaml.safe_load(path.read_text()).get("jobs", {}).items() for text in ("\n".join(_strings(body)),) if "pytest" in text - for globs in (tuple(GLOB_CALL_RE.findall(text)),) + for globs in (tuple(token for token in TEST_TOKEN_RE.findall(text) if GLOB_CHARS & set(token)),) for named in (frozenset(TEST_TOKEN_RE.findall(text)) & test_files,) for required, excluded, understood in ( _keyword_terms(tuple(KEYWORD_RE.findall(text)), attributable=len(globs) < 2), @@ -352,16 +351,11 @@ def _matchable_names(relative_path: str) -> frozenset[str]: def _workflow_named_tokens() -> frozenset[str]: - """Test tokens a GitHub Actions job names directly. - - A CircleCI `-k` that deselects a file no longer means the file runs nowhere once a - workflow names it, so the slice check has to credit those the same way the census does. - """ return _invoked_test_tokens( scalar for path in _config_files() - if path != CIRCLECI_CONFIG for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name) + if not KEYWORD_RE.search(scalar.value) ) @@ -566,13 +560,28 @@ def _owns(workflow: str, relative_path: str) -> bool: return owned_root is not None and relative_path.startswith(owned_root) +def _expression_suites(value: str) -> tuple[str, ...]: + suites: Final = chain.from_iterable(map(json.loads, re.findall(r"'(\[[^']*\])'", value))) + return tuple(suite for suite in suites if isinstance(suite, str) and "{" not in suite) + + +def _matrix_suites(value: object) -> tuple[str, ...]: + if isinstance(value, list): + return tuple(suite for suite in value if isinstance(suite, str)) + if isinstance(value, str): + return _expression_suites(value) + return () + + def _tokens_outside_owner(relative_path: str, workflow_tokens: Mapping[str, frozenset[str]]) -> frozenset[str]: return frozenset[str]().union( *(tokens for workflow, tokens in workflow_tokens.items() if not _owns(workflow, relative_path)) ) -def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]: +def _integration_ownership( + repo_root: pathlib.Path = REPO_ROOT, *, allowlist: Allowlist | None = None +) -> tuple[frozenset[str], tuple[Finding, ...]]: runner: Final = repo_root / "tests/integration/run.py" if not runner.exists(): return frozenset(), () @@ -589,26 +598,19 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens browser_manifest: Final = repo_root / "tests/e2e/ui/tests/integrationCritical/expected.json" browser_nodes: Final = json.loads(browser_manifest.read_text()) if browser_manifest.exists() else () browser_paths: Final = frozenset(node.split("::", 1)[0] for node in browser_nodes) - circle_path: Final = repo_root / ".circleci/config.yml" - circle: Final = yaml.safe_load(circle_path.read_text()) if circle_path.exists() else {} - circle_test_path_tokens: Final = _invoked_test_tokens( - scalar for scalar in _scalars(circle, "config.yml") if scalar.key == "test_path" - ) - steps: Final = circle.get("jobs", {}).get("integration_contracts", {}).get("steps", ()) + integration_path: Final = repo_root / ".github/workflows/test-integration.yml" + integration: Final = yaml.safe_load(integration_path.read_text()) if integration_path.exists() else {} + integration_jobs: Final = integration.get("jobs", {}) invoked: Final = any( - ".circleci/scripts/run_integration.sh" in scalar.value - for scalar in _scalars(steps, "integration_contracts") - if scalar.key == "command" + ".github/scripts/ci/run_integration.sh" in scalar.value + for scalar in _scalars(integration_jobs, "integration") + if scalar.key == "run" ) scheduled: Final = frozenset( suite - for job in circle.get("workflows", {}).get("integration", {}).get("jobs", ()) - if isinstance(job, dict) and "integration_contracts" in job - for suite in job["integration_contracts"] - .get("matrix", {}) - .get("parameters", {}) - .get("suite", (job["integration_contracts"].get("suite"),)) - if isinstance(suite, str) + for body in integration_jobs.values() + if any(".github/scripts/ci/run_integration.sh" in scalar.value for scalar in _scalars(body, "integration")) + for suite in _matrix_suites(body.get("strategy", {}).get("matrix", {}).get("suite", ())) ) required: Final = (frozenset({"browser"}) if browser_paths else frozenset()) | frozenset( group @@ -630,19 +632,27 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens } ) gha_tokens: Final = frozenset[str]().union(*workflow_tokens.values()) - circle_owned: Final = ( + invoked_github_files: Final = frozenset( + path for path in github_files if any(_token_covers(token, path) for token in gha_tokens) + ) + grouped: Final = ( frozenset(str(path.relative_to(repo_root)) for path in integration_root.rglob("test_*.py")) - github_files ) findings: Final = ( tuple( Finding(path, "integration file outside GITHUB_FILES is selected by GitHub Actions") - for path in sorted(circle_owned) - if any(_token_covers(token, path) for token in gha_tokens) + for path in sorted(grouped) + if any( + _token_covers(token, path) + for workflow, tokens in workflow_tokens.items() + if workflow != "test-integration.yml" + for token in tokens + ) ) + tuple( Finding(path, "GitHub-owned integration contract has no invoking job") for path in sorted(github_files) - if not any(_token_covers(token, path) for token in gha_tokens | circle_test_path_tokens) + if path not in invoked_github_files and not (allowlist is not None and allowlist.covers_test(path)) ) + tuple( Finding(path, "GitHub-owned integration file is missing") @@ -658,6 +668,7 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens browser_commands: Final = tuple( scalar.value for path in (repo_root / ".github/workflows").glob("*.y*ml") + if path.name != "test-integration.yml" for scalar in _scalars(yaml.safe_load(path.read_text()), path.name) if scalar.key in {"run", "command"} ) @@ -690,14 +701,14 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens else () ) group_findings: Final = tuple( - Finding(group, "canonical integration group is not scheduled by CircleCI") + Finding(group, "canonical integration group is not scheduled by GitHub Actions") for group in sorted(required - scheduled) ) + tuple(Finding(path, "canonical node must have exactly one integration group") for path in sorted(ungrouped)) if not paths or not invoked or not scheduled: return frozenset(), findings + ( - Finding(str(runner.relative_to(repo_root)), "dedicated CircleCI runner is missing"), + Finding(str(runner.relative_to(repo_root)), "dedicated GitHub Actions runner is missing"), ) - return paths | browser_paths | github_files, findings + group_findings + browser_findings + exclusion_findings + return paths | browser_paths | invoked_github_files, findings + group_findings + browser_findings + exclusion_findings def main() -> int: @@ -709,7 +720,7 @@ def main() -> int: allowlist = _load_allowlist() scalars = _all_scalars() - integration_paths, ownership_findings = _integration_ownership() + integration_paths, ownership_findings = _integration_ownership(allowlist=allowlist) test_findings = _uncovered_tests( allowlist, _invoked_selections(scalars) + (Selection(included=integration_paths, ignored=frozenset()),), @@ -731,8 +742,8 @@ def main() -> int: ) if ownership_findings: browser_remedy = ( - " CircleCI's browser group owns the integrationCritical Playwright contracts, so no GitHub workflow may " - "select them and the default Playwright config must exclude them." + " The browser group in test-integration.yml owns the integrationCritical Playwright contracts. " + "Other workflows and the default Playwright config must exclude them." if any("browser" in finding.detail or "Playwright" in finding.detail for finding in ownership_findings) else "" ) diff --git a/.circleci/scripts/classify_changes.sh b/.github/scripts/ci/classify_changes.sh similarity index 86% rename from .circleci/scripts/classify_changes.sh rename to .github/scripts/ci/classify_changes.sh index a33695d0dbe..82851fa0a82 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.github/scripts/ci/classify_changes.sh @@ -14,22 +14,22 @@ outside_cost_map_set=false while IFS= read -r file || [ -n "$file" ]; do [ -n "$file" ] || continue case "$file" in - litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .github/workflows/test-unit.yml | .github/actions/detect-changes/* | .github/scripts/detect_changes.sh | .circleci/scripts/classify_changes.sh | tests/integration/run.py | pyproject.toml | uv.lock) + litellm/_redis.py | litellm/_redis_credential_provider.py | litellm/caching/redis_cache.py | litellm/caching/evicted_client_closer.py | tests/unit/test_redis.py | tests/integration/sdk/test_redis_cluster_iam_auth.py | tests/unit/caching/test_redis_connection_pool.py | tests/unit/caching/test_redis_cluster_cache.py | tests/unit/caching/test_evicted_client_closer.py | .github/workflows/test-unit.yml | .github/actions/detect-changes/* | .github/scripts/detect_changes.sh | .github/scripts/ci/classify_changes.sh | tests/integration/run.py | pyproject.toml | uv.lock) has_redis_compat=true ;; esac case "$file" in *.md | *.mdx) : ;; - pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .circleci/scripts/classify_changes.sh | tests/unit/test_circleci_path_filter.py | tests/unit/test_detect_changes.py) + pyproject.toml | */pyproject.toml | uv.lock | uv.toml | .python-version | rust-toolchain.toml | litellm-rust/* | litellm/__init__.py | litellm/proxy/proxy_server.py | litellm/*mcp* | tests/*mcp* | litellm/integrations/arize/* | tests/base_sdk_tests/* | scripts/check_mcp_sdk_install.py | .github/workflows/test-mcp-dependency-resolution.yml | .github/actions/detect-changes/* | .github/actions/setup-uv-with-retries/* | .github/actions/cache-cargo-build/* | .github/scripts/detect_changes.sh | .github/scripts/uv_sync_with_retries.sh | .github/scripts/ci/classify_changes.sh | tests/unit/test_ci_path_filter.py | tests/unit/test_detect_changes.py) has_mcp_dependencies=true ;; esac case "$file" in - litellm-rust/* | litellm/rust_bridge/* | rust-toolchain.toml | pyproject.toml | uv.lock | tests/windows_tests/* | .circleci/*) + litellm-rust/* | litellm/rust_bridge/* | rust-toolchain.toml | pyproject.toml | uv.lock | tests/windows_tests/* | .github/scripts/ci/* | .github/actions/*) has_windows_release=true ;; esac case "$file" in ui/* | tests/e2e/ui/*) has_client=true ;; docs/* | *.md | *.mdx) : ;; - .github/* | .circleci/*) has_ci=true; has_backend=true ;; + .github/*) has_ci=true; has_backend=true ;; *) has_backend=true ;; esac case "$file" in diff --git a/.circleci/scripts/path_filter.sh b/.github/scripts/ci/path_filter.sh similarity index 78% rename from .circleci/scripts/path_filter.sh rename to .github/scripts/ci/path_filter.sh index c3cf6e566ff..e8ac6e5b918 100755 --- a/.circleci/scripts/path_filter.sh +++ b/.github/scripts/ci/path_filter.sh @@ -6,10 +6,12 @@ here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" run_full() { echo "path-filter[$category]: running job ($1)" + echo "decision=run" + [ -z "${GITHUB_OUTPUT:-}" ] || echo "decision=run" >> "$GITHUB_OUTPUT" exit 0 } -[ -n "${CIRCLE_PULL_REQUEST:-}" ] || run_full "not a pull request" +[ "${GITHUB_EVENT_NAME:-}" = pull_request ] || run_full "not a pull request" candidate_bases="${PATH_FILTER_BASE_BRANCH:-main}" merge_base="" @@ -36,5 +38,6 @@ if [ "$decision" = run ]; then run_full "$category-relevant changes detected" fi -echo "path-filter[$category]: only unrelated changes detected; halting job as successful" -circleci-agent step halt +echo "path-filter[$category]: only unrelated changes detected" +echo "decision=skip" +[ -z "${GITHUB_OUTPUT:-}" ] || echo "decision=skip" >> "$GITHUB_OUTPUT" diff --git a/.circleci/scripts/prepare_replica_roles.py b/.github/scripts/ci/prepare_replica_roles.py similarity index 100% rename from .circleci/scripts/prepare_replica_roles.py rename to .github/scripts/ci/prepare_replica_roles.py diff --git a/.circleci/scripts/run_integration.sh b/.github/scripts/ci/run_integration.sh similarity index 92% rename from .circleci/scripts/run_integration.sh rename to .github/scripts/ci/run_integration.sh index fb91bcce77f..ed405634796 100644 --- a/.circleci/scripts/run_integration.sh +++ b/.github/scripts/ci/run_integration.sh @@ -30,7 +30,7 @@ egress_cgroup=litellm-integration cleanup() { original_status=$? trap - EXIT INT TERM - sudo .venv/bin/python .circleci/scripts/stop_integration_processes.py \ + sudo .venv/bin/python .github/scripts/ci/stop_integration_processes.py \ "$integration_identity" "$(id -u)" "$proxy_pid" "$peer_pid" "$upstream_pid" \ > "$results/process-cleanup.txt" 2>&1 || original_status=1 for owned_pid in "$peer_pid" "$proxy_pid" "$upstream_pid"; do @@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out" test -f "$LITELLM_UI_PATH/index.html" fi -export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')" +export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')" export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED" uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1 @@ -93,7 +93,7 @@ export INTEGRATION_PROXY_DATABASE_URL="" export INTEGRATION_PROXY_READ_REPLICA_URL="" export INTEGRATION_ROUTING="" if [ "$mode" = replica ] || [ "$mode" = parity ]; then - .venv/bin/python .circleci/scripts/prepare_replica_roles.py > "$results/prepare-replica-roles.log" 2>&1 + .venv/bin/python .github/scripts/ci/prepare_replica_roles.py > "$results/prepare-replica-roles.log" 2>&1 export INTEGRATION_PROXY_DATABASE_URL="postgresql://litellm_writer:litellm-writer@127.0.0.1:5432/circle_test" export INTEGRATION_PROXY_READ_REPLICA_URL="postgresql://litellm_reader:litellm-reader@127.0.0.1:5432/circle_test" fi @@ -202,7 +202,7 @@ start_proxy() { if [ "$isolated_workers" = false ]; then start_proxy 4000 proxy.log proxy_pid="$launched_pid" - .venv/bin/python .circleci/scripts/wait_integration_services.py + .venv/bin/python .github/scripts/ci/wait_integration_services.py curl --noproxy '*' -sSf -X POST "$INTEGRATION_PROXY_URL/config/update" \ -H "Authorization: Bearer $LITELLM_MASTER_KEY" -H 'Content-Type: application/json' \ -d '{"router_settings": {"num_retries": 0}}' > "$results/seed-router-settings.json" @@ -210,7 +210,7 @@ if [ "$isolated_workers" = false ]; then export INTEGRATION_PEER_URL=http://127.0.0.1:4001 start_proxy 4001 peer.log peer_pid="$launched_pid" - .venv/bin/python .circleci/scripts/wait_integration_services.py + .venv/bin/python .github/scripts/ci/wait_integration_services.py fi fi @@ -233,14 +233,13 @@ if [ "$suite" = browser ]; then E2E_UI_BASE_URL="$E2E_UI_BASE_URL" E2E_UI_ARTIFACT_DIR="$E2E_UI_ARTIFACT_DIR" \ LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" CI=true \ node tests/e2e/ui/node_modules/@playwright/test/cli.js test --config tests/e2e/ui/integration.config.ts - .venv/bin/python .circleci/scripts/verify_integration_browser.py "$results/browser-results.json" + .venv/bin/python .github/scripts/ci/verify_integration_browser.py "$results/browser-results.json" exit 0 fi shard_arguments=() -if [ "${CIRCLE_NODE_TOTAL:-1}" -gt 1 ]; then - .venv/bin/python tests/integration/run.py "$suite" --list \ - | circleci tests split --split-by=timings --timings-type=filename > "$results/node-files.txt" +if [ -n "${INTEGRATION_SHARD_FILES:-}" ]; then + cp "$INTEGRATION_SHARD_FILES" "$results/node-files.txt" shard_arguments=(--shard-files "$results/node-files.txt") fi @@ -265,7 +264,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \ "${license_env[@]}" \ INTEGRATION_SEED="$INTEGRATION_SEED" \ INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \ - INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \ + INTEGRATION_CASE_SHARDS="${INTEGRATION_CASE_SHARDS:-1}" INTEGRATION_CASE_SHARD_INDEX="${INTEGRATION_CASE_SHARD_INDEX:-0}" \ LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \ INTEGRATION_PROXY_DATABASE_URL="$INTEGRATION_PROXY_DATABASE_URL" \ INTEGRATION_PROXY_READ_REPLICA_URL="$INTEGRATION_PROXY_READ_REPLICA_URL" \ diff --git a/.circleci/scripts/stop_integration_processes.py b/.github/scripts/ci/stop_integration_processes.py similarity index 100% rename from .circleci/scripts/stop_integration_processes.py rename to .github/scripts/ci/stop_integration_processes.py diff --git a/.circleci/scripts/verify_integration_browser.py b/.github/scripts/ci/verify_integration_browser.py similarity index 94% rename from .circleci/scripts/verify_integration_browser.py rename to .github/scripts/ci/verify_integration_browser.py index 4468fadbcde..452a6cd43ae 100644 --- a/.circleci/scripts/verify_integration_browser.py +++ b/.github/scripts/ci/verify_integration_browser.py @@ -31,7 +31,7 @@ def main() -> None: result: Final = json.loads(Path(sys.argv[1]).read_text()) assert not result.get("errors"), result.get("errors") expected: Final = json.loads( - (Path(__file__).resolve().parents[2] / "tests/e2e/ui/tests/integrationCritical/expected.json").read_text() + (Path(__file__).resolve().parents[3] / "tests/e2e/ui/tests/integrationCritical/expected.json").read_text() ) assert expected and result["stats"]["expected"] == len(expected) assert all(result["stats"][name] == 0 for name in ("unexpected", "flaky", "skipped")) @@ -41,7 +41,7 @@ def main() -> None: suites: Final = TypeAdapter(list[BrowserSuite]).validate_python(result["suites"], strict=True) specs: Final = tuple(spec for suite in suites for spec in cases(suite)) - repository: Final = Path(__file__).resolve().parents[2] + repository: Final = Path(__file__).resolve().parents[3] report_root: Final = Path(result["config"]["rootDir"]) assert report_root.is_absolute(), "Playwright rootDir must be explicit" observed: Final = tuple( diff --git a/.circleci/scripts/wait_integration_services.py b/.github/scripts/ci/wait_integration_services.py similarity index 100% rename from .circleci/scripts/wait_integration_services.py rename to .github/scripts/ci/wait_integration_services.py diff --git a/.github/scripts/detect_changes.sh b/.github/scripts/detect_changes.sh index 2d427c92fb5..acc19be57f1 100755 --- a/.github/scripts/detect_changes.sh +++ b/.github/scripts/detect_changes.sh @@ -16,7 +16,7 @@ run_full() { } here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -classify="${here}/../../.circleci/scripts/classify_changes.sh" +classify="${here}/ci/classify_changes.sh" [ -n "${PR_NUMBER:-}" ] || run_full "not a pull_request event" [ -n "${REPO:-}" ] || run_full "no repository in the environment" diff --git a/.github/scripts/verify_linux_native_wheel.py b/.github/scripts/verify_linux_native_wheel.py index 4c4cb399dce..571092263ed 100644 --- a/.github/scripts/verify_linux_native_wheel.py +++ b/.github/scripts/verify_linux_native_wheel.py @@ -214,7 +214,7 @@ def main( native_module: Final = load_native_module(native_path) native_module_loads: Final = native_module is not None panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test") - native_size_limit: Final = 48_000_000 + native_size_limit: Final = 55_000_000 native_size_within_limit: Final = native_member.file_size <= native_size_limit validations: Final = ( (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG), diff --git a/.github/workflows/compat-matrix-image.yml b/.github/workflows/compat-matrix-image.yml index ddd2aa1a50b..73a77c1e861 100644 --- a/.github/workflows/compat-matrix-image.yml +++ b/.github/workflows/compat-matrix-image.yml @@ -26,17 +26,24 @@ jobs: with: persist-credentials: false - - name: Build the Render cron image - run: docker build -f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix:${{ github.sha }} tests/e2e - - - name: Resolve and install the Claude Code CLI as the cron user + - name: Install the pinned uv the cron image ships + env: + UV_VERSION: 0.10.9 + UV_SHA256: 20d79708222611fa540b5c9ed84f352bcd3937740e51aacc0f8b15b271c57594 run: | - docker run --rm compat-matrix:${{ github.sha }} bash -c ' - set -euo pipefail - whoami - gh --version - uv --version - version="$(uv run --no-project --python 3.12 python /opt/litellm/tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)" - /opt/litellm/tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" /tmp/claude-cli - /tmp/claude-cli/claude --version - ' + set -euo pipefail + curl -fsSLo "$RUNNER_TEMP/uv.tar.gz" "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" + echo "${UV_SHA256} $RUNNER_TEMP/uv.tar.gz" | sha256sum -c - + mkdir -p "$RUNNER_TEMP/bin" + tar -xzf "$RUNNER_TEMP/uv.tar.gz" -C "$RUNNER_TEMP/bin" --strip-components=1 uv-x86_64-unknown-linux-gnu/uv + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + + - name: Resolve and install the Claude Code CLI as an unprivileged user + run: | + set -euo pipefail + whoami + gh --version + uv --version + version="$(uv run --no-project --python 3.12 python tests/e2e/cli_harness/npm_version_resolver.py @anthropic-ai/claude-code)" + tests/e2e/claude_code/cron_vm/install_claude_code.sh "${version}" "$RUNNER_TEMP/claude-cli" + "$RUNNER_TEMP/claude-cli/claude" --version diff --git a/.github/workflows/image-scan.yml b/.github/workflows/image-scan.yml index 3e2bca8779d..54051104cb9 100644 --- a/.github/workflows/image-scan.yml +++ b/.github/workflows/image-scan.yml @@ -8,31 +8,25 @@ on: - "litellm_**" paths: - Dockerfile - - docker/Dockerfile.non_root - - docker/Dockerfile.database - - migrations/Dockerfile + - .dockerignore + - docker-entrypoint.sh - migrations/run.py - - gateway/Dockerfile - - gateway/launch.py - gateway/main.py + - gateway/launch.py - gateway/routes/allowlist.py - litellm_cli/** - - backend/Dockerfile - backend/main.py + - backend/routes/allowlist.py - litellm-rust/** - litellm/proxy/lens/** - - docker/component_entrypoint.sh - - docker/entrypoint.sh - litellm/proxy/prisma_migration.py - litellm/proxy/admin_mcp.py - litellm/proxy/proxy_server.py - - backend/routes/allowlist.py - pyproject.toml - litellm-proxy-extras/** - tests/integration/database/image/** - uv.lock - ui/litellm-dashboard/package-lock.json - - ui/Dockerfile - ui/nginx.conf - .github/workflows/image-scan.yml - .grype.yaml @@ -46,14 +40,18 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +env: + IMAGE: litellm-image-scan:${{ github.sha }} + ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }} + jobs: - image-scan: - name: image-scan + build: + name: build runs-on: ubuntu-latest if: >- github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - timeout-minutes: 30 + timeout-minutes: 60 permissions: contents: read steps: @@ -61,7 +59,129 @@ jobs: with: persist-credentials: false + - name: Build the image + run: docker build --build-arg MATURIN_PEP517_ARGS="--profile ci" --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -t "$IMAGE" . + + - name: Save the image for the verify matrix + run: docker save "$IMAGE" | zstd -T0 -3 -o "$RUNNER_TEMP/litellm-image.tar.zst" + + - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: litellm-image + path: ${{ runner.temp }}/litellm-image.tar.zst + retention-days: 1 + compression-level: 0 + + - name: Build the Admin MCP schema image + run: docker build --build-arg MATURIN_PEP517_ARGS="--profile ci" --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci --target builder -t "$ADMIN_MCP_SCHEMA_IMAGE" . + + - name: Save the Admin MCP schema image for the verify matrix + run: docker save "$ADMIN_MCP_SCHEMA_IMAGE" | zstd -T0 -3 -o "$RUNNER_TEMP/litellm-admin-mcp-schema.tar.zst" + + - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: litellm-admin-mcp-schema + path: ${{ runner.temp }}/litellm-admin-mcp-schema.tar.zst + retention-days: 1 + compression-level: 0 + + verify: + name: verify (${{ matrix.check }}) + needs: build + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + check: [scan, migrations, gateway, backend, ui, collector] + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: litellm-image + path: ${{ runner.temp }} + + - name: Load the image + run: zstd -d --stdout "$RUNNER_TEMP/litellm-image.tar.zst" | docker load + + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + if: matrix.check == 'migrations' || matrix.check == 'backend' + with: + name: litellm-admin-mcp-schema + path: ${{ runner.temp }} + + - name: Load the Admin MCP schema image + if: matrix.check == 'migrations' || matrix.check == 'backend' + run: zstd -d --stdout "$RUNNER_TEMP/litellm-admin-mcp-schema.tar.zst" | docker load + + - name: Set up Python + if: matrix.check != 'scan' + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install pytest + if: matrix.check != 'scan' + run: python -m pip install "pytest==9.0.3" + + # The prisma bake must migrate a fresh DB with no egress as an arbitrary + # non-root uid (OpenShift restricted-v2 / air-gapped / readOnlyFilesystem). + # `docker run` as the default uid with network hides a broken bake because + # the migration entrypoint exits 0 even when it applied nothing; asserting + # the schema was created is what catches it. + - name: Verify offline migration as a non-root uid + if: matrix.check == 'migrations' + env: + LITELLM_IMAGE: ${{ env.IMAGE }} + LITELLM_ADMIN_MCP_SCHEMA_IMAGE: ${{ env.ADMIN_MCP_SCHEMA_IMAGE }} + run: | + python -m pytest --noconftest -v \ + tests/integration/database/image/test_offline_image_migration.py \ + tests/integration/database/image/test_image_bedrock_realtime_extra.py \ + tests/integration/database/image/test_image_admin_mcp.py + + - name: Verify the gateway component serves offline as a non-root uid + if: matrix.check == 'gateway' + env: + LITELLM_IMAGE: ${{ env.IMAGE }} + LITELLM_COMPONENT: gateway + LITELLM_IMAGE_COMPONENT: gateway + run: | + python -m pytest --noconftest -v \ + tests/integration/database/image/test_component_image_serves_offline.py \ + tests/integration/database/image/test_image_admin_mcp.py + + - name: Verify the backend component serves offline as a non-root uid + if: matrix.check == 'backend' + env: + LITELLM_IMAGE: ${{ env.IMAGE }} + LITELLM_COMPONENT: backend + LITELLM_IMAGE_COMPONENT: backend + LITELLM_ADMIN_MCP_SCHEMA_IMAGE: ${{ env.ADMIN_MCP_SCHEMA_IMAGE }} + run: | + python -m pytest --noconftest -v \ + tests/integration/database/image/test_component_image_serves_offline.py \ + tests/integration/database/image/test_image_admin_mcp.py + + - name: Verify the ui component serves as an arbitrary uid with a read-only root fs + if: matrix.check == 'ui' + env: + LITELLM_IMAGE: ${{ env.IMAGE }} + run: python -m pytest --noconftest -v tests/integration/database/image/test_ui_image_serves_offline.py + + - name: Verify the collector creates its socket as the image user + if: matrix.check == 'collector' + env: + LITELLM_IMAGE: ${{ env.IMAGE }} + run: python -m pytest --noconftest -v tests/integration/database/image/test_collector_image_serves_socket.py + - name: Download Grype v0.114.0 + if: matrix.check == 'scan' run: | curl -fsSL --retry 3 -o "$RUNNER_TEMP/grype.tar.gz" \ https://github.com/anchore/grype/releases/download/v0.114.0/grype_0.114.0_linux_amd64.tar.gz @@ -69,33 +189,6 @@ jobs: tar xzf "$RUNNER_TEMP/grype.tar.gz" -C "$RUNNER_TEMP" grype chmod +x "$RUNNER_TEMP/grype" - # Dockerfile.non_root is the rootless variant we ship. The other - # Dockerfiles share the same wolfi base and apk set, so OS-layer coverage - # is the same; matrix-scan if those variants ever diverge. - - name: Build runtime image - run: docker build -f docker/Dockerfile.non_root -t litellm-image-scan:${{ github.sha }} . - - - name: Tag the cached builder for Admin MCP schema setup - run: docker build --target builder -f docker/Dockerfile.non_root -t litellm-admin-mcp-schema:${{ github.sha }} . - - # The prisma bake must migrate a fresh DB with no egress as an arbitrary - # non-root uid (OpenShift restricted-v2 / air-gapped / readOnlyRootFilesystem). - # `docker run` as the default uid with network hides a broken bake because - # the migration entrypoint exits 0 even when it applied nothing; asserting - # the schema was created is what catches it. - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Verify offline migration as a non-root uid - env: - LITELLM_IMAGE: litellm-image-scan:${{ github.sha }} - LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }} - run: | - python -m pip install "pytest==9.0.3" - python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v - # Scans the whole shipped artifact: OS/apk plus every language package # baked into the image, including ones no lockfile declares (e.g. prisma's # vendored node engine) that osv-scan cannot see. osv-scan stays the fast @@ -103,175 +196,12 @@ jobs: # free OSS, run as a pinned, checksum-verified binary; no GitHub Action # dependency and no vendor SaaS callout. - name: Scan image for fixable HIGH/CRITICAL CVEs + if: matrix.check == 'scan' env: GRYPE_MATCH_PYTHON_USING_CPES: "true" run: | - "$RUNNER_TEMP/grype" litellm-image-scan:${{ github.sha }} \ + "$RUNNER_TEMP/grype" "$IMAGE" \ --config .grype.yaml \ --only-fixed \ --fail-on high \ --output table - - runtime-image: - name: runtime-image (${{ matrix.dockerfile }}) - runs-on: ubuntu-latest - if: >- - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository - timeout-minutes: 45 - permissions: - contents: read - strategy: - fail-fast: false - matrix: - dockerfile: [Dockerfile, docker/Dockerfile.database] - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Build runtime image - run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f "${{ matrix.dockerfile }}" -t litellm-runtime-scan:${{ github.sha }} . - - - name: Tag the cached builder for Admin MCP schema setup - run: docker build --target builder -f "${{ matrix.dockerfile }}" -t litellm-admin-mcp-schema:${{ github.sha }} . - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Verify offline migration as a non-root uid - env: - LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }} - LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }} - run: | - python -m pip install "pytest==9.0.3" - python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v - - - migrations-image: - name: migrations-image - runs-on: ubuntu-latest - if: >- - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository - timeout-minutes: 30 - permissions: - contents: read - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Build migrations image - run: docker build -f migrations/Dockerfile -t litellm-migrations-scan:${{ github.sha }} . - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Verify offline migration as a non-root uid - env: - LITELLM_IMAGE: litellm-migrations-scan:${{ github.sha }} - LITELLM_MIGRATION_INTERPRETER: python3 - LITELLM_MIGRATION_SCRIPT: /app/run.py - run: | - python -m pip install "pytest==9.0.3" - python -m pytest --noconftest tests/integration/database/image/test_offline_image_migration.py -v - - gateway-image: - name: gateway-image - runs-on: ubuntu-latest - if: >- - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository - timeout-minutes: 30 - permissions: - contents: read - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Build gateway image - run: docker build -f gateway/Dockerfile -t litellm-gateway-scan:${{ github.sha }} . - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Verify the gateway serves offline as a non-root uid - env: - LITELLM_IMAGE: litellm-gateway-scan:${{ github.sha }} - LITELLM_COMPONENT_PORT: "4000" - LITELLM_IMAGE_COMPONENT: gateway - run: | - python -m pip install "pytest==9.0.3" - python -m pytest --noconftest tests/integration/database/image/test_component_image_serves_offline.py tests/integration/database/image/test_image_bedrock_realtime_extra.py tests/integration/database/image/test_image_admin_mcp.py -v - - ui-image: - name: ui-image - runs-on: ubuntu-latest - if: >- - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository - timeout-minutes: 30 - permissions: - contents: read - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Build UI image - run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} . - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Verify the UI serves offline as an arbitrary uid with a read-only root fs - env: - LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }} - run: | - python -m pip install "pytest==9.0.3" - python -m pytest --noconftest tests/integration/database/image/test_ui_image_serves_offline.py -v - - backend-image: - name: backend-image - runs-on: ubuntu-latest - if: >- - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository - timeout-minutes: 30 - permissions: - contents: read - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Build backend image - run: docker build -f backend/Dockerfile -t litellm-backend-scan:${{ github.sha }} . - - - name: Tag the cached builder for Admin MCP schema setup - run: docker build --target builder -f backend/Dockerfile -t litellm-admin-mcp-schema:${{ github.sha }} . - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Verify the backend serves offline as a non-root uid - env: - LITELLM_IMAGE: litellm-backend-scan:${{ github.sha }} - LITELLM_ADMIN_MCP_SCHEMA_IMAGE: litellm-admin-mcp-schema:${{ github.sha }} - LITELLM_COMPONENT_PORT: "4001" - LITELLM_IMAGE_COMPONENT: backend - run: | - python -m pip install "pytest==9.0.3" - python -m pytest --noconftest tests/integration/database/image/test_component_image_serves_offline.py tests/integration/database/image/test_image_admin_mcp.py -v diff --git a/.github/workflows/interactions-spec-canary.yml b/.github/workflows/interactions-spec-canary.yml index a2e4646dce5..89ebdd87a7c 100644 --- a/.github/workflows/interactions-spec-canary.yml +++ b/.github/workflows/interactions-spec-canary.yml @@ -49,9 +49,9 @@ jobs: id: compliance run: | uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \ - --interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \ + --interactions-spec-url=https://aistudio.google.com/api-reference/interactions.openapi.json \ --tb=short -vv -p no:rerunfailures -p no:pytest-retry \ - --allow-hosts=127.0.0.1,::1,localhost,ai.google.dev + --allow-hosts=127.0.0.1,::1,localhost,ai.google.dev,aistudio.google.com tracking-issue: name: tracking-issue diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 73159bd64d2..8be7b89e076 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -55,7 +55,6 @@ jobs: services: postgres: image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5 - command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -81,6 +80,17 @@ jobs: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: persist-credentials: false + - name: Configure PostgreSQL for integration workers + env: + POSTGRES_CONTAINER: ${{ job.services.postgres.id }} + run: | + docker exec "$POSTGRES_CONTAINER" sh -c 'printf "%s\n" "max_connections = 1000" "shared_preload_libraries = pg_stat_statements" "pg_stat_statements.track = all" "pg_stat_statements.max = 20000" >> "$PGDATA/postgresql.conf"' + docker restart "$POSTGRES_CONTAINER" + for attempt in $(seq 1 30); do + if docker exec "$POSTGRES_CONTAINER" pg_isready -U postgres; then exit 0; fi + sleep 1 + done + exit 1 - name: Set up Python timeout-minutes: 3 @@ -155,8 +165,8 @@ jobs: LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }} run: | case "$SUITE" in - *-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;; - *) bash .circleci/scripts/run_integration.sh "$SUITE" ;; + *-replica) bash .github/scripts/ci/run_integration.sh "${SUITE%-replica}" replica ;; + *) bash .github/scripts/ci/run_integration.sh "$SUITE" ;; esac - name: Verify every case ran exactly once diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index cbeea71ad0f..fd1d2973951 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -21,7 +21,7 @@ jobs: pull-requests: read actions: read runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 services: redis: image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f diff --git a/.github/workflows/test-mcp-dependency-resolution.yml b/.github/workflows/test-mcp-dependency-resolution.yml index 20efb351985..90589cf1e6f 100644 --- a/.github/workflows/test-mcp-dependency-resolution.yml +++ b/.github/workflows/test-mcp-dependency-resolution.yml @@ -28,7 +28,7 @@ on: - ".github/actions/cache-cargo-build/**" - ".github/scripts/detect_changes.sh" - ".github/scripts/uv_sync_with_retries.sh" - - ".circleci/scripts/classify_changes.sh" + - ".github/scripts/ci/classify_changes.sh" permissions: contents: read diff --git a/.github/workflows/test-merge-smoke.yml b/.github/workflows/test-merge-smoke.yml index 99dbd1c101b..912fc90fbe5 100644 --- a/.github/workflows/test-merge-smoke.yml +++ b/.github/workflows/test-merge-smoke.yml @@ -200,8 +200,36 @@ jobs: - name: Install the test client dependencies run: .github/scripts/uv_sync_with_retries.sh --frozen --no-install-workspace --extra proxy --group dev --python 3.12 - - name: Build the image from the published package - run: docker build -t litellm-build-from-pip:smoke -f docker/build_from_pip/Dockerfile.build_from_pip . + - name: Cache Prisma binaries + timeout-minutes: 3 + uses: ./.github/actions/cache-prisma-binaries + + - name: Install the published proxy into its own venv + run: | + uv --no-config venv --python 3.13 /tmp/litellm-pip + uv --no-config pip install --python /tmp/litellm-pip/bin/python \ + "litellm[proxy,proxy-runtime]==1.83.0" \ + "google-cloud-aiplatform==1.133.0" \ + "google-genai==1.37.0" \ + "anthropic[vertex]==0.84.0" \ + "grpcio==1.78.0" \ + "prometheus-client==0.20.0" \ + "langfuse==2.59.7" \ + "opentelemetry-api==1.28.0" \ + "opentelemetry-sdk==1.28.0" \ + "opentelemetry-exporter-otlp==1.28.0" \ + "ddtrace==4.11.0" \ + "sentry-sdk==2.21.0" \ + "mangum==0.17.0" \ + "azure-ai-contentsafety==1.0.0" \ + "azure-storage-file-datalake==12.20.0" \ + "pypdf==6.7.5" \ + "llm-sandbox==0.3.31" \ + "detect-secrets==1.5.0" \ + "prisma==0.11.0" \ + "openai==2.24.0" + export PATH="/tmp/litellm-pip/bin:$PATH" + /tmp/litellm-pip/bin/prisma generate --schema=schema.prisma - name: Start the fake OpenAI endpoint run: | @@ -219,23 +247,21 @@ jobs: echo "::add-mask::${key}" echo "LITELLM_MASTER_KEY=${key}" >> "${GITHUB_ENV}" - - name: Start the proxy container + - name: Start the published proxy + env: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/circle_test + LITELLM_LICENSE: bad-license + FAKE_OPENAI_API_BASE: http://localhost:8190 + OTEL_EXPORTER: in_memory + USE_DDTRACE: "True" + DD_TRACE_OPENAI_ENABLED: "False" + GCS_FLUSH_INTERVAL: "1" + LITELLM_LOG: ERROR run: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e LITELLM_MASTER_KEY \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - -e OTEL_EXPORTER=in_memory \ - -e USE_DDTRACE=True \ - -e GCS_FLUSH_INTERVAL=1 \ - -e LITELLM_LOG=ERROR \ - --add-host host.docker.internal:host-gateway \ - --name litellm-build-from-pip \ - -v "${GITHUB_WORKSPACE}/docker/build_from_pip/litellm_config.yaml:/app/config.yaml" \ - litellm-build-from-pip:smoke \ - --config /app/config.yaml \ - --port 4000 + cd /tmp/litellm-pip + nohup /tmp/litellm-pip/bin/ddtrace-run /tmp/litellm-pip/bin/litellm \ + --config "${GITHUB_WORKSPACE}/tests/basic_proxy_startup_tests/build_from_pip_config.yaml" \ + --port 4000 > /tmp/litellm-pip.log 2>&1 & for _ in $(seq 1 150); do curl -sf --max-time 5 -o /dev/null http://localhost:4000 && exit 0 sleep 2 @@ -270,7 +296,7 @@ jobs: - name: Show the proxy logs if: always() - run: docker logs litellm-build-from-pip || true + run: cat /tmp/litellm-pip.log || true smoke-passed: name: smoke passed diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index a090ec7bdb1..6f980b458fc 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -45,7 +45,7 @@ jobs: id: rust-bridge-key if: steps.changes.outputs.decision != 'skip' env: - RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml') }} + RUST_BRIDGE_CACHE_KEY: ${{ runner.os }}-rust-bridge-editable-${{ hashFiles('litellm-rust/Cargo.lock', 'litellm-rust/Cargo.toml', 'litellm-rust/crates/**', 'model_prices_and_context_window.json', 'pyproject.toml', 'packaging/litellm-proxy/pyproject.toml') }} run: echo "key=$RUST_BRIDGE_CACHE_KEY" >> "$GITHUB_OUTPUT" - name: Set up Python @@ -98,6 +98,18 @@ jobs: mkdir -p rust-bridge-dist/litellm/rust_bridge cp litellm/rust_bridge/_native.abi3.so rust-bridge-dist/litellm/rust_bridge/_native.abi3.so + - name: Build the gateway wheel once for test shards + if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true' + timeout-minutes: 15 + working-directory: packaging/litellm-proxy + env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_GHA_VERSION: litellm-ci-v1 + SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} + CARGO_INCREMENTAL: "0" + run: uv run --no-project --with maturin==1.15.0 maturin build --profile dev --strip --out ../../rust-bridge-dist + - name: Upload Rust bridge artifact if: steps.changes.outputs.decision != 'skip' timeout-minutes: 10 @@ -170,6 +182,7 @@ jobs: tests/unit/google_genai tests/unit/router_strategy tests/unit/router_utils + tests/unit/redis tests/unit/proxy/common_utils/test_cache_aware_routing.py tests/unit/enterprise/enterprise_callbacks/send_emails tests/unit/enterprise/proxy/test_afile_retrieve_returns_unified_id.py @@ -384,6 +397,7 @@ jobs: tests/unit/llms/voyage tests/unit/llms/wandb tests/unit/llms/watsonx + tests/unit/llms/webiq tests/unit/llms/xai tests/unit/llms/xinference tests/unit/llms/you_com @@ -419,6 +433,7 @@ jobs: - shard: router test-path: >- + tests/unit/router_rust tests/unit/test_router python-version: "3.12" workers: 4 @@ -445,7 +460,6 @@ jobs: tests/unit/ocr tests/unit/passthrough tests/unit/rag - tests/unit/cookbook tests/unit/realtime_api tests/unit/rerank_api tests/unit/search @@ -576,6 +590,7 @@ jobs: tests/unit/proxy/credential_endpoints tests/unit/proxy/discovery_endpoints tests/unit/proxy/shutdown + tests/unit/proxy/rust_forward tests/unit/proxy/public_endpoints tests/unit/proxy/prompts tests/unit/proxy/rag_endpoints @@ -618,23 +633,32 @@ jobs: job-timeout-minutes: 60 dist: loadscope - - shard: proxy-infra + - shard: proxy-infra-db-spend test-path: |- tests/unit/proxy/db --ignore=tests/unit/proxy/db/db_transaction_queue/test_e2e_pod_lock_manager.py --ignore=tests/unit/proxy/db/test_update_daily_tag_spend.py - tests/unit/proxy/middleware - --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py tests/unit/proxy/spend_tracking --ignore=tests/unit/proxy/spend_tracking/test_search_api_logging.py - tests/unit/proxy/pass_through_endpoints - tests/unit/proxy/mcp_server/test_tool_registry.py tests/unit/proxy/common_utils --ignore=tests/unit/proxy/common_utils/test_cache_aware_routing.py --ignore=tests/unit/proxy/common_utils/test_check_batch_cost.py --ignore=tests/unit/proxy/common_utils/test_check_responses_cost.py --ignore=tests/unit/proxy/common_utils/test_proxy_encrypt_decrypt.py --ignore=tests/unit/proxy/common_utils/test_realtime_cache.py + python-version: "3.12" + workers: 4 + reruns: 0 + timeout-minutes: 20 + job-timeout-minutes: 60 + dist: loadscope + + - shard: proxy-infra-routing + test-path: |- + tests/unit/proxy/middleware + --ignore=tests/unit/proxy/middleware/test_request_size_limit_middleware.py + tests/unit/proxy/pass_through_endpoints + tests/unit/proxy/mcp_server/test_tool_registry.py tests/unit/proxy/enterprise_billing tests/unit/proxy/types_utils tests/unit/proxy/logging_endpoints @@ -977,10 +1001,10 @@ jobs: dist: loadscope steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - timeout-minutes: ${{ matrix.shard == 'db-and-spend' && 10 || 3 }} + timeout-minutes: 3 with: persist-credentials: false - fetch-depth: ${{ matrix.shard == 'db-and-spend' && '0' || '1' }} + fetch-depth: 1 - name: Detect relevant changes id: changes @@ -1013,27 +1037,37 @@ jobs: with: artifact: ${{ needs.rust-bridge.outputs.artifact }} - - name: Install dependencies and check schema + - name: Install dependencies if: steps.changes.outputs.decision != 'skip' - timeout-minutes: 8 + timeout-minutes: 5 env: RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }} - UNIT_SHARD: ${{ matrix.shard }} - SCHEMA_BASE_REF: origin/${{ github.event.repository.default_branch }} run: | diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))" fi uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]' - if [ "$UNIT_SHARD" = "db-and-spend" ]; then - uv run --no-sync python scripts/check_schema_compatibility.py --base-ref "$SCHEMA_BASE_REF" - fi + + - name: Check schema.prisma against the merge base + if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'db-and-spend' && github.event_name == 'pull_request' + timeout-minutes: 3 + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref) + merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') + test -n "$merge_base" + git fetch --no-tags --depth=1 origin "$merge_base" + uv run --no-sync python scripts/check_schema_compatibility.py --base-commit "$merge_base" - name: Cache Prisma binaries if: steps.changes.outputs.decision != 'skip' @@ -1270,7 +1304,7 @@ jobs: - name: Install dependencies if: steps.changes.outputs.decision != 'skip' run: | - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard - name: Cache Prisma binaries if: steps.changes.outputs.decision != 'skip' @@ -1319,17 +1353,16 @@ jobs: - name: Run unit tests run: | - for chart in helm/litellm-helm helm/litellm; do - declared="$(grep -h '^suite:' "$chart"/tests/*.yaml | wc -l | tr -d '[:space:]')" - output="$(mktemp)" - helm unittest -f 'tests/*.yaml' "$chart" | tee "$output" - executed="$(sed -n 's/^Test Suites:.*[[:space:]]\([0-9][0-9]*\) total$/\1/p' "$output")" - if [ "$declared" != "$executed" ]; then - echo "::error::$chart declares $declared test suites but helm-unittest ran $executed. Suites are being skipped silently, so their assertions never execute." - exit 1 - fi - echo "$chart: all $declared declared test suites ran" - done + chart=helm/litellm + declared="$(grep -h '^suite:' "$chart"/tests/*.yaml | wc -l | tr -d '[:space:]')" + output="$(mktemp)" + helm unittest -f 'tests/*.yaml' "$chart" | tee "$output" + executed="$(sed -n 's/^Test Suites:.*[[:space:]]\([0-9][0-9]*\) total$/\1/p' "$output")" + if [ "$declared" != "$executed" ]; then + echo "::error::$chart declares $declared test suites but helm-unittest ran $executed. Suites are being skipped silently, so their assertions never execute." + exit 1 + fi + echo "$chart: all $declared declared test suites ran" openai-sdk-compat: name: OpenAI ${{ matrix.openai-version }} compatibility needs: rust-bridge @@ -1473,9 +1506,9 @@ jobs: RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }} run: | if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so fi diff --git a/.gitignore b/.gitignore index b08ee92d6af..1b4785d193a 100644 --- a/.gitignore +++ b/.gitignore @@ -61,7 +61,7 @@ ui/litellm-dashboard/next-env.d.ts *.tsbuildinfo ui/litellm-dashboard/package.json ui/litellm-dashboard/package-lock.json -helm/litellm-helm/*.tgz +helm/litellm/charts/*.tgz helm/*.tgz litellm/proxy/vertex_key.json **/.vim/ diff --git a/AGENTS.md b/AGENTS.md index 9aa9ab32528..d8de447be90 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,14 +45,13 @@ Run QA proxies with `python litellm/proxy/proxy_cli.py --config litellm/proxy/de Simple PRs may skip before/after e2e when the full diff is mechanical and focused checks completely prove it, such as #49030 restoring an existing helper name. Trace affected callers and run relevant tests/static checks at the tip. For a bug fix, establish that the regression catches the original defect. Size alone never qualifies. New behavior, API/provider semantics, auth, billing, persistence, concurrency or uncertain integration risk need live e2e. In Screenshots / Proof of Fix, record `E2E exception at `, scope, rationale, commands and results. Tests count as proof only for this exception. Reassess changed diffs, retain CI/bot gates, and honor explicit e2e requests (2026-10-28, unnecessary live QA for #49030) -For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce +For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. The body alone must let a reviewer rerun it, with the proxy command, config, env var names and client code exactly as run, and long blocks collapsed in
. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y: - don't use emojis - don't use "—". Don't use ";" or ":" unless it would be nonsensical not to. Instead, reach for ",", ".", conjunction words, etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them also feels AI-y. A word cap does not penalize you for adding more sentences - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc. - unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose -- don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a ".". Exception is that docs (i.e., litellm-docs) should have trailing "." - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead - use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e956b27d5da..426cf986d4c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -305,8 +305,8 @@ uv run litellm --config your_config.yaml If you want to build the Docker image yourself: ```bash -# Build using the non-root Dockerfile -docker build -f docker/Dockerfile.non_root -t litellm_dev . +# Build the image (runs as a non-root user by default) +docker build -t litellm_dev . # Generate a master key. Requests send it as the bearer token export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)" diff --git a/Dockerfile b/Dockerfile index 4cef63c8d71..0e9337e658b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,6 +76,7 @@ ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ COPY pyproject.toml uv.lock ./ COPY enterprise/pyproject.toml enterprise/ COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ +COPY packaging/litellm-proxy/pyproject.toml packaging/litellm-proxy/ # Install third-party dependencies (cached unless pyproject.toml/uv.lock change) RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ @@ -101,7 +102,9 @@ COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/ RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh # Install project and workspace packages (fast - deps already cached) -RUN uv sync --frozen --no-default-groups --no-editable \ +ARG MATURIN_PEP517_ARGS="" +RUN uv build --package litellm-proxy --wheel --out-dir /wheels +RUN uv sync --frozen --no-install-package litellm-proxy --no-default-groups --no-editable \ --extra proxy \ --extra proxy-runtime \ --group admin-mcp \ @@ -111,12 +114,18 @@ RUN uv sync --frozen --no-default-groups --no-editable \ --extra bedrock-realtime \ --python python3.13 +RUN uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm_proxy-*.whl + RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ prisma generate --schema=./schema.prisma -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh +RUN sed -i 's/\r$//' docker-entrypoint.sh && chmod +x docker-entrypoint.sh + +RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \ + cp -r litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \ + cp litellm/proxy/logo.png /var/lib/litellm/assets/logo.png && \ + touch /var/lib/litellm/ui/.litellm_ui_ready FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder COPY --from=uvbin /uv /usr/local/bin/uv @@ -141,24 +150,35 @@ USER root # https://github.com/BerriAI/litellm/issues/33518 RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories -# node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent +RUN for i in 1 2 3; do \ + apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libatomic libevent nginx && break; \ + [ "$i" = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ + sleep 5; \ + done COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer WORKDIR /app + ENV PATH="/app/.venv/bin:${PATH}" \ + PYTHONPATH="/app" \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + HOME=/app \ + LITELLM_NON_ROOT=true \ PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \ PRISMA_CLI_QUERY_ENGINE_TYPE=binary \ + PRISMA_SKIP_POSTINSTALL_GENERATE=1 \ + PRISMA_HIDE_UPDATE_MESSAGE=1 \ + PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \ PRISMA_OFFLINE_MODE=true # Copy only what runtime needs. The application is installed inside the venv; # the rest of the builder's /app is source and build metadata that must not # ship (manifest-scanning tools attribute everything in it to this image). -# entrypoint.sh invokes litellm/proxy/prisma_migration.py by source path. COPY --from=builder /app/.venv /app/.venv COPY --from=liteadmin-builder /opt/liteadmin /opt/liteadmin -COPY --from=builder /app/docker /app/docker +COPY --from=builder /app/docker-entrypoint.sh /app/docker-entrypoint.sh COPY --from=builder /app/schema.prisma /app/schema.prisma COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py # enterprise/ is imported by source path at runtime (proxy_cli puts the @@ -166,21 +186,33 @@ COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/pr # enterprise.enterprise_hooks from it) COPY --from=builder /app/enterprise /app/enterprise COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras -# Prisma CLI + engines are baked under /opt/prisma, a fixed path every -# runtime uid can read and that no cache volume mount shadows. The paths are -# pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and recorded into the -# generated client at build time, so `prisma migrate deploy` on a fresh -# database needs no npm and no network access (#33650, #24554). +COPY --from=builder /app/gateway /app/gateway +COPY --from=builder /app/backend /app/backend +COPY --from=builder /app/migrations /app/migrations COPY --from=builder /opt/prisma /opt/prisma +COPY --from=builder /var/lib/litellm /var/lib/litellm +COPY ui/nginx.conf /etc/nginx/nginx.conf +COPY ui/docker-entrypoint.d/40-litellm-runtime-config.sh /app/ui/docker-entrypoint.d/40-litellm-runtime-config.sh -RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ - find /app/.venv -type d -path "*/tornado/test" -delete && \ +RUN find /app/.venv -depth -type d -path "*/tornado/test" -exec rm -rf {} + && \ + mkdir -p /app/.cache /var/run/litellm && \ + chown -R 65532:0 /app /var/lib/litellm /var/run/litellm && \ + chmod -R g=u,g+w /app/.cache /var/lib/litellm /var/run/litellm && \ + PRISMA_PATH="$(python -c 'import os, prisma; print(os.path.dirname(prisma.__file__))')" && \ + PROXY_EXTRAS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))')" && \ + chmod -R g=u,g+w "$PRISMA_PATH" "$PROXY_EXTRAS_PATH" && \ chmod -R a+rX /opt/prisma && \ test -x /opt/prisma/binaries/node_modules/.bin/prisma && \ test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \ - python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" + ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null && \ + python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \ + python -c "from litellm.rust_bridge import _native" && \ + python -c "import gateway.launch, backend.main" -EXPOSE 4000/tcp +USER 65532:65532 -ENTRYPOINT ["docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] +RUN nginx -t + +EXPOSE 4000/tcp 4001/tcp 3000/tcp + +ENTRYPOINT ["/app/docker-entrypoint.sh"] diff --git a/GEMINI.md b/GEMINI.md deleted file mode 100644 index 5fc00e0b5ae..00000000000 --- a/GEMINI.md +++ /dev/null @@ -1 +0,0 @@ -Read @AGENTS.md for coding guidelines diff --git a/Makefile b/Makefile index 48df3c8e215..c256d273d8a 100644 --- a/Makefile +++ b/Makefile @@ -341,7 +341,7 @@ test-integration: install-test-deps $(UV_RUN) pytest tests/ test-unit-helm: install-helm-unittest - helm unittest -f 'tests/*.yaml' helm/litellm-helm + helm unittest -f 'tests/*.yaml' helm/litellm # LLM Translation testing targets test-llm-translation: install-test-deps diff --git a/SECURITY.md b/SECURITY.md index d3603448d46..6d0a9a6414d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,23 +4,19 @@ LiteLLM maintains a security announcements mailing list that is open to anyone. Subscribers receive advance notice, typically one to two days, before we release a fix for a particularly severe vulnerability or for any vulnerability exploitable by an unauthenticated attacker. This notice is provided on a best-effort basis -To subscribe, visit [https://berriai.github.io/security-announce-signup/](https://berriai.github.io/security-announce-signup/) +To subscribe, visit [https://security.litellm.ai/mailinglist/](https://security.litellm.ai/mailinglist/) ## Security Vulnerability Reporting Guidelines -> [!WARNING] -> Reports that do not include a video demonstrating the exploit will be closed without review. See [Reproduction Video Requirement](#reproduction-video-requirement) below. -We value the security community's role in protecting our systems and users. To report a security vulnerability: +We value the security community's role in protecting our systems and users. +To report a security vulnerability, please submit a report through our vulnerability disclosure form: -- File a private vulnerability report on GitHub: [Report a vulnerability](https://github.com/BerriAI/litellm/security/advisories/new) -- Include steps to reproduce the issue -- Include a video or screen recording demonstrating the full exploit against a live LiteLLM instance, from initial access through to impact. A terminal recording (for example asciinema) is fine for CLI-only exploits. -- Provide any relevant additional information +[LiteLLM VDP](https://security.litellm.ai/vdp/) -### Reproduction Video Requirement +These reports are submitted to HackerOne and are triaged there, though no account is needed. -A video demonstrating the exploit is required for every report. AI tools have made it easy to produce plausible-sounding vulnerability reports that do not reproduce in practice, and triaging them takes time away from real issues. Reports submitted without a working reproduction video will be closed without review. If you add a video to a closed report, we will reopen and triage it. +**Please do not submit vulnerabilities as draft GHSAs. As of 2026-11-11 new GHSA submissions will be closed.** ### Vulnerability Categories @@ -55,7 +51,9 @@ To qualify for a bounty, reports must include clear reproduction steps, a reprod ### Known Non-Issues -- Attacks that require a misconfiguration on setup (e.g not setting a `master_key` on the proxy configuration), are **explicitly not in scope** and are not considered vulnerable. +- **Misconfigurations:** attacks that require misconfiguration of the application (e.g. not setting a `master_key` on the proxy configuration). +- **Insecure configurations:** attacks that require deliberate configuration changes to disable security functionality. (e.g. IDOR on response IDs requiring configuration of `disable_responses_id_security` to `true`) +- **Social engineering:** attacks that rely on social engineering as their primary mechanism (e.g. convincing a user's agent to install a malicious package we did not publish). ## Security Measures diff --git a/backend/Dockerfile b/backend/Dockerfile deleted file mode 100644 index aad59ba17c6..00000000000 --- a/backend/Dockerfile +++ /dev/null @@ -1,146 +0,0 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a - -FROM $UV_IMAGE AS uvbin - -# ---------- Rust extension ---------- -# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes -FROM $LITELLM_BUILD_IMAGE AS rust-native - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /uvx /usr/local/bin/ - -RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -ENV UV_PYTHON_DOWNLOADS=0 - -COPY pyproject.toml README.md rust-toolchain.toml ./ -COPY .cargo ./.cargo -COPY litellm-rust ./litellm-rust -COPY model_prices_and_context_window.json ./ - -RUN --mount=type=cache,target=/root/.cache/uv \ - mkdir -p litellm/rust_bridge /wheels && \ - touch litellm/__init__.py litellm/rust_bridge/__init__.py && \ - uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \ - python -c "import maturin; maturin.build_wheel('/wheels')" - -# ---------- Builder ---------- -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /uvx /usr/local/bin/ - -# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE -# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi -# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. -RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start. -# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a -# BuildKit cache mount (different filesystem). -# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of -# silently pulling a managed interpreter. -# PRISMA_USE_GLOBAL_NODE explicit (matches default) so an env override can't -# silently re-enable nodeenv's Node download. -ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - UV_COMPILE_BYTECODE=1 \ - UV_PYTHON_DOWNLOADS=0 \ - PRISMA_USE_GLOBAL_NODE=true \ - PATH="/app/.venv/bin:${PATH}" - -# Stage 1 — install dependencies only. -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ - --mount=type=bind,source=uv.lock,target=uv.lock \ - --mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \ - --mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \ - uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra saml \ - --python python3.13 - -# Stage 2 — copy source and install the project + workspace members. -COPY . . - -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-install-project --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra saml \ - --python python3.13 - -RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \ - uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl - -RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ - -RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - npm_config_cache=/root/.npm \ - prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh - -# ---------- Runtime ---------- -FROM $LITELLM_RUNTIME_IMAGE AS runtime -ARG LITELLM_RELEASE_TAG="" -ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} - -USER root - -RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with -# /home/nonroot. We run the backend as that user -WORKDIR /app -ENV HOME=/home/nonroot \ - PATH="/app/.venv/bin:${PATH}" \ - PYTHONPATH="/app" \ - PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries - -COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv -COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm -COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise -COPY --from=builder --chown=nonroot:nonroot /app/backend /app/backend -COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh -COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma -COPY --from=builder /opt/prisma /opt/prisma - -RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ - find /app/.venv -type d -path "*/tornado/test" -delete && \ - chmod -R a+rX /opt/prisma && \ - python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" - -USER nonroot - -EXPOSE 4001/tcp - -ENTRYPOINT ["/app/docker/component_entrypoint.sh", "uvicorn", "backend.main:app"] -CMD ["--host", "0.0.0.0", "--port", "4001"] diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index d5fd755ba18..09138bd6c14 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -20,7 +20,7 @@ import json import os import subprocess import sys -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType from typing import Final @@ -28,8 +28,13 @@ from typing import Final COST_MAP_PATH: Final = "model_prices_and_context_window.json" BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" PINS_PATH: Final = "ci_cd/cost_map_pins.json" +PINS_PATHS: Final = ( + "litellm/scripts/ci_cd/cost_map_pins.json", + "scripts/ci_cd/cost_map_pins.json", + "ci_cd/cost_map_pins.json", +) GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH) -CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH) +CHECKED_PATHS: Final = (*GUARDED_PATHS, *PINS_PATHS) BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_" PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync" TARGET_ENV: Final = "COST_MAP_GUARD_TARGET" @@ -193,7 +198,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str def pin_author_failures(head_ref: str, author_type: str, changed_files: Sequence[str]) -> tuple[str, ...]: - if PINS_PATH not in changed_files: + if not any(path in changed_files for path in PINS_PATHS): return () if head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX): return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",) @@ -251,8 +256,11 @@ def _git(*args: str) -> str | None: return result.stdout if result.returncode == 0 else None -def snapshot(revision: str) -> Snapshot: - return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS)) +def snapshot(revision: str, *, read_git: Callable[[str, str], str | None] = _git) -> Snapshot: + return Snapshot( + *(read_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS), + next((text for path in PINS_PATHS if (text := read_git("show", f"{revision}:{path}")) is not None), ""), + ) def merged_tree(merge_base: str, target: str, head: str) -> str | MergeFailed | None: diff --git a/ci_cd/security_scans_readme.md b/ci_cd/security_scans_readme.md index dd64b01c296..3003ea1d105 100644 --- a/ci_cd/security_scans_readme.md +++ b/ci_cd/security_scans_readme.md @@ -4,6 +4,5 @@ - Trivy scan on `./docs/` (HIGH/CRITICAL/MEDIUM) - Trivy scan on `./ui/` (HIGH/CRITICAL/MEDIUM) -- Grype scan on `Dockerfile.database` (fails on CRITICAL) - Grype scan on main `Dockerfile` (fails on CRITICAL) - Grype CVSS ≥ 4.0 scan on main `Dockerfile` (fails any vulnerabilities with CVSS ≥ 4.0) diff --git a/codecov.yaml b/codecov.yaml index 521088868bd..419a28ce6c6 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -6,7 +6,7 @@ codecov: ignore: - "litellm-rust/**" -# Uploads are flagged per workflow tier (GHA) or "circleci". carryforward makes +# Uploads are flagged per workflow tier. carryforward makes # a re-upload of a flag replace its prior session instead of accumulating a # conflicting one, and lets a commit reuse a flag from its parent when that flag # was not re-uploaded. Required because the same commit can receive the diff --git a/cookbook/Benchmarking_LLMs_by_use_case.ipynb b/cookbook/Benchmarking_LLMs_by_use_case.ipynb deleted file mode 100644 index 6ea6211bfb6..00000000000 --- a/cookbook/Benchmarking_LLMs_by_use_case.ipynb +++ /dev/null @@ -1,753 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "4Cq-_Y-TKf0r" - }, - "source": [ - "# LiteLLM - Benchmark Llama2, Claude1.2 and GPT3.5 for a use case\n", - "In this notebook for a given use case we run the same question and view:\n", - "* LLM Response\n", - "* Response Time\n", - "* Response Cost\n", - "\n", - "## Sample output for a question\n", - "![Screenshot 2023-09-07 at 4.45.37 PM.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAACXAAAALmCAYAAADhDS65AAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkEAIEHoNvQkiNYCUEFoA6UWwEZIAocQYCCp2ZFHBtYsFbOiqiGIHxI7YWRR7X1BRUdbFgl15kwK67ivfm++bO//958x/zpw7c+8dAGgnuGJxHqoBQL6oUBIfFsQYnZrGID0FREADegAFRlxegZgVGxsFYBls/17e3QCIrL3qJNP6Z/9/LZp8QQEPACQW4gx+AS8f4gMA4NU8saQQAKKMt5xcKJZhWIG2BAYI8XwZzlLgahnOUOA9cpvEeDbErQCoqHG5kiwA1C9DnlHEy4Ia6n0Qu4j4QhEANAbE/vn5E/kQp0NsB23EEMv0mRk/6GT9TTNjSJPLzRrCirnIi0qwsECcx536f6bjf5f8POmgDxtY1bIl4fGyOcO83cqdGCnDahD3ijKiYyDWgviDkC+3hxilZEvDkxT2qDGvgA1zBnQhduFzgyMhNoY4VJQXHaXkMzKFoRyI4QpBpwgLOYkQ60M8X1AQkqC02SiZGK/0hTZkStgsJX+OK5H7lfl6IM1NYin1X2cLOEp9TL04OzEFYgrEVkXC5GiI1SF2LshNiFTajCzOZkcP2kik8bL4rSCOF4jCghT6WFGmJDReaV+eXzA4X2xjtpATrcT7CrMTwxX5wVp5XHn8cC7YZYGIlTSoIygYHTU4F74gOEQxd+yZQJSUoNT5IC4MileMxSnivFilPW4hyAuT8RYQuxcUJSjH4smFcEEq9PFMcWFsoiJOvDiHGxGriAdfAqIAGwQDBpDCmgEmghwgbO9t7IV3ip5QwAUSkAUEwEnJDI5IkfeI4DUBFIM/IRKAgqFxQfJeASiC/NchVnF1Apny3iL5iFzwBOJ8EAny4L1UPko05C0ZPIaM8B/eubDyYLx5sMr6/z0/yH5nWJCJUjLSQY8M2qAlMYQYTAwnhhLtcUPcH/fFo+A1EFZXnIl7D87juz3hCaGD8JBwndBJuD1BWCL5KcpRoBPqhypzkfFjLnAbqOmBB+F+UB0q47q4IXDC3aEfFh4APXtAlq2MW5YVxk/af5vBD09DaUd2IaNkPXIg2e7nkeoO6h5DKrJc/5gfRawZQ/lmD/X87J/9Q/b5sI382RKbj+3HzmInsfPYEawRMLDjWBPWhh2V4aHV9Vi+uga9xcvjyYU6wn/4G3yyskwWuNS59Lh8UfQVCqbI3tGAPVE8VSLMyi5ksOAXQcDgiHjOwxiuLq5uAMi+L4rX15s4+XcD0W37zs39AwC/4wMDA4e/cxHHAdjrBbf/oe+cHRN+OlQBOHeIJ5UUKThcdiHAtwQN7jQDYAosgR2cjyvwBL4gEISACBADEkEqGA+jz4brXAImg+lgDigDFWAJWAnWgg1gM9gOdoF9oBEcASfBGXARXAbXwV24errBC9AH3oHPCIKQECpCRwwQM8QacURcESbij4QgUUg8koqkI1mICJEi05G5SAWyDFmLbEJqkb3IIeQkch7pQG4jXUgP8hr5hGKoGqqNmqA26HCUibLQSDQRHYdmoZPQYrQUXYSuRmvQnWgDehK9iF5HO9EXaD8GMFVMFzPHnDAmxsZisDQsE5NgM7FyrBKrweqxZvicr2KdWC/2ESfidJyBO8EVHI4n4Tx8Ej4TX4ivxbfjDXgrfhXvwvvwbwQqwZjgSPAhcAijCVmEyYQyQiVhK+Eg4TTcS92Ed0QiUZdoS/SCezGVmEOcRlxIXEfcTTxB7CA+IvaTSCQDkiPJjxRD4pIKSWWkNaSdpOOkK6Ru0gcVVRUzFVeVUJU0FZFKiUqlyg6VYypXVJ6qfCZrkK3JPuQYMp88lbyYvIXcTL5E7iZ/pmhSbCl+lERKDmUOZTWlnnKaco/yRlVV1ULVWzVOVag6W3W16h7Vc6pdqh/VtNQc1NhqY9WkaovUtqmdULut9oZKpdpQA6lp1ELqImot9RT1AfWDOl3dWZ2jzlefpV6l3qB+Rf0ljUyzprFo42nFtEraftolWq8GWcNGg63B1ZipUaVxSOOmRr8mXXOEZoxmvuZCzR2a5zWfaZG0bLRCtPhapVqbtU5pPaJjdEs6m86jz6VvoZ+md2sTtW21Odo52hXau7Tbtft0tHTcdZJ1puhU6RzV6dTFdG10Obp5uot19+ne0P2kZ6LH0hPoLdCr17ui917fSD9QX6Bfrr9b/7r+JwOGQYhBrsFSg0aD+4a4oYNhnOFkw/WGpw17jbSNfI14RuVG+4zuGKPGDsbxxtOMNxu3GfebmJqEmYhN1picMuk11TUNNM0xXWF6zLTHjG7mbyY0W2F23Ow5Q4fBYuQxVjNaGX3mxubh5lLzTebt5p8tbC2SLEosdlvct6RYMi0zLVdYtlj2WZlZjbKablVndceabM20zrZeZX3W+r2NrU2KzTybRptntvq2HNti2zrbe3ZUuwC7SXY1dtfsifZM+1z7dfaXHVAHD4dshyqHS46oo6ej0HGdY8cwwjDvYaJhNcNuOqk5sZyKnOqcupx1naOcS5wbnV8OtxqeNnzp8LPDv7l4uOS5bHG5O0JrRMSIkhHNI167OrjyXKtcr7lR3ULdZrk1ub1yd3QXuK93v+VB9xjlMc+jxeOrp5enxLPes8fLyivdq9rrJlObGctcyDznTfAO8p7lfcT7o4+nT6HPPp+/fJ18c313+D4baTtSMHLLyEd+Fn5cv01+nf4M/3T/jf6dAeYB3ICagIeBloH8wK2BT1n2rBzWTtbLIJcgSdDBoPdsH/YM9olgLDgsuDy4PUQrJClkbciDUIvQrNC60L4wj7BpYSfCCeGR4UvDb3JMODxOLacvwitiRkRrpFpkQuTayIdRDlGSqOZR6KiIUctH3Yu2jhZFN8aAGE7M8pj7sbaxk2IPxxHjYuOq4p7Ej4ifHn82gZ4wIWFHwrvEoMTFiXeT7JKkSS3JtOSxybXJ71OCU5aldI4ePnrG6IuphqnC1KY0Ulpy2ta0/jEhY1aO6R7rMbZs7I1xtuOmjDs/3nB83vijE2gTuBP2pxPSU9J3pH/hxnBruP0ZnIzqjD4em7eK94IfyF/B7xH4CZYJnmb6ZS7LfJbll7U8qyc7ILsyu1fIFq4VvsoJz9mQ8z43Jndb7kBeSt7ufJX89PxDIi1Rrqh1ounEKRM7xI7iMnHnJJ9JKyf1SSIlWwuQgnEFTYXa8Ee+TWon/UXaVeRfVFX0YXLy5P1TNKeIprRNdZi6YOrT4tDi36bh03jTWqabT58zvWsGa8ammcjMjJktsyxnlc7qnh02e/scypzcOb+XuJQsK3k7N2Vuc6lJ6ezSR7+E/VJXpl4mKbs5z3fehvn4fOH89gVuC9Ys+FbOL79Q4VJRWfFlIW/hhV9H/Lr614FFmYvaF3suXr+EuES05MbSgKXbl2kuK172aPmo5Q0rGCvKV7xdOWHl+Ur3yg2rKKukqzpXR61uWmO1ZsmaL2uz116vCqraXW1cvaD6/Tr+uivrA9fXbzDZULHh00bhxlubwjY11NjUVG4mbi7a/GRL8pazvzF/q91quLVi69dtom2d2+O3t9Z61dbuMN6xuA6tk9b17By78/Ku4F1N9U71m3br7q7YA/ZI9zzfm773xr7IfS37mfvrD1gfqD5IP1jegDRMbehrzG7sbEpt6jgUcail2bf54GHnw9uOmB+pOqpzdPExyrHSYwPHi4/3nxCf6D2ZdfJRy4SWu6dGn7rWGtfafjry9LkzoWdOnWWdPX7O79yR8z7nD11gXmi86Hmxoc2j7eDvHr8fbPdsb7jkdanpsvfl5o6RHceuBFw5eTX46plrnGsXr0df77iRdOPWzbE3O2/xbz27nXf71Z2iO5/vzr5HuFd+X+N+5QPjBzV/2P+xu9Oz82hXcFfbw4SHdx/xHr14XPD4S3fpE+qTyqdmT2ufuT470hPac/n5mOfdL8QvPveW/an5Z/VLu5cH/gr8q61vdF/3K8mrgdcL3xi82fbW/W1Lf2z/g3f57z6/L/9g8GH7R+bHs59SPj39PPkL6cvqr/Zfm79Ffrs3kD8wIOZKuPJfAQxWNDMTgNfbAKCmAkCH5zPKGMX5T14QxZlVjsB/woozorx4AlAP/9/jeuHfzU0A9myBxy+oTxsLQCwVgERvgLq5DdXBs5r8XCkrRHgO2Jj4NSM/A/ybojhz/hD3zy2QqbqDn9t/AendfFfdLXJmAAAAimVYSWZNTQAqAAAACAAEARoABQAAAAEAAAA+ARsABQAAAAEAAABGASgAAwAAAAEAAgAAh2kABAAAAAEAAABOAAAAAAAAAJAAAAABAAAAkAAAAAEAA5KGAAcAAAASAAAAeKACAAQAAAABAAAJcKADAAQAAAABAAAC5gAAAABBU0NJSQAAAFNjcmVlbnNob3Q0Yv8EAAAACXBIWXMAABYlAAAWJQFJUiTwAAAB12lUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNi4wLjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczpleGlmPSJodHRwOi8vbnMuYWRvYmUuY29tL2V4aWYvMS4wLyI+CiAgICAgICAgIDxleGlmOlBpeGVsWURpbWVuc2lvbj43NDI8L2V4aWY6UGl4ZWxZRGltZW5zaW9uPgogICAgICAgICA8ZXhpZjpQaXhlbFhEaW1lbnNpb24+MjQxNjwvZXhpZjpQaXhlbFhEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlVzZXJDb21tZW50PlNjcmVlbnNob3Q8L2V4aWY6VXNlckNvbW1lbnQ+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgqAaPP7AAAAHGlET1QAAAACAAAAAAAAAXMAAAAoAAABcwAAAXMAAp5dSS8XJAAAQABJREFUeAHsnQe8HUX5v4cgRTD0EkqoMSAdIlIkGIr0KlIEBARpBpSmIkVAqvRepUmTQKSIIII06SUC0iMl0iEUQaSJ/PPMjzn/uZs95+4pN7lJnvfzufdsmZ2dfXZmd3bnu+872fLLL/95mAitb9++E+FReUgSkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQwPgk8P7778fd//vf/w6TTz55mGyyyUKfPn3iH9NpvmoZJ1PAVRWV6SQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCCBSZ2AAq6KNUAPXBVBmUwCEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISqExAAVdFVAq4KoIymQQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCRQmYACroqoFHBVBGUyCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSKAyAQVcFVEp4KoIymQSkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQQGUCCrgqolLAVRGUySQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCCBygQUcFVGZUIJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAI9Q+Df//53mHzyycNkk00W+vTpE/+YTvNV9zrZ8ssv/3nVxKaTgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQggRAUcFkLJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJjCcCCrjGE3h3KwEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAGXdUACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkMB4IqCAazyBd7cSkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQUcFkHxgmBGWecMcw///zhkUceCZ999tk42ac7mTQITDnllGGxxRYLI0eODO+///4kcdDzzDNPmGaaacJTTz01SRyvBykBCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKYmAko4JqYz24vObbZZ589HHHEEbE0d955Zzj//PN7Sckm7GLMNddcYciQIWHyyScPH3/8cRg+fHj473//O2EfVAulP+OMMwIirtGjR4ef//znLeQwbjYZPHhwFDHme6PM119/fb6o2+kNNtggbLjhhjHd8ccfHx5//PFutxkfCWaaaaaw5pprhimmmKK2+88//zzceuut4aWXXqotK5toZ9uy/FwmAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAIS6M0EFHC1eHbmnHPOsNBCC4XnnnsujBo1qsVcJozNFlxwwfD888+H//3vfy0VeMkllww//vGP47bwOvzww1vKx426Ejj00EMD9TDZSSedFB599NE0O85+260f7Rb0N7/5TZhsssnCJ598Enbdddd2s+uR7aeaaqpw+umnl+a9ww47lC6vt3DHHXcMyy+/fFx9zTXXhGuvvbZe0i7Lx/V52mOPPcLiiy/epQzMPPzww+GUU04Za3m+oJ1t83xamf7Sl74UVl555TDffPOFOeaYI8wwwwyBG+ULL7wQhg0bFj788MNWsnUbCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQQF0CCrjqohl7BSKRoUOHBgRJffr0qSXAqwzCpGOPPTaKSGorJoKJ733ve2H11VePodqOOeaYlo6IcG8HHXRQ3PaJJ54Ixx13XEv5uFFXAt/85jfD2muvHUUmrBkfAq5O1I+uR9X83FlnnRUQ3fznP/8Ju+++e/MZjKMtdttttyi4w2MaHsOmm266uOdmBVybbbZZ9GzFxhdffHH0aNXdIYyP87TEEkuEjTfeOHz5y1+OAruZZ545/hJG9eSTT25Y5Ha2bZhxNyu5rtOOCE9ZZni4Yz3XMU0CEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCTQKQIKuCqSnHHGGcOBBx4Ypp9++toWDOYjHEmGZ5ajjjqq2/BgKf2E8IvnLARrb731VvjZz37WcpEXXXTR6MlmxIgRerBpmWL5hueee25cMT4EXJ2qH+VHVm3pV7/61dC/f//o2entt9+uttF4TrXIIouEvffeO5aiWQEX3rxWWGGF8M4770SPawhIu7PecJ5+/etfh1lmmSVUEXAVj6edbYt5NZondGMSqiIIxKPdp59+GhCUpWs/nt7gyXJNAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQk0AkCCrgqUjzyyCPDbLPNFlMTAoywbQi2pphiirD55puHVVZZJa5j0J/B/Sqiioq7Hq/JkvADscg+++wzXsvizssJ9AYBl/Wj/NzUW9qOgKteno2W94Z23I4Iq51tG3EprsNb2Kmnnhqv3zvttFOXsLHML7fccnETvL7df//9xc2dl4AEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCbREQAFXBWyEqtt+++1jSgbtGbwv2gYbbBA23HDDuPhPf/pTuOKKK0Lfvn3DnnvuWQvHdfvtt4cbbrghpiF82h577FFbh+Drt7/9bXjyySeLWcf5hRdeOArF8GCDyAD74IMPwj//+c9wzjnnhPfeey8uK/7Da8wWW2wRFltssYDnHkKE4TmG9AjQ5phjjvD73/8+3HjjjcVN43yrwo+vfOUrYa+99qodH5n973//CxdccEF45plnSveVLyTE3DbbbBPwrgQrhHKUF09gN910U7jrrrvy5PG48GhEmDbsscceC3fccUfAu9Hss89eC7F38803h2uvvbbLtvVm4Ixg429/+1v417/+VS9ZS8s5D3g0m2GGGWrbw+fyyy+PHopqC8dMbL311gEPZoTwxKgrV199dbjvvvvifBJwEZZumWWWCUsttVSYdtppAx7iXnrppXDKKafULT/loN4iKIId2xHiDy9DL7/8cjjjjDNCI69WrdaPWPA2/u26665h3nnn7ZJD3r66rCjMzD///LFN4LWLeoZ9/PHHYfTo0bFtUd8OO+ywHvWk16yAi/RbbrllF49/lPnwww+vFLa1nfOE90Ha0VxzzRVo19RD9v3mm2+Giy66KDz77LMFwuWz7YiwutsWT4i0pxSW8oUXXgicZ+r0Z599Fr2zXXnlldHrGddQjPpNvohuc1t33XXjteaWW27JF0cPXMcff3xc9pe//CVceumlXdY7IwEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEmiVgAKuCuQITYdwAYHNbrvtFsULZZudfvrpUSSFcGbnnXeOYpqhQ4fWkr766qvhgAMOiPO5KCwluOeee6JnrzSffhGBIcCqZwh6EBMUBQeIGk488cSa4Kve9v/4xz8CHsaSrbfeelHUg7inO6sXDm3ppZeOrIrbX3/99WH48OHFxV3mCQ+33XbbdRGrdEkwZgaWRx99dE24loc+I23ygJZET/n2d955Zzj//PPzRWNNb7bZZmGNNdaoiaYefPDBKGYaK2GLCxDOIRIqGmIxPADllupVvuypp56qhXpLAq5iSM+UnuW77757qdCH+onoq55R5xHivf/++7UknagftcxanMADXvHcImY85JBDGuY4zzzzhF/+8pdjbVvcCLFPElsW13VivlkBF9eTb3zjG2PtmusJbaHMOnGeEKbyV2Sd76+eqDVPw3R3Iqxi+ny+u205rwcddFC+yVjTXBOKx4H47IgjjhgrbdmCVVddNWy11VZx1bBhw+qKXsu2dZkEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUigEQEFXI3ofLEuiUVGjhwZjjrqqLpb4KVmxRVXjOsRXeHlas011wx4dMGz0WuvvRb233//uB5x1frrrx+mnnrqMHjw4Cj8KhMJ/eAHPwgrrbRS3AYPVHiPQuSDNy0EBSuvvHIUJSBOQBCE55lkP/rRj8KgQYPi7HXXXRfwPoXXrsUXXzx6FEOUhj333HPRk0+cGfMPz1ff+ta30mzD3+K2eWKOHWEVRlkRhHUn4CoKbPBIhietN954IwwcODCyTOV+/fXXw3777VfbJZxgmjzssAKBxkMPPRQQlA0YMKDGCo9ERc87tYzGTJx99tnRE1VaBl9EUJyDTtnaa68dvvvd78bsEB89+uij4bbbbguEI8wNsQ8CviQoS1618BiFJQEX03hGQsiHxzLq4jTTTMPi6LHsvPPOi9P5vyQOpF48/fTTYcSIEbGuIjDkXGB4MzvhhBNqm3WqftQybGGC84lXOgxhE56Xqgi48KCEVzq8MuENDvERHscIgfqd73ynds6vuuqqQJvpKWtWwEWd//a3vx2vF1wz0jWhkYCr3fOUizAR8uHhjLZIu+G6stFGG9U8mFXh1Z0IqxHrKtsOGTIkIFrDYxhGO8HbH9cyzjFGO37ggQeixzmYMv/DH/4wrmv0b9NNN43XcgRgbPPTn/50rHbaaHvXSUACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABBoRUMDViM6YdbmnpO48NyFYwnMTRphFxCHYvvvuG0MB5gKuuOKLf8ccc0wUOhUFXIQs+9WvfhVTIaBAbMMJyw3vScnLF+EFCSOWDLHZrLPOGkVbCJZyQ4iAmIwwY4h2TjvttHx1LVTaLrvsEoUyhBAserhBBNNIBJVnSCg+wtV1J+BKAhu2RXBUDJVIuSkH4e+wa665pktIxFzwVtxXHuYSL1cI4eoZ5w+RXW54oup0KMXjjjsuhlHMhXBzzz13FIu9+OKLNW9chHNMnrkQUyGqSpYEXJwL6gget5IlIVpR7JbW84v4qSwEJ3UCsRDHzLHnhviFc9Gp+pHn3ew09XiBBRaoJOBKYsyiKI19IuxCBInoDW4pRGWz5amSvlkBV54nAqVjjz02Lmok4CJBO+cptVnyKWsv8MILHu0EUdNOO+0UvRSSvsyqiLDKtmNZ1W0Jrfn1r389XpcQXGI77rhjWH755eN08qyW80foivCxzJZccsnoDTCFZuQ4y7wdlm3rMglIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkEBVAgq4uiGFdx/Cl2FFsVBx01xMhfcevNJgrQq48M6ElybsxhtvjOKnOFP4R0i4mWeeeSxvMnvvvXf0NENyxEoPP/xwePfdd6PnGDxaffrppzF83uOPP15XwIDwCxEDXqH22Wefwp6rzyYxSFFUleeQi5RyQVOehmlEKYSGRED08ssvx5B4KU0ScKUwlmk5v3379o3bMd2dx6Ci9yI8gSXRDNt3yrbeeuvo/SkvLyK8hRZaKO4CgQyesVL4NkRzCGVySwKusmNC4INID3FXErTk2zLNejyXUYdghJEeoRACLuoJQq0y61T9KMu76rJmBFynnHJKFGghxLn11lsD4UMRqCF+xKMZnppox0l8WbUMzabLBUR47mvGmhFwpXybPU9zzjlnOPTQQ+PmeLJCZFpmW265ZVhuueXiKsSITzzxRFmyuKyqCKssg6rbJgFXLmbFw+F2Y0KyYlzDuJYhPkMsih144IHhlVdeidP5v9lmm61LaFk8vOG1bdSoUXkypyUgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAJtE1DA1Q1CxC2IYDDCh1144YV1tyDE3eabbx7XI6q5++6743SrAq4kTKm7w5IV7OvNN9+Mawg5iBgIoVOZffLJJ+GRRx4JF1988VievVL6ZoUfabvibxUB1worrFALZzZs2LAoWivmk+aToAPPOXjQSZYEXHiUwhtV0ZJnre7EeGxHKEbCEOIZrUzgUcy7lflcJHLkkUdGQdGZZ54ZQyCSH+EfTz/99PDzn/88hpAsE7YlAddJJ50UwzDm5UgCsXoCrpRvvk1xOheXFdd1qn4U821mPrWTKiEUCfuHWK2eUW/wtDd8+PB6STqyvLcLuAhFuPHGGzd1rH/4wx/C1VdfXXeb1Ga55px88sl105WtqLptmYCLcJNcF/BiiDcuDG+AXJMwPPohUisaYTVpP4j9EHm9+uqrxSTOS0ACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABDpCQAFXNxgJD4boB3vqqafqeqJhPV5e8PaC5UKqVgVchxxySCCcHpaHxYsLSv7hSQhBDoKDZAhFEGLMO++80btQWp7/NvKu1SmBThUBV+4pJxfA5WVN04lN0SNVdwKuJI6qIuBK++rp3+QV6p577gm33HJLDG3JceENKgmvEr9LLrkkpsnL1KqAa9111w3f+c53YlYffvhh9CyEpzGEUP369QurrbZaFLFNTAIuDnb11VePHs0Qz9UTN+KBK7X7nHWnpnu7gItQsISExRA+8dfIEIPipYu6U8+qirDKtq+6bScFXIMGDYriUNrGbrvtVlYsl0lAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQ6QkABVwWMSWCDqAZvT/XEVCkdAqof/vCHtZyTgCsP61VbOWYCbzTTTjtt9PSUvMKwntCNhHDE2C/eppqxeeaZJwq3/vrXv8bNEIMRfnCqqaYKc8wxR8DLDmELMQRRZeILwu4ttdRS0UPXT37yk5i2lX9JgNQohOJMM81UE8jhBen888+vuyu8UnEciNb22muvWrpOCrgWW2yxgPcePAbde++9XYRxtR12YCKdZ46FcJZ4/cLbG4I2BEYnnHBC9CZGvaIeIJbJrVUBV/JGRjjN/fbbb6zjS967Ggm4OlU/8uNpdroZD1xf//rXY11GjDnFFFNEgSRhIml/eNtDtATzRsfcbPnK0o9rAVez52nppZeuiZbwOkh9bNeqirDK9lN1204KuCgHHF588cUYXrOsXC6TgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJdIKAAq4KFDfccMOwwQYbxJS33nprDDlY3AyvPt/73vfiYkLu5UKsoUOHhmWWWaZUFILIihBeWHG7FMKLdYiwLrjgAiZLbaGFFopeckaOHFlbT0g+vAzVC22Wh3wkjCLHVjSEaIQ2rCdoQezCPl5//fXipl3mqwi42CCJivD4s88++0SBVpeMxszgUQzxGfa3v/0tnHrqqXGaf50ScO2xxx5h8cUXr+VLiLV0nmoLOzSx4IILRgEV2eFxa5pppgmHHXZY2H777cOcc84ZCOs33XTTRREJHtaK1qqAK203YsSIcNppp3XJFqEf9Yey1Dv3bNCp+tFl503ONCPg+s1vfhNzP+KIIwLhKItGyFHaErbTTjsFRJs9YeNawNXseULYmdoVIVkRodaz2WefPfTv3z9ev+qlYXlVEVZZHlW37ZSAC3HoDjvsEBZeeOEo+Pvzn/8cbrvttrKiuUwCEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCTQNgEFXBUQEsrupJNOqnmrQkyFV5oUqjAXeCF2wdtN7iUpX/+Pf/wjDB8+PDzzzDOBEF0IK6accspYiqIYCXHUiSeeGL1mkQCvVBdddFEU1LAOEQgeusgnedLaZZddwqeffhrzO+6448IMM8wQpwnPh2AnlZljOvTQQwPiC6yeBy6EUgimsLvuuitcccUVUdSy3HLLBbwZDRw4MPTp0ydcd9114aqrrorp+EfoSfaRjOPgOBFCXH311WlxPJZcJIMHJMK3YR999FEUwj322GNxnmPeZpttap6p2A5BEyEgkyG6wIMVFXvPPffsEvqNciIko2yNPIGRF0If9pcb+SGm6glLAjfyToKpjTbaKKy//vq13cHu8ssvr80zwTGdc845cRmCG+pQbvD61re+FQgDhwe1nHXaJ2I56hXnl3O01lprRU9UeKjCKA8iRH6L1mr9KObTzDzHnMrGdj/96U+j9yxEdgizknFcqS2kZUm0xrrLLrusSzjKvn37hmOPPTbWD7ajLXXS8jaBMIjwpBhe1ZLRPvNrR1rOeUn1kTadjvNXv/pVeO2111Kyul76WjlP1He80GHPP/98FPmltoY3v+WXXz4su+yyMcwmaWB79913M1kzhFDJKOsss8wSaM940EtGvcrrZVreyrZJLPv2228HxHjwxIsewk7OOaI8lsGT+o9RrlGjRqXdxt/kFS8tZBtEnVxXNAlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkECnCSjgqkgU4cHhhx8exR1swoA+IQ0RGSRhBUIEPBk9+uijXXJFXIUALBc0sX3aLk/M8r333rvmeYqwbngYSmlZj8AjF3Sk7YuecnIBF2nY9oMPPoiiH8qU8nz44YcD4R/LDA9M5MP+GtnZZ58d7rvvvpgkiagapc/XFcVjCHIQuCRD3IGgJmfNsSBcSvuknIjEcsZsf8kll0SRDqIwxGG5kS8ee8rEI5xHQuslY38I8xBC9YQRBnLRRReNWSMmQVQy44wzRkFR2h9itdGjR6fZKABacskla/NMvPvuu7H+5GK1PEEutEveyvL1+TSCF/JJVgxXyfJW6kfKr5VfhIOIcKpa0ftcEnCl7TlGvJ5xnBxLsr/85S/h0ksvTbNt/xYFQY0yJJQpbSLZ0UcfHWaeeeY02/CXa1IuCEuJWzlPiOSOP/74Llxoh7AqtjOufb/4xS8CwikMD26E/szrTypL2S/X1uQRrZVtCU+L+C7fH20Wr4WcY65JyRBn4u3wzDPPTIsCAtfknY2FRQ98LDvmmGMCoTc1CUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQQKcJKOBqgihh7LbbbrtQFM0gEHjxxRej2KGehxa2QcSRe5VhOzxBLbHEEoFQihjL8ByTPN2wDCEP4qF5552X2S6GSIF933TTTWN5vzn44INjaDPyRKyVBFspA7Z98skno7iszLtSSsf+DzjggJo3r7QcIRmexPAohugkGYwGDx6cZhv+UgY8gRU94OA1Z8stt+zCK2WEpyUEZ7mYCbEJHqjwcpSMvPFYBZtNNtkkrLPOOmlV/EXsgleqopcmVhISc9NNN435we+WW26JHpu6ZNDBGQRrCNewYcOGhRtvvDFOI6CZfvrpo+cfyppbmSgIAQ35cK4RqOQ82LYoaMILVFl9xtvbtddeGxAPJVFMyjsvA9PN1o/i9s3ML7300tEbWLEu18ujeLwpRCf1vciGPFjOuS56OquXf9Xl2267bVh55ZUrJUfIhKApGd62kqe8tKzebz0BF+lbOU8ItRD64W2rjDl1Ao+Ef/rTn7p4DuM6d/LJJ5cyLpaddpqHtGxl21dffTXuL9XVtA88JT799NORZyp/EpsiqkUsht1xxx3Rq2LajtClXPPS9frZZ5+teT1LafyVgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJdIqAAq4mSW6++eZhjTXWiB6yXnnllYDnlwceeCB6ZkH8gces999/v4u4KN8FQoz55psvIDjIRU95mnrTiCnInz9C+SHcIh8EEGWGOAFx2P333x+FKQjAECbgWerll1+OoikEH1UNccRXv/rVGL6OsvdUOMG8PBwDYRrxQISwhVBuCKrGlXG+3njjjbqMO1kORGOE8ctDUXK+Vl111Sgwaba+VC0bwkQEZLB+/PHHw+uvv1510y7pxkf96FKACjNzzTVXDOP3yCOPRHHRgAEDwqyzzhrPL3wRB47L+lWhyB1P0up5oi7SFrnecO3hr0z82PECj8cMqS+I1HrK8954PDR3LQEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACvYiAAq4mTgYh+AjFV88QfiCOICQbHrM0CUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACjQgo4GpEp7Buzz33DIsttlj0ZoX3Lbz1zDHHHPEPr1YYIi5Civ32t78tbO2sBCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCCBrgQUcHXl0XAO71qLLLJIeOyxx8ZKN+WUU4Z+/fo1HRZxrIxcIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlMMgQUcE0yp9oDlYAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAR6GwEFXL3tjFgeCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSGCSIaCAa5I51R6oBCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJNDbCCjg6m1nxPJIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAApMMAQVck8yp9kAlIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQggd5GQAFXbzsjlkcCEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISmGQIKOCaZE61ByoBCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJ9DYCCrh62xmxPBKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpDAJENAAdckc6o9UAlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUigtxFQwNXbzojlkYAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQmGQIKuCaZU+2BSkACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAK9jYACrt52RiyPBCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJDDJEFDANcmcag9UAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEuhtBBRw9bYzYnkkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQggUmGgAKuSeZUe6ASkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQQG8joICrt50RyyMBCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJTDIEFHBNMqfaA5WABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEehsBBVy97YxYHglIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhgkiGggGuSOdUeqAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCTQ2wgo4OptZ8TySEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAKTDAEFXJPMqfZAJSABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIIHeRkABV287I5ZHAhKQgAQkMJ4JfOUrXwkLLLBAGDlyZPjwww/Hc2ncvQQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAT+j4DjWNaEiZWAAq6J9cx6XBKQwCRBYM455wwrrrhimGaaaWrH+/nnn4c///nP4fXXX68t68TE0ksvHRZddNHQp0+fWnbs48Ybb6zNOzHhE/jGN74Rdt5553gg//rXv8Jee+014R+URyABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJDDBE3Aca4I/hR5AAwIKuBrAcZUEJCCB3k7gzDPPDFNMMcVYxbzrrrvCeeedN9byVhd87WtfC/vss0/p5jvssEPp8t60cJ555gkDBgwIo0aNCs8991xA5DYx2qyzzhpWXnnlKN678847WzrErbfeOqyyyipx288++yzstNNOLeXjRhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhLoJAHHsTpJc/znNd1004XFF188FuTFF18M//znP8OXv/zlsNxyy4URI0aE9957b/wX8osSDBw4MAwaNCg89NBD4ZlnnumRcing6hGsZioBCUyMBOgQfPOb3wxf+tKX4uFxwzjssMPCO++80+Vw11tvvbDOOuvUhFUff/xxOProo+MNp0vCDsycffbZYfLJJx8rp/vuuy+wrlO2zDLLhKFDh5Zm11sFXDPOOGPYe++9Q79+/cJkk01WKzvirbfffjuceOKJ4ZVXXqktnxgmTj311Nip4ViOPfbY8OSTTzZ9WN///vfDkCFD4nb//e9/a964ms7IDSQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSGCiJcDgOuMmU089dekx/u9//wsffPBB/LD6kksuCaNHjy5N58JJm4D1aOI//zvuuGNYdtllu0T4aXTUjOPhjOHII48sTeY4VimWCWoh47Y4kGD8OY27pwNgbJL7x5RTThnHc3/605+mVeP1F6HZCSecEMtAHaVe94TDEAVc4/U0u3MJSGBCInDWWWeNdRN59tlnwxFHHFE7DBTBp5xyShfBECs77REr7XD11VcPgwcPjp0ewikm67SAi3x33XXXwD5mnnnmMNVUU6Vdhd4o4EJot9122zXsDHJTveKKKyaqEJDnnntu7bxcf/31Yfjw4bX5qhMLLbRQDJtI5wkBWOqMVN3edBKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJDDxE/jlL38Z5p133koHyvv44447rqWPjivtwEQTLAHr0QR76ioX/KSTTgpf+cpXKqcnYSMHA45jNYWy1yWeffbZw7777hsQRHVnn376adhll126SzZO1uehO9nhgQce2COOQhRwjZPT6U4kIIGJgcB+++0XFlxwwS6HwkMHNw46EtgGG2wQNtxwwy5pmDn99NOjO8WxVnRwAcKxaaaZJubYEwKuVNQtt9wyrLbaamm21wm4ZplllnDUUUeNJaKrFbgwgRe1559/vrC0d8726dMnqs7rlS4XcN1www3hyiuvrJfU5RKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCCBlglssskmMRpJ1QyIVvKjH/2oavK20/GRMn94cplUjIgtn3322QR1uNaj3ne6Ol2PTjvttLqe+uodPeOvP/zhD+utdvkETKBM0Md1i2v1FFNM0eXIxrWAq1HdV8DV5dQ4IwEJSKB3EMCrEx6vcrvsssvCzTffHBchHJp11llrq+lg7L777uHDDz+sLWNivvnmC1tssUWYbbbZouqchwgUtW+99Va49tprw6OPPtolfT7DFy0rrbRSzANPWK+++mr405/+FL0mdSfgQgDEfhdddNEwwwwzRE9an3zySQwDeffdd4c//vGP+a5Kp3u7gOuggw4K88wzT63sHB/hEp9++ukw99xzB1xt5kp/wikm95vf/e53w1JLLVXztMY2559/fswreTtL3sfefPPN+MVQbUfZBEK/zTbbLNaFtC/OLyEbL7jggrquovv27Rvgu9hii8XObHq4xM30v/71r9C/f/8Y63nPPffM9vb/J9sRcPGQtPTSS9eOnVwRJhL+s7v40hzjtttuG7lPP/30MQ/qPGzvv//+seoV9ZSO9xxzzFF7gCb843e+853w1a9+NUw77bSBThmxrjl3//nPf/7/QTolAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACvYLA1772tbDPPvvUykKoxAceeCC+yyZSxvLLL19bx8RPfvKTOBaSL8QLC++Xef/NNOG0eCfMeAlRJh577LE8eW2a8Q4+ql955ZXjO2UG3hmTQSj22muvRe9gvGPH00sK3zhgwID47j55fiHKChFUeC/P2A558D7+5ZdfDr/5zW/iO+7aDrMJ8mUMYPHFFw8zzjhjbayF9/gjR44MF198cWBsIretttoqLLLIInEfLGcsCE80cGK8BgHDu+++Gx0CjBo1Kt+0Ns3YB+Hj5pprrhhejBVpO8pEBJVHHnkknHzyybVt8olWxy7yPHpiuhP1qNXxr1bqEWMijHFw/uDOeAjMESgSyYZxJOoh9W7YsGHh8ccfL8XWSj361re+FVZdddVapJwnnngi/P3vfw+bbrppPP/U4ffffz9cffXV4fbbby/db7v1qNk2e+aZZ9aEOYceemhsG1w3GE/CYIWzBexXv/pVZFom4GpmHIu6/r3vfS+OB5LXbbfdFsdW+/XrF4VCMHvooYdiO2ZsjnZE2Eai0hTbLuWC+5AhQyJjxrhIT3t/5plnwoUXXhjHtEinNSZAPV1rrbVqieB4xhlnhL/97W9xGW3q4IMPrl3fyjyxtdJu0g7bqfsKuBJFfyUgAQn0IgLc7BHy5IaA6oADDojer/CCVbQ8xCAdQR5QEOg0MjxC4U44F35xQ9p+++3Diiuu2GjTuK7MAxdxhOlQJgFSWSZ0NogpjTipnvVmAReCuGJM7F/84hfhjTfeqB0Ox4/aH57J6JC98MILAdV3bjyo/fjHP46Lzj777NqDFQvKOo90jBFX8bBRz9jupptuCpdffnmXJJSHDn4S4XVZWZjB6xsCJ2y99daLD6nsu5GxXx528cxVNEJ/IqAqM9jUe0AmPR7nKAN1u54hAIMtjLGyL2ropJUdQ34O4sb+k4AEJCABCUhAAhKQgAQkIAEJSEACEpCABCQggV5BALEKgoxkvEdnfCJZLtxgWTHkFO+K11577S7v69O26fcf//hH+PWvfz2WJ6299947CqJSunq/iMpuueWWuLoYro/35lg+XhAXjPnHuiuuuCLceOONaVH8XWKJJcLOO+/c0KMQooNLL720JqApciKjeu/E2W8uOks7Z/zjiCOOKC1rSsMv4jccC+TWzthFnk9PTRf5NFuP2hn/aqUebbzxxnFcJOfBOUd8WGaIU4pjMK3UI/I+66yzuuynUR3O634qVzv1iDxaabN5mRmnpMw4D0BwiH300Udh6NChcfqcc86pjTfl46vNjmMxVtfdWGzcYeFfUQAJL+oI0X/qGeeecj/44IP1krj8CwL5+WUR7SKJtxKkFVZYoeZ9reiBq9V2Q96t1v299tor3mvK7hOpzPxyTUeI+M9//jNf3PS0IRSbRuYGEpDApExguxIPXHQ06Fiss846Y3XYYLXjjjvWHiya6TDgfQiVcbLddtstekhK841+iwIuPE+RV35zodzcTIodSsQ2dEbquRXuzQIuFPDbbLNNDQ0er3ggLFrxPCBq+v3vfx87vrkQKRcPIa7CM1Qy+NHRzA1PXgsvvHC+KHqx4uEoZ08CFPl33HFHLe3WW28dVlllldo8YjoeTPl6Z/755++y/a677lr7AoDOAJ6sqhhfHe2///5jJaVsPBDlx54SNRJwUV7KXcX4YoFODqJERIh5x7u77fFa9te//rW7ZK6XgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCCBcUigkfCGaAsIkZLxjph328nWWGONsPnmm6fZ+Mt7d6z4Pr0oqih6bCJvPOGw3UILLdRl3COPooKHpEGDBsV9VPlHeXgHnz5O5l08grVi+erlhfCMciE+wQFA1e3wDkR0itwQb+GdJhljSK+//nr0XJYvLxNwtTN2kfbXk7/t1KN2xr9arUdFTzxV2FxzzTXR6xppW61HbMuYTfIgx3wjQwiCU4nc2qlHrbbZ3EFCGhuqJ+DKxV4pLeWn7TQzjsW15utf/3p+6JWmc49PjJ/Cu+h4AVFRMdQf14r99tuvi0OJSjuchBIhguOamKysfqZ1XGe5LjBOmhx3tNNuyLfVul8UnaUylv3eeuut0QNj2bqqyxRwVSVlOglIQAJjCPBVBR0zDIFNcu951VVXxdCK3HzoHNMZTx3x5BKY8HSIsHLDlTBekegQIABDEJO2I92VV14ZPSbxwPGzn/0s3zTgpQu3qwiGcPubW1HAdfzxx9fKSjq8P3EToZNB5xaRGb/J+BoFZX6Z9WYB1w9+8IPoAjWV+957742q9zSffvEYxRcSyXAxi8czzh8PMklJnwu4cImLAIkQllhRwLXccsuFnXbaKWUZXePyQPbSSy/FjhxukBE7pfOLQA7uyfLOKsvw6oZ3NwwRF57AcO2Zdx5ZVzwWlpUZ5cUlM39lxvknfCS/yy67bC1JPQHXlFNOGV05p+NhAwRz5513XnRPDQ881uUCQVxHE2YU48uHPfbYo8aDZbCifGybP0Tz1QIuVDUJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhLoPQSKwhtCofHhOOKGosAhF2EhiOCj6fR+mffevE9P0SB4f4wAY+qpp64dLO+WeceM8XE1XlqSXXfddYFxGox30nyMT/hG8ickWwpJyEfMKapE2pbf+++/PyD0Iowi4zi5QObtt9+O4wakO+aYY8JMM83EZDQ+WOYDZARXjNMgOEnjRiSAB6IxDFZEDMnFIIxBEOqObdZdd90aD8afGI/IjZCOiRdjO/DhvT/GOBHzjGPg/eWQQw6pbdru2EUtox6caLUeUaR2xr9arUfsd9FFF40RWdI5YRkf0RMBBkEhEXUY20vGueIcce7aqUecY8YdF1hggZR1bHM333xzHFPaYosturSbXATFBq3Wo3baLPWRsSccSOBgAcvHxHIPXAcddFAcCysT9zQzjoVjBdoe417JGHMljzy0K2EmKdf666+fktUcEBQFn3iK+u1vfxvTc43AG9lKK61U2y5FbKotcKILAVgxjpuMkJWHH354mh3rl3FI2lKydtoNebRa9xmfz9tyKk/xl/sY9ZrQvO2YAq526LmtBCQwyRFARLPkkkvG40b0M3DgwPgwkIu5EE8h8kqdtp///OdRzINYhXjoyYoiK5YXxTgIWuisFLct3tToyNNZTJbnTWcuDw1IzG06GLnhWQpxWrJGnYzeLOAiZnYevhDPWnTIipa732Qd8exxnYzleeQCLtYRW37IkCFMjiXgKn7BwkMbD3a5Edu5f//+tUV0TDiXWOqUppV05uk0Uja+7kHERNxybtx5hyWlpzPK1wfJcOuMODAZD81VrPhVVD0B1+DBg+NDcMozf4hNyxCcwTW1haL4LP+SAkEbArj0wJm71kasyFdOmgQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCfQeAkXhTb2S8bE1g+fp/e/qq68ePwBO6e+8884ookrz/DLmseaaa9YW5R+eI5zgo/jc+Liej4wZV0FowXt1BFu8Z89trrnmiqKutCwXlbCsuJ4yI/QphnDjnTZjM7y3T4Z4DGHaVFNNlRbFd+S858eK4wC5KI2PuonGgTEGgNgnt/x9Ossp1xtvvBFFO3j5wmEAojm8cuXW7thFnldPTbdaj9od/2qnHsEiPyecD0JXIupLlq9nGWN1iBDzcIqt1KNiuW+77bZw0UUXxd0iUNxggw1SEUIao0wLimWqWo/abbOMF+E1jv1h9QRcrKMd0H5TWpblVnUcK48iQ144VUDQlTvbQFCGUCj3DIVwC/El54l2jzHGRtsulom82B4rOm6IC/1XI4DTB+pRsrLQomld8bcT199W6z5lYZwT4V8emQlhYrq2k6bqOCxpG5kCrkZ0XCcBCUigQCDv6D766KPxplwMmcfXHHS0Uzg6LuB88VB0a5o8cxV20UUBnL7OoOOQvEKRHm9EeSzlYmclF3DlHZTivurNp/2Wre/NAi6+JCA8X7K77747nHvuuWm29suDHR3cZLk74lYFXHwdlH85k/Ju9IvA6vrrr49JiuEfy7bj5k8owdQRL6bJj7WeeK24TXG+WJfqCbhyMSN5DBs2LCAaK1qx7tIZTg/MeWepKNLKv5rJBXbF/J2XgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCCB8UOgivAGQQvCllz4sPfee4dFFlmkqULj1YQQWBiesgirlT4eLsuI/T399NNxPIUB8WRFgVZZyCuEGnz4niwJr3JvRoimctFHSluMFJIirbA+5cM0nph22WUXJqMNHTo0LLPMMnGasYA84gcLix/6x4SFf4jYeFfPOEKydscuUj49+dtqPWp3/KudegSPfIwD71v7779/F0xF5wuMoSAwbLceFQVcuUir6HEtjVGmgrVaj9pts2n/6beRgCulqfdbdRwrrx9JFIkwNHm34xqBIKdv375dQpYiykKMdfrpp9crQt3l5I3TD21sAquuumrYaqutaivya3ptYZ0Jxn7bbTet1v1UpGLo1AMPPDCKhtP6Tv0q4OoUSfORgAQmCQJ0vpJb0hEjRsQvGnLPVUn4lMdzxqXjU089FZX1fA2QjO3wSFS0fNuk1sblau4qmIcdOuLJ+KqDjmKyXMBVvCGmNPV+6bDw4EHnqcx6s4BrrbXWCni5SlZ0FZyW5w9CLMvdK+cCLhjDOhkubwmFiKWOXVrX7EMQ5/7ggw+uhUkkH76K4Bjyr3NS/vlv/jVFvnx8CrjwOJY/FKZypTjVaT7/oih/uEEQmXuKO/bYY2PoSLZTwJXo+SsBCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIoPcQKApvCEf28MMPBz4AzsVVd9xxR7jwwgtrBc/fw9cWNpjgffwf/vCHcM0119RSES1l66237hLSsLYym8Aj1X777VdbUhRw5Z69UqKigIuP+4n+wRhBsvzD8LSM36KXmfx9fi7gYpCeD/2TMWaEQAArE3ARhQPBFx6EkgOBtG3xN/8wuxNjF8X8Oz3faj3qxPhXq/UIBvkYR1lkGz5oJxxoMurvW2+91XY9Kgq4cmELIkDGwJIVBVyt1qNOtNlUJn57u4CLMuae0pjvzmjTnHPGdrWxCRSvvWWhYsfe6v+WMDbb7vW31bqfyqSAK5HwVwISkEAvIkBHh/jIGDHREVvhDSvFcsfN7/nnn9+l08YNHjeQxDYnBnoywhjyMJMb60mXLN28+Kpk9tlnT4vjgwpx0ZMVO6m5gIsvCBDNJEPUhKisniFCa+TmsTcLuIgTf9xxx9UeDHmo46uAXO3ODZpzxm8yvpJBtIblHeriFzC5qKgo4CJU4LzzzpuyjF/e8CVFmbFt7kaXNJQdd9CIoAizuPTSSwceWvr16xfDdOb5FB/s0rpcwFV8IE5puvut+uVC0YvZY489Fr3M5fnjNpaHQwSGGMfNA2YSLuYPNwq4cnJOS0ACEpCABCQgAQlIQAISkIAEJCABCUhAAhLo/QSKwps///nP4fLLLw/FMG68G+YdOiEOsY022iisv/76tQMkUgVRJcqMbT/66KMuHrx458z7dPJ79913AxEuEAfMMccctZBneV6EI8QDD9adiKC4nv3jpYd3+ESOSPbBBx9EoVqaT7+HHXZYLEea5z0440lYOwIuwsotu+yyMarHSiutFAYOHBj3wxhQLpZjP/n7+nbHLsivp63VetTu+Fc79Qgm+RgH9QThFGNsyfL1LGMsCkFhu/WoHQFXq/Wo3TabmKTf3i7gYgzvzDPPrI3/Ml6IkLPe+CnjXukak47R37EJEEo3v14x7vrkk0+OnXDMEsSPMGX8thPX31brfipcUcCVHLik9Z361QNXp0iajwQkMEkQyIVUKTwfgpf11lsvPiQgyuLmnd/UuRndc889YeONN47pEihu5nTWnnvuubgIgRbuFlM8ZRYSJhGxUf7lBct5MOCLES7ipD/88MPjzYt1WC7gYj736sU8IjPEZsnwDDZ48OCwxhprhOmmm24sgVhKx29vFnBRvjxOPPMwwtPVO++8E1nxsDLbbLOxKlrRre0222wTH/bSerxCIS5aYYUVusQ2Tg9tKd3mm28e+aV5vqIgnCb7T8YD1brrrhtw0Uqng848+WDp6wXm8WaVn5+FFloodgxTp4a6k3t+S/nnHR9Eazwo8osYjPNLPnQyKdebb76ZNuvyW1XAVXygIpNLLrkk8LUShngLMSJxzZMlQWKazx9e8gdK1udiOR7CaRuaBCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJ9B4CxffEScBFCXOBBvOjR48OhHrD5ptvvi7vfBlX4T1+EniRhjETxizwvILQhpCJhNzC8o/aEUfxrjkZ4dD4qD2PasK4AJEesKJAi2WM91x22WUxNCNesRALJMs9K+XvtFnPx/tEUEnv+Yvet1jOu/wk+mhHwJU8aTE+wPEgBEpWFMzxcTn7wtodu0j76MnfVusRZWpn/KudesS+i/WB+ov3NsZ/dtxxx+i1jXQY5+1HP/pRrAvF7ZqtR80IuBgPGjVq1P8VYsz/VutRu202FQDvcYx10Z5nnnnmuBiBJl77aC9VvFdVHcdqJ4QiAi7Ycb1IxpjtOeecU3NSwHVp0KBBceyXOtxMSMCU56T2W4yQRFthPDBd26kbq6++ehT4pjC2qQ63225arfvpHC2xxBJdvCY+8cQT0UsbTl4QdzEOS31544034lh12q7ZXwVczRIzvQQkMEkSoEOBy9k8BCIgCI2Ye7OaZZZZAiHjEK8ko8OBN6SLL764NA9ELXTeZ5xxxi6qYzpzqLnfe++9+LDADSx3i0snhi9LeJDIvUmxX/b5/PPPR2EX89/+9rfDFltswWTN2B4RD4KtXDRGAkRliMIwhEvbbbddzYtSXFjnX70vTuok75HFiLMQLhWZIFxKntLSjuHEg07+UDhkyJDw/e9/PyWJv5wLOmJFY/mJJ54Y1eHsj+lpppmmSzJutHQ+6YgmAVZKkMclp7OaC8s4B4i4EJjhwpeviZIVXT6n5cTj7i78Imlvvvnm+DDKNF8kIY6qsl3xITr3VkZeGHWAji31uXgOkjc6OjBwLzLlPHDu+Pokf7gmX9oB+9MkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhg/BNg3AFxUPG9d4q2wLtnxkvy9byT5703HzEz/rHwwgt3ORAG8/kYm/fpxffHV111Vbjuuuti+uKH2IyzIB574YUXYhSVTTfdtLY9YyF44OJ9PlYm4IorSv5R3v33378mlsoFISk5Yw+UmbGa4nt2Im5wrBjvvXNhGMs4XqKI8IF3HuGDdZSXcRqiqmD5B9yUCzEJwh/eya+11lpdxEJEf8HhANbu2EXMpAf/tVuP2hn/aqcegaQoKGmEKf8Avp16hMAQIUlu1HHKgrBptdVWG3bZV5IAAEAASURBVKvNDR8+vObhrtV6xP7aabOMryLqLLaR/DiY5lgOHiPmTIJLlrUyjrXIIovEcdH8+vOf//wncsrHmpLTA65VyRgLQzREOyMyU54Hy2jvtCvGV/N1tGeuNVp9AozTIqQqjtdybhhLLRvzRqSJ45J22g0laqfus/1MM83URRPAsnqWRGf11jdaroCrER3XSUACEviCALHN6UAXjRs1rnOTFcPKpeVJ2IQCm68eig8eKV36JV9U+jzoJENxzNcbVa1YNtTrCIG6s+K+99prr+gxqrvt0vr8S5a0bFz/cr546Mk7TsUy0AnEExlf1uTGNnlYzHxd2fQDDzwQPa6xDm9TPMx1d35Ji1gp9ypVFHCRpsw4P0XRWUr3gx/8IOA6uZGx/b777hu/diLdBhtsEN1ZN9omX4cb6yuvvDIuooOKFznEWt1Z/uVV8euQfFs6UHmbytfRgeYBXJOABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIIHxSyD3JpWXBPHDTjvtFBcVI3qwkEF6vLDwHh1BR1HUlOeVphFZILrg42GsKLxJ6cp+CenI++lkVQVcjCHwYT5iqNyKHmTydfk0Xo8YxMcQoPDxcpldeOGFYdttty1b1eVj7Fx8UJr4i4UIIRD5UP5k7YxdpDx66rfdekS5Wh3/aqcesd+qAi6i9HD+cmulHjXaJ1FO+vfvX9qeci9y7dSjdtosnvS23377HEHd6RtvvDEMGzastr6VcSzEbAMGDKjlkSbId7PNNkuz8bds2TXXXBOuvfba6AWQ9I3GG1Nmf/nLX8Kll16aZv2tQ4DQiLvvvnulsdRiFKVW2w1Faafup0NhTBRnLo2sXYcUCrga0XWdBCQggS8I4PkKRXDuFQghDHF3ecBIxkWbMIu55yHSpS9KSIe6HIEKYe2KN/yUJwKi999/P2Vb+0WYhIvVopcnPGmNHDkyrLjiirW0Za46l1lmmbDdGG9aye1kLfGYCfaNov3cc8+tfdHB+uWWWy66ei2WNd82TeMRDOFUbzDU/AikijdSHlq44eNR7e233y4tKiEO6TzkCnC+duFBb6uttqptwzLcI+dCO84vD6eI5cqY8fBEZ73YiUNURYeykbEtHUm+2qlnlA8Ver5vzi0dBsRPF110Ufw6IG0PH77iqSI64+sBHjLzLx/Ih84zoSHL8iCUJCFFU6hQ0iNk5AuKvJ2wHLfO5E/YyuJXGHxBRd3iWDQJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhIYvwQIb1gUNvD+9vHHH4/v31PpGNNgbIJ31qy/9957u4hZNt544+hBquz9Mu/giYSCUIb348nwOIWXrUaGkIyxmeSJKqUtCrgQahCJZIYZZohjAozN8AE2YyX1xhB4/48YpRi1hX18/PHH4Xe/+12MzJL2yS9jEnjsyQ0xG5E6ePeNyCo3RGuEguS9PlYlAgdjRbyPT9vEDb/41+rYRZ5HT0x3qh61Mv7VTj2CRS7gGjFiRKxDjH/AmnPL+cD7FeKqMmulHu2xxx5h8cUX75Id7YT6uuCCC8bwc/lKxsTwXnf99dfHxe3WIzJppc0SCYg2kI+75eVM04xD4fAgeZ5jeSvjWIynEpkoHyvD2QbRjlJ4UfKmnR1wwAGxraW08MShAGNWGAJMhEP8lhnXiauvvjrcddddZatdVkKA+oADkfnnn7/LOUpJETEhiENEV7RW2g15dKLuU24EowMHDuxSLOoM9YCwioz/cv9p1RRwtUrO7SQgAQl0gAAh87jI82CCAKsojKm3C75IQaFMx+uRRx7p8uBSb5t8OSEeEQvNPffcUdhDJ5KQi+3cUPL8e8M0x8eDT+pwjR49Ogq3/v73v8cbKMIiFP88YOF2Of8ahfKz3XzzzRfdFiPUy8MsVj2+2WefPXJG+Adjzi/lKDO8WPGQ+PDDDwfKhntX3ESzLfMvvvhirbNYtn1xGXWLjg8dzKr1qphHs/N9+/aN9Rk3ojwgItqamOpUszxMLwEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAtUIIG5ivIT34oQo40PsXMSR58KYCh8y41mIAXO269evXxRI8TEz78TrvZ8uCrgIkVZvP/k+y6YZa8HLD2MtlJdxnuQlrCx9O8sQ57AvRCKIYDhmRGewgAPv5OsJzor7bWbsorhtb59vZvyrnXoEh1zAVfQa1QynCbUeNdNmm+HRW9Pi7INQp7RFhF+M+3HtQHCjtUaAsVh4MiaK0weuozgoeeONN7rNsNl208lrKHUB0S3XUsaQuWd1yhRwdYqk+UhAAhKQQK8hgCjruOOOK/UIVVbIyy67LLoiLlvnMglIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCZ8AogFllpqqbDbbrvVDgavLAigEAzgRUWTQBUCxYg8d999d7jhhhsCkWpyb3FV8jKNBCQggURAAVci4a8EJCABCUw0BNZZZ52wySabVD4e4h4T1lCTgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhg4iNAxIr999+/FrWjeIRE6SA0Vk950Cruz/kJkwAiwFNPPTVGT6l3BGeccUZ48MEH6612uQQkIIG6BBRw1UXjCglIQAISmFAJEMLvsMMOix64cF15//33R9fFCy+8cOCvf//+Ydppp41ulO+8887AlxGaBCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEyeBjTbaKKy//voND46QeIwnaBKoR2C22WYLRx55ZL3VcTniLURcmgQkIIFmCSjgapaY6SUgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIYIIhwIffhxxySKnnpM8//zy89dZb4cADDzSM4gRzRsdfQXEggJALb1xFI3ziaaedFnAuoElAAhJoloACrmaJmV4CEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkECHCCjg6hBIs5GABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJNAsAQVczRIzvQQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQggQ4RUMDVIZBmIwEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIoFkCCriaJWZ6CUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACHSKggKtDIM1GAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpBAswQUcDVLzPQSkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQ6REABV4dAmo0EJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIIFmCSjgapaY6SUgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCXSIgAKuDoE0GwlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAs0SUMDVLDHTS0ACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAIS6BABBVwdAmk2EpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEmiWggKtZYqaXgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCTQIQIKuDoE0mwkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQk0S0ABV7PETC8BCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSKBDBBRwdQPya1/7WjcpXC0BCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSKA5Ak8++WTcQAFXN9z69u3bTQpXS0ACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISaI7A+++/HzdQwNUNNwVc3QBytQQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCTQNAEFXBWRKeCqCMpkEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkEBlAgq4KqJSwFURlMkkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQggcoEFHBVRKWAqyIok0lAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEAClQko4KqISgFXRVAmk4AEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQqE1DAVRGVAq6KoEwmAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCVQmoICrIioFXBVBmUwCEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISqExAAVdFVAq4KoIymQQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCRQmYACroqoFHBVBGUyCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSKAyAQVcFVEp4KoIymQSkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQQGUCCrgqolLAVRGUySQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCCBygQUcFVEpYCrIiiTSUACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAKVCSjgqohKAVdFUCaTgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCoTUMBVEZUCroqgTCYBCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJVCaggKsiKgVcFUGZTAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKoTEABV0VUCrgqgjKZBCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIQAISkIAEJFCZgAKuiqgUcFUEZTIJSEACEpCABCQgAQlIQAISkIAEJCABCUhAAhKQgAQkIAEJSEACEpCABCQgAQlIoDIBBVwVUSngqgjKZBKQgAQkIAEJ1AissMIKYfbZZw/PPvts+Pvf/15b3qmJeeedNyy99NIxu4ceeii8+OKL3Wa98MILh2WWWSamu/TSS7tN324C+lCrrrpqmGyyycL9998fXnnllZgl5eAPu/baa8P//ve/OO0/CUhg/BP40pe+FNZZZ53Qp0+fMGLEiPDPf/6z5UJNNdVUYe21147bP/jgg+Gll14Ka665Zvjyl78cnnvuufDoo4+GZZddNsw111wxTX6dKO6U6x3XPeypp56Kf8U0zH/1q18Niy66aFw1atSo8Le//S18+9vfDrPOOmt48803w0033VS2Wa9Y9s1vfjOW85133gm33357j5Rp+umnD0OGDInX5ZEjR4bHH3+80n46fd3+xje+Eeacc87w3nvvhVtuuaVSGcZnoqWWWioMHDgwlvnjjz+Odfmee+4Jo0eP7lKs1VdfPXzlK18Jqe59/etfD3PPPXf417/+FW699dYuaXvTzITSRrpjtuSSS9ba/7jo53RXnmbXV7lmkucCCywQllhiiZj9ddddF/773//G6XFxDWFH9fp39coVC9eD/7g3cO779+8fJp988vDqq6+Ghx9+OHCNyy3dR3gZ+pe//CXeU1j2+eefhz/+8Y81jvk2E9r0hN4GqvDmuWLdddcN9Fd66jmrSjlM838EJoU6N7Ge63buOeOyHda75/SGvmSjft9HH33U8fdBPf2OiWfAKaecMjTzLDLzzDOHwYMHx2Zy7733htdee62tJjO++hJtFfqLjcdlu+hEec1jwiMw9dRTh4033jg+y9922221d6yNjmRcPR80KoPrJCABCUigOQIKuCryUsBVEZTJJCABCUhAAhKoEfjRj34UB3sZ6P35z39eW96piS233DLw0hJDkMDAU3f2/e9/PwwaNCgm22OPPbpL3vZ6Bry32267mA9CrTRIn9iw4qc//Wn49NNPYxr/SWB8EGDwIgmIXn755UCbrWKtblcl7/GZZsYZZwwHHXRQLMLNN98cEAe0anPMMUft+sdg+R/+8Idw/PHHR3EYApcTTjgh7LbbbmHAgAFxF4888kg4//zzS3d31FFHBV5YYohmDjvssNJ0++yzTxTMsBLR2MUXXxyOPPLIKBr797//HQ444IDS7Tq5sNW6kcr54Ycfhl/84hedLFItLwRFW2+9dZx/+umnwxlnnFFb12ii3nW70bE2WverX/0qTDfddLG99cQ9stGxNLNuhhlmCEOHDo3CuuJ2iD4QcQ0bNqy26rjjjosCEq4lxxxzTNh///3jtp988kn42c9+Vks3PiYanY9U98ZVG2n3+Oebb754HXn77bfDu+++W8sur6fjop9T23GHJqpcM9kVfSv6WBh17IMPPojT6TwWryH9+vUL00wzTfjPf/7T9sAqO6rXv6tXrli4HviHWIt9Lr744qW5c5/hGscgOkYbRDj62Wefhb333jtsuummgUEt7JBDDokD1nFmAvg3sbaBKuinmGKKeH0l7TPPPBNOP/30KpuZpocI9NR1t14d76HDmCSzbeee06gddvrc1bvn9Ia+ZKN+3wsvvNDx90GpvfXUO6YTTzwxtgX6C/vtt1+ldoF4a5NNNolphw8fHv7617/WtmtUF+qt475e1sepZdqLJxq1i15c7Fi0Rs8Jvb3sk1L5+FiNZ1PsiiuuCHfddVe3h1/v+aDbDU0gAQlIQALjjYACroroFXBVBGUyCUhAAhKQgARqBHghlQYPzz333I574VLAVUPthATaIrDWWmsF/jDEQ4iIqlir21XJe3ym6aSAC09ESWiFeAsR17HHHhu9ZqRB129961vxK1KOGSHGwQcfPNbh52ViJcIZBt/LvPcdffTR8ctx0p155pnRU1d6aTmuxCmt1o1UzqL4gmPplHVawNXoWBut6w2DblWYHnHEEVH4QlrqHR7DGJxBDJPs+uuvD3/+85/j7K9//evAAMjzzz8fTjrppJAEhT15TlM5uvttdD5S3RtXbaS7sjZaj6eHAw88MCbBw96FF15YS54GNlmQ+mC1lRPARJVrJodRb3AzncdifUNMSL3FE1wS6LaDo95ger1ytbOvRtvm55v2yYAz4ize4eEFA0tiYaZ/8pOfhPnnn78mHF1vvfUC3lOwCemDgom5DcST0c2/CXmAvJtDmyBX5+2wU9fdRnV8goTUSwvdzj2nXjvsiXNX757TG/qSjfp9Z511Vq0v0qn3QT39jqmTAq5GdaHRunHdl+hk86zXLjq5j57Kq9FzQk/t03ybJ6CAq3lmbiEBCUhgQiSggKviWVPAVRGUySQgAQlIQAIS6ELg8MMPD9NOO20MQYbXmU7ahCzgItRG8pZAiKMyEUYnWZmXBBoRyF9WnnfeeTGsX6P0aV2r26Xte+tvLpZq1wMXA+h42cLwUnT33XeHNNBB+ER4I4RBJIMxAL/nnnvG6fwf4TzWWGONfFG44IILYoisfCHhAfGiguV5JVHDuBKntFo3CGNH6N233nor3HDDDfmhdWy6VQFXvet2o2NttK43DLp1B3WDDTaIYYBJR0g2xIHUK4ywa9tuu22czutauu8T5hMB4Y9//OMY7g7h1y9/+cuYfnz9a3Q+xnUbaYdBo0G/nhAStFPWZretcs0kz3qDm/WuIROjgAshFoIsDA93tD0EahjXUQRZhNjDTjvttBhOceeddw5f+9rXotALzyKE+aadY50SnsTMevjfxNwGqqCjnWyxxRbR2yHXWrxtauOPQE9cdxvV8fF3pBPfntu559Rrhz1x7nqzgKu7fl9aT0j6Tr0P6ok8U+1uRcDFPZf+B0ZIt5deeilON6oLjdbV6+PETHv5v3rtopcXOxav0XPChFD+SaWMrQi46j0fTCrMPE4JSEACEyIBBVwVz5oCroqgTCYBCUhAAhKQQBcC66+/flhttdXigC/hsFIIly6JWpyZkAVcLR6ym0mgRwi0+rKy1e165CA6mGknBVwUCwEXL7OTd7M06HDfffeFyy67LJY893KUBtrzQyLEHmFecnv88cfDOeecky8KQ4YMCRtttFFc9uabb8YBfWbGtTilN9eNVgVcXUBnM42OtdG6CUHAhRgQUSACrTKPb+kYwEEInRdffDEKCNkmeYbacccdw6KLLhpFeYceemhGbtxPNjof47qNtHP0jQb9ekJI0E5ZW9m2yjWz2cHNiVHAtc0224RlllkmIi7zbPKDH/wgIDzF7rjjjvD73/8+pG2SJzJCkdOfzkWYcYNe/m9ibwO9HL/FKxDoietuozpe2L2zbRLo9D2nJ85dbxZwpb5ivX5fT7wP6ok8UzVqRcCVti3+NqoLjdY128cp7tf51gg0ek5oLUe36gkCrQi4eqIc5ikBCUhAAj1LQAFXRb4KuCqCMpkEJCABCUhAAl0ITD311FE4gHjhlltuCddee22X9f37949fKxLu5Xe/+12YYYYZwiqrrBLDu+CVBgECX5an0Ez5xp0QcA0ePDgOfvHV5Keffhpef/31WE72Wc/mGxMacsUVVwzzzDNP9JzDNo899lhAjFEUqNV72YrnkoUWWiiGsLnqqqvG2hWD3wjf2BdM8FxCme69994wevToWnry4Q/j63s8+pTZd7/73TDddNNFjwtwzq2Z48m3y6fx4EA+2I033hhefvnlOJ3/oz+56aabxkXPPvtsuP3222urZ5111rDuuuuGueaaK5aTc8G5v/POO8NDDz1US5cmvvOd70Qub7zxRrjuuuvS4trvEkssERBp/Pe//w14OOO3O9twww0DL1IJ+8WXs5y75ZdfPsw555zxK1o8J/3973+P2fDSaKWVVgrzzjtv9J7GV7YMjv7jH/8o3U2fPn1iiCK241g5vpEjR8Zz9txzz5Vuw0LaB1xmm2228OUvfzl62OABBibwK3puS8fwwgsvxHpM/cbTW79+/WJd48tnjg2RBUbYkM022ywKgygXhpcd2BMS7bXXXovLiv+a2a7Zc1vcV9k8dYkX93gfwcMfHD744IPIFK9NTJcZPDintCnOCeEKqau0wXyb7gRcyy67bM2DXr06mO+fr83ZXxJmESKRMnAurr766ph0p512Cossskicpt5feeWVeRbxi3XyePvtt+N1h2srobLwoJJbPoj417/+NQwfPjyuzsUpDLQwaM/APm2OfKgbhHd85ZVX8uxq01XrYjN1o5Z5NsHXsbRDykGbyg1mXEMQsnHeud7Cn2sf19+q1qqAq3jdbnSsnNshY8R0lLVe20rip48//jgg0CtaM9fmdC9FhME5x+sO9Z1rIe2DsKjw5F7COgYoBg4cGGaaaaZYp7ge/fGPfxzrmoIoa/LJJw9cd1LIvryc22+/fdwHyy6//PJwzz33hP333z8eM/cr7jff//73w6BBg+K1Be9z3RnXKzzOYfQZKC+eZuDIec77EZ2sl+22kWav82uvvXasH1yDuG8Syg4xDuEnYVjPYME1ij4ExrWLe+rDDz8cRowYEfJrAB6VEM9R5xdccMEo0mF/cKwXKrfZ6+tyyy0X98G1iesZ3p24znJuqDvcH2mjDOxWtSrXzHqDm8VrCO1wgQUWiNds+qK0B/ps3N+4z+XWTJur178rKxd1g+UYfQnu0TDivMCb/h3X6+Sxgz4m+XP94PpMeu5t3LNyw8MW13CMUKXFvg71CcEW9uSTTwZCWXEN/eY3vxn7koT3pX4gsqQfjkizGSN/QgBTHwmZRNhKysi9BIFxmXXiOt7TbYByc27SPZJ5+k30sWljXGdzS31xrpNXXHFF7Jevueaa8dxedNFF8fym9FX79yl9vV/qFP03jLZffG6p2t+pl39a3my/Mm2XfjvdB+R+sPHGGwf607Qd2jPPRfS/eR7Izw19RJ4psT/96U+l/RvuT9Rdzm2ZFzPulYsttli8V3I9w4spbYlrXd7e2r3uJl7pt7s6ntLx28m6yv2Ea2in+xJ5ecumYct+aXM8r3Kt5trHMwl9k3rPIwMGDIjPSfQbuXfSBkmL91yum1Wt1XtOWTvsqXNX755T1pfsqT5hPZ7d9ft64n1QT+SZjq8VARfvC1ZeeeWYBdcb7oWN6gJ98EZ9OfoMnHMMvvlzKsuauZf0VB+EcpRZWbvI03Xq/pTyrPoskNKX/TZ6piu+E+H4CD1d9b1Od/39cd1e2703tnLdbZZZOkdzzz134DmDfjx58N7s/vvvD1NOOWUYOnRoTEa/66677kqb1P0tPh+QkDxpaxjPsDxP0X/jGGeZZZbwzjvvxHtJo/dLcWP/SUACEpBAjxCgb4/xDMbzAs8IXLv5YzrNV935ZGNewnR9kq+6ZS9Px4OxJgEJSEACEpCABFohsNdee0WxEy+eioOSvMDZZJNNYra87CJEGB2xovGwfvLJJ3dZ3K6AiwFrXryUGYObyTNOvp5BAwaryozwNZQxDcCRpt7L1vxFPwNwCHqSsQ0DCnROi8YLdYQd6SUFg3BJFMXgKS9xi8bLhwMOOCAuLoawavZ4inmn+TQgyDwDo7/5zW/SqtpvHoorfaHLSgYq2Z6Od5mNGjUqnHHGGV3EcYTx4sVNvXBwed2ACWy6szR4zwMCggq4FY0688QTTwQGtIrGuTn77LPjwE6+jsErQhzxYrDMchb5evaRBp7y5WmaMjL4mh5oWJ6OAVEL4hZEhkVjsOvUU0+NL8B42fyzn/2smCTOMxDGAEiZVd2ulXNbtr98GW2W9lOvvnAeGKTOBzNpS1x7EKuUGYNw1Nm0TSMBF+2fdoOxr0suuaR00C/fz7HHHhtFKIhXGIzCGyGiUV4IJ3EqorCtttoqbkaaXOiy8MILh1122SWuo+0zUMUyDFFNXgeSdy/WJY9ITKe6wYAYx8sAWdEYwGfQIgn80vpm6mLVupHyLv6mciIEgFOy7urSM888E8P1Ub+7MwY7t95665js6aefjteX7rZhffG6Tduu134QS6WBnGLeqW2VDbqltM1em/N7KQJPhBXFNkJ9PeWUU8J2Y15Ul51/hD3UVdIlI/whg9uIMbhHFy15XWD5QQcdFMO3ISThhXsSbXOPp3xcy1M40WI++Xx+32SfvEhPx8L1l+ss1ul6mepeK22klet8Ov8MDnBvRhCcrFEouySqS2nTL+Ljk046qUs95bwh0CkzhKu5kJo0rVxfU7vg+oH4tF4fiUH4m266qawoYy2rcs2kHpcNbqbzmK4hZd4L2SF9trz9Ntvm8nqKqJD6jpWVizaEBzCM80T7YFnREIUgzEMEVzTu+VzfqSvJ6Dtxn2egGA9cRUvetlj+hz/8IQqrEIUziIVIlr4U9Y6wvUUexbyK87l4rLiOefrCnMfcOnUd78k2QN+SOj3fFx8l5OVnGrEQx5V/sJHOOddO7ulcs5LxLJEExs3079P29X7zOsX97/TTT49Jm+3v1Ms/LU/tqWq/Mm3Hb3fnu6x/n29fnEbci8iwrO2QFq9y3GOS0JHzwMA5xsccDPoWLXleKoZ2o89Ou6C/VWY8t9EHTyFL03WQtM1ed8vy766Os01P1FXyTM/lnexLlB1jWoYQh/sh+65nqT+RryeEc/qIKV/ONG0RYXRZv6WYlvlW7zll7bCnzl29e07qS+QfA/RUn7CMHcuq9Pt64n1QT+TJ8bQi4MqZ8yEFouxGdYF7d9m7ntSXS/cVylMUcDV7L8nraSf7IJStzPL99eT9iX038yxQVta0rOrza0/09/O6My6uu+3cG1u57rbCjPPCx5Lcj9IzYDpX/PJuGFEXVlXAlfoz6fmAbfO6yjs5+uB8RFw03mPTB+cZUZOABCQggXFHIL3rVsDVDXMFXN0AcrUEJCABCUhAAnUJpK/7ScALsfyL2PyFRcqAl+J4taL/wdeFydKgd5rPRToMCjI42J0lTyB5Ol6+v/XWW1Gwkw9qF18GIK5AZIHxYpjBDF4AIMTAO1JazgB58qpU72Vr/qI/F3Dx5TEhb5IxCIH4CDFRXjZeSPOH2I2Xzkn0hjee4ouF733ve/HLNfLMObVyPKlcxV/OEwP5WL1BwIO/8DpEmqOOOip+IY13GDy4JGNgjEFFXvTANR1XepmZ0vWkgCvtI51jvvYvG8DhOKk7iILSC9j85Tn54CWIkGHpOBjwQ5xD3SbP9EKqeHy8kNp9993jesrBIBeD/LxcpC6k/RUHSNOLqXQM/MKUssIzbZfOEUIlhEEMVFFWjGOinLS35HEsrsj+Vdmu1XOb7WasSb6uh2ca5KENIjrhHDCwx3qMawjtKtkPf/jD6EGBec4RA3W0E754TaIuxAcIhmBTT8CFoI4XxRjn5be//W0lrzKIsShbElsxAMm+EWMieMB4gUi9pk4gsGIwJFk+CE9751iTACkXgeUvIYveVIp1g/LjaY1zjZcXtsWY33fffdOu48vRZupilbpRy7xkIpUzf7lKe2GACjaUm+s1IjPqLV6FUvtC3MZ1uzvrlICL/ddrP7d94YGrUdsqG3Sj7K1cm8vupdRx7iF48OM6lhscubdQLyljsmuuuSbceuutabbh75Ah/z9cZ15vdtttt/jlcrpnM4DOYEE+iNMo4/y+maejzHiaufDCC3ukXqa6l/ZZtY20ep1P5z/tL/0iQmRQsp4hqqM/kETGXNM4z1yvYZ73L1IeXNe5h3BvTdd61nFfTmKHVq+vZfvjBRv5UkYG55Nx/ab9dmdVrpn1BjfTeUzXEAZ/8HRBO+AaAl+uffA488wzY1FaaXN5PW1GwJWOnes89zA8UpUNFvGyknsZ19TUV8DLEALsKoZIjHtN2jYJLNN9LPU76Itwb+J6UfToWG8/eIvD+wTGvYZyce/EG1heXsRMyctZJ6/jPdUGOJ5f/vKXXfoF9EnpO+XXUa53pOOYsbwuct1IzJm++OKLo9fUZvv3MeMG//L7fX5tbba/02AXcVVqT3m6Rv3KlK4n+oAIptL9CiEjfTnODX3m1P+jbTPIirUzSJ1fn6nj9N25psw3RtiX+ktcy1JY4LLrYJXrbuJV/O2ujpO+J+oq16Ik4Epl6um+RC4GZZCcZ2jOJ8886XxTlvTsyHQK/cp0Oj9cT2mnXPuSJc+3ab7eb6v3nLJ22FPnrt49J9XV/Bl0XPcJq/T7euJ9UE/kSR3plICrUV3gutWoL5ffV3IBVyv3kryepjbQk32QfH89eX9q9X1JYpD/Vnl+7an+/rhur63eG1u57rbKbIUVVgibb7557RRx/+W5gvOUP1eQoPjOtrZRYSL1Z9LzAavzupqS03fDkyPPzjzHpD5d6jendP5KQAISkEDPE1DAVZExL1s0CUhAAhKQgAQk0CqB9GIUD0a8TE1WfGGBNxQ84TBohfESmTQYQo3kwYD5Tgi4hg0bFsPZkB+We4niRShCBh7iGeDCixUP8LzwwstF7qUmF1jkX3HXe9mav+jPBVxHHHFEbSCv+DIC9+G44qcMSYBDmfO8krCL5clSnhwHAhUGnlo9npRn2W8+kAAfXnIkoy+ZBjlyr1m5t6CicIAXJpQ3iY7wLpEERT0t4OLcs4800FwU/lGO3NsFg+AMwmJ5GKP83BCu6vzzz09IogiHtElYwUslhIsY3lcYJMLwqsDLz2T5l+p5PWB9ejHFNPWU/aUwRgyCMIDLiyosHwghRAx/2HnnnVc3FGdMkP1rtF2r5zbLfqxJvIbgPQRL4dnyRHhTYcAHY0AjeV7jusFxM8hDm07XF9LlL8iToIqXg7DC8EKGKIKQpoRtxGhLcEr1MS5s8A+xFueNQW62pW6zD64htMdkaRCG+VzowDRtKAnTaBMIubgW5CI+PBHwZSyWL2c+rxvUG8LFpJA0lA0BZhoA5dwxCIq1Whcb1Y2YcZ1/qZz5y9U8L8KPcd1ORr2GDyyqChA6JeBK9SgvX7H9NFqXznc+6Nbqtbl4L0VAlsJzwoh9JaEb7YDzzz0VI8zqDjvsEKe5XpxzzjlxutG/3PsjdTr3escxIIxE+MBgLC/vaZe0x3RNbZR3ft8kHQP1eN2iTifriXqZ6h77aKaNtHqdT+ef/cEQ4Rz3cPbdncE4hbQsenHMy0O+CN4QviVLHiuYzwU2rV5fi/tDKJ5EO+yD+zgCYixdT+NMg39Vrpn5tTsf3EznMb+GsKt0H0BUka7vLG+1zeX1tFkBF22PAeJ0Dcn7EJSJ0KPc4zCu/Vyfab95/ymurPOP9oZ4K/Wf8usBAg2EVqk9ki9CWHjlbaxO1nFx6oNxLaHvRztPhleRJEDM+8P5tbAT1/GeaAN4eyDkOAYfvKekY6MPQV1Oou+8v53XRbblWYd2x/lKlvrizFft36dty37zQcd8gDzV86r9nbK882WpPbGsmX5lp/uA9JuSN+FcpEW5qMO06fTRD173uK+2Okid39+4HlPHU1+NDwgQkqUPCZIwv3gdrHrdpfz1rFEd76m6Oq77EghYuBdiCAOpN9y3kuXP5Xmby+9jRZFWfm3O20bKs+y31XtOvXbYE+cuP678npP6EnlfclyfR463Sr+vJ94H9USenRJwUdca1YVG6/L7St7HaeVektdTytTTfZB8f3kb7PT9qdVnARjUs7yvUnymy6/zzbzXSW2UfZb198d1e2313tjKdbdVZnk9z/uwMCR8NB4+kxX7VGl58Tf1Z/Lng7yukp5nBCIqpOdVPOLuvPPOMSv6QPkHdsX8nZeABCQggc4TUMBVkakCroqgTCYBCUhAAhKQQCkBhEdjQk3HlxY8+DKwgOUvLOiYpQHJlAkvyBkowoqDbu0KuOoNVOchoZLHsKFDh9bCLfI1/YMPPpiKWPtNLxp4MYNIhBep9V625i8zkoArD81WFAilneDtJYVOu+CCC+Kg7IABAwJfvmK8/GZgIRmDeClEUC7oaPV4Ur5lv/ng76OPPhoFLild/gKekE2Ebho0aFAMFUmavGxpG36pM9QdLA+jmQYP6w1m5nWDl2ZJyBMzqvMvvdRhdfGFXS5wYDCHwaF8gCEPIYlHD0Lx5V7Jyuo2+8kHiRhg55xivCDj3FHnk/AtrvjiXwoJRRkI8ZIsP4ayesoLqBSaCXEXLx+xRi8rU95lv/W2a+fclu0nLdtoo42iO32Om3aaD46ShvVDxngFwvBSgmAKSyFyOHe8BE+D5qxDeJdezBGi5YEHHojiqjTAj+CAtpyEY+wbgQuh3DpteViCFG4McRViOyz/8pNrJS/+uZYyUI/lQkNCx9xwww1xOf/yupGuHbWVYyZ23XXXGDaAZXn9b7Uu1qsb+T7LplM585eruUA2H8BL2xP6DA87DLRX8U7TmwVcrV6b83tp8V4Jp1wgUsYQQReD4LlHkcS3+Mv1EM+JiOawS8aEEaXddMry+yYvyxFN5G2W/fREvUx1j/yrtpF2rvP5gE5xcIIyNLJGg355/yK/r6T88vtOLv5q9fqa76+s75J/RZ/vL5Wn1d96g5vpPObXEPaRBg6L7aPVNpfX03wwvaxc+QAR9xCEKEkYRNnya3cuemIdxn0L72nFe/7/re36n3sa6dkn9v/YOw9oS4qijzcGkh5ABYEl7BIkiSDg6hJEgqCL5CxBEBQRFcSEYNgPRfw+xbAoQRBRAQExICw5LCsiSbLCquCCREEJHkVR0W9/DXWt19s909Mz9777dqvOeW/mTujp+Xd1d3X1f6qxV7EHuhQmzCGHsWQ2dSUUWTZK26Rdt+P9qAP0tfS54Eybia5ogfyGrY9ALIXYg+gyD5dA5nypfc+9KdE6pSfIm9o7qfTluNQnfufalf2wAbUdHqsjLBvMOAThQyD6stJJah3pi4lciarsE5/9Ty+3Kh+f6HawSbsraca2VTreL10dpC3BOzOGlSXC+YhCiKuCB9G0ZMJcEz/1WF1/kCL30a5DXmccwHJ2XYiu50KoSdXDfpRdqs8RWyJF4Ar7PLDo0iZsgm0//EH9SHNYCVylfYnW037bIOiDfl4/+6fSsUCVzqbGr/209wfd7pb2jU3b3VLMtA0Rs6koP52Xrghc1A3sOiELiJ5oQjqkQRNDwBAwBAyBwSEgbTK+f/wP+CDxW/LHvvzOzdF8syeZ/vupSO5dY+A6I3CNgUKyLBoChoAhYAgYAkOMgP7C9vzzz3dXXHGFz612WAixJ3wNIeuE0VU0SUcvDRjer3/rSTImoHAKhCLLPXEcp+/VV1/tSTTYQ1VLG7373e92hPJHJFpUytmqHf1C4CICChMUiBDH/A/1T4cuv+mmm9xpp53mz8rXpzgemOyWL8X1hJmekJaIPk3fR2Vljl3IduQDA1o7kblQO4qZuMT41l/PacKITpjILThNEO2AFp3oB4ELDDUpimdT9kKkijmSIPfIxNHZZ5/trr32WqdJIiyTx8REKEQ8AjOEsPDglBImIvhCnEkq0gbnMK8y0aZJPTo97ZTUzi59PFUWOh3ZT93Xpmwl7dwtGBJNBAIPZSARRzSBSzve0E0mtKnXEoEqfJaOwAX5TyJucJ3U7fCeLn7rpUCEsKid9RAfaScRTRqUCUacjpLXkLgouhHqjORbL4mldUPO622OLqZ0Q6cT25d8avKFJoBwD5HLfvrTn3oCa0jsiaUZHtN1k6iPJ5xwQnhJ9Hes3ebCqnetOhebdCttm3Vfev3117szzzxzxDsccsghboUVVvDHaD+JjqWFiG5EA4RUwsRkSiA3UJ+kntXpSiqdquO632xC+Gmrl6J7TeqI1qWm7byUP89jEoptrlRNDms9JTLaXXfdNSJZlmCVZVIhG8sygiMuev5HTvuqnwdxR0cnJRnaJCG61D0vlofUsdhkOtdKOeo2hOMpAldpndN62oTAFSNJshwhbTACkZj0tEBgkGXB6iaPhNzL/RCsILMMQtBJbJRNZpOoJzwfQVQTuLpux7uuAziBIbIiLMUmRK0QOyH36HZC66Imx8u9bex7SSPcpibIm9o7Ybrhb6lPTezKftiAmsxOHlkCFUIPhB9x7Id5L52kFhJcaozEeIcJZoSPgSDz6Xawbbsr75HS8X7q6qBsCXnH2Jb3Y2lvovlttdVWfgzGdZrApT9IoZzo57CPIYv0S3Q9LyVwtSm7VJ8jtoQeew9DOcbKoR/+oH6kOawErtK+RPcX/bZBKHf9PE3g6rp/iulYzlggdp8cS43b+mnvD7q+lvaNTdvdUsy0/zLld9EfjuaORcWe0eMDrav4heSjOdEHtozR6I9knz7HxBAwBAwBQ2AwCMg4zwhcNXgbgasGIDttCBgChoAhYAgYArUIyDI6mqiiHRYs90QUilDkK+OuCVyQdGITpvorb3EWSzQB8ha7h+MQakSEaJFytmpHvxC4BB/SSD2Dc/IcvRzl29/+dscSi4hE7mFfiF2QHHiOSOn7yP2prZ5kZCKMr+RxpElUMD2ZR9QwvrxGmEwW0lmYthALdNjyfhK4tANc8qKJZLEJ6BiBa8cdd3Qbb7yxJJEsUylP3l8m1bmJrxYpVyZCmUSX63oJzt7RE4gcF8dU7B04r8lA2tmVclZyT5Wk7mtTtlXP4xwkE6IfrLXWWn5pNiZDYqIJXBMnTvRLroYYolPoKBOAN9xwQy8ZTeDqHXx+JxW5L7yu5Df5o96wFX3QxBtdT3RUAvL+ve99b0SkMYm8J/mo0w1NINC6wf0lupjSDclPaiv51M5VyEJHHHGEjzgW3gexE5ICpAcmJXJEO5SHjcBV2jbrvlSTSQSPgw8+2K244or+Z6zvk3a2jsCl2zXqDUu9dS2639T9WficrvVSdC/VfsbqiMaD/KX6bml7pF5zrUy6EhkwrK+cr5LUxD73aPtCJpl1WpqQHOp/Sfuqn6fbKHmmJl6Ez5NrSraxyXTSkXLUbQjHUwSu0jqn9VTXuVi+9ASRkHPJk8hmm23ml/Dmd7i0N8e0bVVF4NLLF2p7izS6FtoTljknMpgsJxc+Q+eh63a86zqgseM96uoy18hSfbrMIStC6NHSxr7X6eh9rVN6grypvaPTjO1LfUq1izG7sl824F577eU/YAjzSV1nTASxWyKvck3JJHUqylr4zPC3bgebtrthWvI7peP91NVB2RLyjrIlMiRLldOvC0FczslWxuT8pt2h36QeaKHeshz9zTff7PUhNbbU9+Tu63ouZZyqh/0ou1SfI7aErqOjVY45WEp72JU/iGd2neawErjkPXnnVB/FObE5xVek9bSfNgjPRvTz+tk/8aySsQD3pSQ1fu2nvT/o+lrSN4JX03a3FDMii/NBACJtrf+h/q2xxhrugAMO8EdCv4W6bMSu2DN6fKB1FfshFslbR+olb7KSxIjE7YchYAgYAoZAXxAwAlcmrEbgygTKLjMEDAFDwBAwBAyBJAIQjCCkIAygcbBqhwUEBE2ikIQGTeAikg+OeARnMU4BvsZuIrKkRsrZqh39QuDSXyXmPEs7GTThRCIHjB8/vhdJSkcxwalX+j51+dIRwmQJER3ZR09yEolr8cUXn4OEFD5DnC366/86Atc73/lOt/baa/ukcGw3WUIxFtVLE7j4ypuv67XECFw6Ipu+NrWvHe+bbrqpn8wVB6zcg7MWIiMT4kxw8FtHCxOsYu9AGrGJNo6nnJWcq5LUfW3Ktup5OPMgNEEyCAX8EMhuiCZw8ftVr3qVw5FI9JkQV86DGZFYSEfXJ87xpSX3yH3HHXecnyzkXNeio6dAhiBPOBf1ck3yTE1IIOqHTOzHnPN1uhEjp/CcUl1M6YbkPbWVfGrnKtcKsQTylZRxmIaO7hie07+HlcCFfpW2zXV9qSZwiZ5oTHIJXDqdfjmxU/2mzm8/9FJ0L9V+xupIm3Y+Numq37FqPzU5zD3avoiVdYrAVdq+1j1vmAlcbepcSk/rJvljJGBN4IIUGS4jlkvg0umkbOoqvco9l9J7bBIiI0kkSE3gIu0u2/Gu64C2X3NxkInFWJnrNNrY9zodva8nHfUEOdc0sXd0mrH9unYxZlf2ywYkfxtttJHbcsstHVF3YqKxyJmkFqKGLMsIMZF+DtHLVseepY/VtYOpdlenEe6ndLyfujooW0LeFfIn9QgSRih8ZEF/zLKwiCZw8RtbfbfddvP6HiN9MW6cOnWq/1CD69tKrJ6n6mE/yi7V58RsiUGXYxNsu/YH8eyu05R2Ifx4sOo9NeYSwZ3rU7pQdy6mb6V9idbTftoggo9+nm6TOd9l/1Q6FpB8xrap8WvK7omlwTHt14nVUX2f1p2Y7abHXjG7PncMJ88s6Rvl3ibtbilmRKSXdj/2vuSFMQvjUKQrAlesbpC+EbhAwcQQMAQMgdFBwAhcmbgbgSsTKLvMEDAEDAFDwBAwBCoRkEgHt912m4NwUOewILF+EbggPzz44INz5JfJgZ133tkf/9GPfuS/4CUqDpF+iJTBcmV1wvJUkD5Szlbt6BcCl/6qEschjusqgZSEY1FElk9jAo2oACwXKUsyMgEDYU6k9H3k/tSWyVCcSDjSxXGl88XX0rLkWSqyUJi26IwmsNQRuPTX/xBgciLzVE1SlRC4iEyBfiNEBYiRE/W7MknB18hELCPPQhZ64IEH/Jfk1Bl5D3AcN27c0BK42pStxiTcJ5Ib+CDoA8uI8pU99Zj6tu222zomsJGQwOUPzv7HBDLRu9ZZZx0fkYiyFREnsyZwUQ8h1TA5IBHVIBcx4dSPEPp66SGiSsn76CVTJb8a52uuucYRwQCRCIByHdsq/eZ8jJzSRhdTDnCeVSWSz5DApe9ZcsklHVFG+PqWZXakrnANBDgZ5Ot79P6wErjIY2nbXNeXduX8F6d6GNVR49t2P9VvSrr90kvRvSYErtJ2nnepm9CR941tqyYEtX0Rm/hIEQlK29e65w0zgQtsS+tcSk9jk656MjM2QaSJV20IXHqZcMoTAlXXoskj2JtEVWNZYqKTUncQsdtCApfOS9t2vOs6oD96wKYIl6HVeWcf20CWYY6Vub6+rX2v05J9rVNiu8g52ebYO3JtalvXLsYIXNo2iUXlk2eJnmj7Xs7VbbHd6MsZ51B24CHCkuUsaVs3Sa3bJiFwQSQ68sgjfVLoAWPFHKlrB1PtblXaKR3vp64OypaQ99YT/NgVjHVoTyDPMY7ExmNMi4QELkmDLaQQlrXkIywhkHK8CQGH66skVs9T9bAfZZfqc2K2xKDLsQq32Dmp+134gyT9LtMcVgJXaV+i9bSfNoiUhX5ev/qnNmMByWdsmxq/9tPeH3R9LekbY1jVtbulmOmPFqZMmeKIth2KHssbgStEx34bAoaAITD3ICC+XfwMzDPh+2Vujj/25XfuG883adKk/+RePJauMwLXWCoty6shYAgYAoaAITC8CEhkJImmVOew4E36ReDSX0dqxN7znve41Vdf3R/iy12cyDKxyWQVX3vFiBtMIghhikhTGJgpZ6t29AuBSzuxifBEpKdQiFjFl+fIjTfeOCIK0BZbbOGIBIXwfBxQfNmM0wPnh5bS99FppPb1u5144onuwAMP9JfK5Ijct/vuu7vZtrP/ecYZZ/j3kXOy1ZMUEJkghyFC4BKSmFwvW/2F6mgRuPQkpyxxJ/mTLYONXXbZxZOKwIcJp0022cRtv/32/pLUclOyNCb6OIwRuNqUrWATbnHUorcIEzJEeAjroSbuCYGLpQaFeHXhhRd6kpxOmwkfJr0lXZbq0wSuK664whHZCTn66KMdy+sgqTL1J1v8W2GFFXyUMZKgnZSoArGoX3riVF8rUUF0NuomYWMErja6mHKA6zzF9iWfmsA1efJkXyZMps6YMWPEbeADSVSiN8SWHxtxw+wf2umbqmPhPfzWbZu02xyveteqc7FJt9K2ua4v7YrABcGZCVWIGdOnT+f1O5dUvykP6pdeiu41IXCVtvO8S6z85R3rtqnJYe7TeppL4CptX3Oep0kSTepbHQaxyXTukXLUbQjHdcRCbROV1rmUnsbypScz+zl5utpqq3k7FNsAEks/RPezsUkzTVTRBK6u2/Gu6wB9CQQAhA8kqJ8xYZk3yGfoF2RpJFbm+t629r1OS/a1TskEeYm9I+mltlKfUu2itkNEH/phA2Krrbrqqv5Dmphuy/iS95ClzhkvbbXVVv7Vzj33XHfVVVf5ffmno/boMQqkfexzveSt3MMWsjy2O3LxxRf7v5J21ydQ8S+l4/3U1UHZEvLaejxDnSOCnxYi7Gy33Xb+kBC4ICbysQPCRxyUtxYit2AXyhLrfPTCB1htJVbPY/WQ5/Sj7FJ9TsyWGHQ5NsVW6msX/iB5dpdpDiuBq7Qv0XraTxtEykI/r1/9U5uxgOQztk2N2/pp7w+6vpb0jSXtLm3WHnvs4WG8TuJTAABAAElEQVRO+U1ivjD9MYJ8TBuWFelSJojYHuE14W+xZ/T4QOtqrG6QhkXgCpG034aAIWAIDA4BI3BlYm0Erkyg7DJDwBAwBAwBQ8AQqESASQ++HkT4uh2CEV9nIbGQ4RzvF4ELQ5BJPE0CYaIRkgZOX74ExunLRJiesLrsssvcBRdcQNZ6gjOdfOIEID2+HGObcrZqR78QAbQjSpOVeg+ZvaO/SAsJHTwbYpNMOvAuiEwu+B/P/yt9H51Gap+v8XB0IBCsZLmz73znO46lHEWIgLTPPvv4n0SWgmgVinaYyLKUXCNRvdgPl/HaYIMNeo59zo8WgUuTgLQukScRvfQiSyYReYOJCiYskOuvv36OKBAa334QuIiMxxfROaKdnPq+NmWbeu4yyyzjqCtITF8gIBAdCAcjIgQultLEqY9AkItN/MkEkkzW6bK7/PLL3bRp0/z9IfZEbrn//vv9uS7/yVfkkqZMcMhv2WpihBxLRRsQp2VqEjZG4GqjiyndkHymtpJP7VwlGgYELdpU6jt6r0VP8MWij+lr2e8ngUvXA55VhUNs0q20bR60859365ek+k15Xr/0UnSvSR3RbUWTdp53iZW/vGPdNjU5zH3avsglcJW2rznP0+3UMBK4SutcSk/rJvljE0RdReCq05suzhMNBxInwiT3vffe6/flH8QWiQapCVxdt+Nd1wHyL7YA+7EovXrJHv1usTInDZG29r2ko7d60lEmyEvsHZ1mbL+uXYwRuPphAzJeIF0kpneQ6xm/Ib/97W8dYyQIjfIRyR133OFOOeUUf55/jPNog7EdEU3gwpaU5ZtOPvlkR53VEvtIpKTd1WnG9qt0vF+6OmhbQuxd7LrwAynGs3xUscQSS3h4hMBF2XEf5x999NFeuWsMdbse+6hBX5u7H6vnsXpIev0ou1SfE7MlBl2OuRjKdV36g/qR5rASuEr7Eq2ng7BB9PP61T+1GQuIzsS2qXFbP+39QdfXkr6xpN3FzysfTDQZI2myXOxDVPyu+DzZIkbgimmyHTMEDAFDYO5AwAhcmeVoBK5MoOwyQ8AQMAQMAUPAEKhFQBzjLOkHoWK0CFxkFIc9hA62Sy21lPvABz7gZEk1HXmH5RhYmgtnMU5mHAU4khGc/NyHsxYhchYRtJCUs1U7+oXAxfV6UoAlJIhMxfNwUEBCWXPNNbnM/fWvf/VLuPkf6h/RmIhaJcK9TLaFXx6Xvo+kW7fVEwtcy1IzkM9CkQk9jjPhQsQuyCo4iYhCJZGTwkhb8pUt9xGV56STTvL38TX91ltv7cuJc8hoEbh4to7mxiQDhB9IQgiEGQhcolM44PnyXE/AUW7cwzI9TE7h4MOxyD0I5fuhD32oR2gpmWgjHe04xNH6rW99q5dPzqek6r7Ssk09i3cGC3l38nj77bf7usGSiG9/+9s9IVTuZ6KOCTuch+gj96GH1E10DYHsxVeoEtWOiWic9tpJqwlc3KPrbsmyP6RRJ3qCnGurlvFBv/VY7c477/T1IXxGnW7ECFxtdLFKN8K86d+ST03g2n///XvRDWlfmVAV4i2TQLS/MglL/8JSpFWiCVyQddD5KoFQy3KWuux1u131rlXnYpNupW2zfk6MDN1VBC4c8YsssojvU4SMXYVdyblUvylp9UsvRfeaELjIU0k7z32x8ud4jmiyAvYAeYe8Sb3QeppL4CptX8lr3fOGhcAl5Uu/CTmHfhW8SutcSk/rJvn7OXlKW8nSssjXvva1OchV/kTLf5pIgx3PsuLoIDYwE6r0xyJ64q3rdrzrOkCe9QcA2J0QwYWkzdJs73rXu3p2ho72GCtzwUC2be17SUe2sQnyEntH0kttpd6k2sUYgYu0urYBdb9NH0+ULFlWiXEbZQDJHuEjGz62YewkUdWo9/SNLEe98sore12FuCqiCVzUoQMOOMCfYkzyzW9+04/taCf33HNPTwLnpB6L1bWDOjJdLpG1Ssf7pauDsiUEd60nfMjC+BrMiUYL1kSeFiHa1ne/+13/U9u+RMllzC5CNGzGiYwlGUfxMVYXEqvnsXrIs/pRdqk+J2ZL9KscdXkRFRv7uFS68gfp53eVphC40MW6j5rQMT7A0pjrKO9VulB1LqZvvGtJX6L1tJ82iJSFfp4QuLrun9qMBSSfsa0ux9An0i97Xz+zn2M4ed/SvrGk3W2LGXnGP3PCCSf4D0Pp7/Ep4qsRMQKXIGFbQ8AQMATmPgSMwJVZpnpSIPMWu8wQMAQMAUPAEDAEDIEoAiw/ss022/hzONll2b+Yw4KL+hWBS2cOx74QQzgOMQMnJY47kR122MExUSHCPRBCcFKJQM6B6MVXZkjK2aod/ZoIwBdxODokLzyDtHB6iXCMCYmY01Q7s7j+vvvu85Mccq/elryPvr9qX4c+5zqcn0SmCYXJFrCQ9+U8mON0l2O8L0Q2CG0ienJFjukt98j9o0ngYuKa52sdoTwhDkn+yLeefMCpRhQBiVzGefRMIkvxW0c2Ay+WzGRpudKJNpxhOIS1pELW62uq7istW51+uK9JKJwLdQXSz0ILLeRvQwcgM0H2CfURZzvnqVdSDvzma86HH364ksBFmYIV5YSwjBzR4bqUbbfd1hGRRYSlHy+99FL5OWL7jne8w6277rq9Y6nlSOt0I0bgaqOLVbrRy2xkR/KpCVyQs5igol1AKCvKkPzpeiFRNyLJjjikJ4JHnEj8kIhvqXa76l2rzsUm3chCSds8KOe/LENHGejlWxPQFR1O9ZuSWL/0UnQvRVSI1RHyVNLOc1+q/DmXI5JfuVb0X+tpLoGLNErb17rnDQuBS+eT99VtTEmdS+lpbNJVT2b2c/IUQvfyyy/P6/kIREJW9gc6+gfphckz6TtJlr5Y+kTaBm0bY/Ogq7TZXbbjPLfrOkCaIYGad0Pk/dj/3e9+54lr7COxMn/uzH//t7Xv/5vSc3tap2SCnDNN7Z0w3fC3YJxqF1MErn7YgELOII/oGbYw5aJtbIisjMOk3Ii+JBGcwncjDdFjTeDiOh0BmN9ci8j1/IbkwTgL0e1Lk3bX31zxT/CXS6Sd53c/dHVQtoS8j/YJcCzEORwzQd5DD8LxLuXNtYydpIxILycqK9flSKyep+oh6XVddqk+J2ZL9KscNXkIX4QQXHPwC6/RZd/GH6TT7SpNIXDptFP76Cz2sMZcE7i4r0oXUudi+kZaJX2J1tN+2iDkD9HP61f/1GYs8Fwu4/+rxm39sve17sT8odo+j/UvkCkZD2tScfzt/nu0pG8saXdLMQv1nJxrf5jeNwLXf8vV9gwBQ8AQmNsQMAJXZokagSsTKLvMEDAEDAFDwBAwBGoRwOGCowEHK45o+WI6XGJPEkoRuHbffXc3adIkf9lFF13kLrnkErklud1rr738l9M4elkWjWhNelKGG3HiT506tef814mtv/76juVhhEigzxElB7KIjv5CJIL99tvPX6YJMSzpgWMC0QQufrNEy3vf+17/9S6/tUAsw7Ezc+ZMfXjEvkywc1CiEI24QP1o+j7q1spdvTQCF+Lwfeyxx6L38L5MfAjxRl/EhN/xxx8fjSJB2e+2224jHPXcy1d6OCeFGJhL4II0xZewsahK2pkXLsPCM/VSiKeffvoIshlp8n7LLrssl44Q3o8oAfxpSWEievvLX/7SL0UqenjVVVe5c8891xO/Uu9A+qmJNs7pLyT5Tf0g+lSdVN2Xeg/SrCrb1DOZmGGCGv3SgvMcTIjKddRRR/Wi6EHGIvoWdZw6J22Nvpd9JgEhPhHRC2G5PgicSBiBi2ObbLKJjxDHPs8mEpFEVuNYWyEa4Mc//vFeMkxGysC1d/D5HdoRWSKIvNCe4NQMpUq/uTZFTkmVYZ0ukmaVbnA+JpJPTa7gOiIqEPlFE7b0/bSJRMmIvbu+jn29vFN4LvZbCFxV7XbVu6bOSSSD8F3JQ9O2mSXL6JuQWF+a6/xHz9C3lEByhPgoE1ap69ocT/WbOs1+6KXoXqwP4NmpOsK5kna+qvxJs0622GILN3ny5J49MmvWLG+7aD2NTfSkIsGUtq91z+sXgUtHgqK9lDZYyjGsV7T/RFAScjR9kI7M0rTOpfQ0li/aLexeJEZo10sohjYE9+jls2NlyjWIjsIaLrP93BXd/Ke9IXqu2CCSKphjB7PEorRHnJNILV2246TbdR0gTYSInkR0DYV2DzuBNlaiQHJNrMzDe/lNu9XGvtdpap3SE+RN7R2dZmxf6lOqXayyK1PtNM8psQGJ8kZfho0WE5a1JMqqHm/QX1HPdSQn7oXsw6Tvjjvu6Em4IYGLa9BxJtZDYZKcsRhjDZG6djDV7sr9qW1Kx+X6rnV1ULaE5J9tSDqUc5QjRJo99tjDvfrVr/aHsT1ZahFhDA9hRxO2/InZ/6ifRE6VJdDleJttrJ6n6iHP6brsUn1OzJboVzkyxpKIt/hniGRZKl35g/Tzu0qzhMClMQ8JXFW6kDoX0zd516Z9idbTftogkj/9vH72T6k+JmeMKnmNbVPjNq7th72vdaefYzj9rqV9Y0m7W4IZeV1xxRW9zaRJ2hzHb0NUdokEnUvgEntGjw+0rsbqBs/ThG76H/TLxBAwBAwBQ2AwCIgfnI+JsPOw+/FB8Me+/M7NzXyzJ5Ke+ywn944xcp0RuMZIQVk2DQFDwBAwBAwBQ6AxAjgH+KoVJyQDdxzzVYLDY9VVV/UELBwKTO7feOON7vHHH6+6rdE5jFCWjyDSFMv7kDbRhPjqv2vpx/uQpixbR975OrhKML4nTJjgMeVLeaKLscRILMqYTgecuI+lbTDoIXFQHsMokHIoT4hc5BWiGcuBpARMiKyEHhBRgGshJIlgn6+00kr+K1dIZTJ5LedLtkyOEV2DQRJRBfQEZVV6VfeVlm3V85iIX3311T2RhMlUiKBMrCLUSUhNEALvueeeEfrABN7EiRP95DL5YllLdOyWW27p3V/13Hn1XBtdrNKNpngyYIcAQD1iAol6RLvNJCpLeY22VL1r1blUvvvRNqeeNRaPD4teauyatvP63pJ9yFH0f5Ah6FO66AdK29eS/I/GPfQd6A5tvyzBJvmwOidI1G8htmOjYNM89NBDvh+F5CMCiQt7gok22mixJ7pux/tRB3gHbCxII4wRIAbTx1x77bW+rsk7lmwHZd8Pi71DXcNOxy5rYt9XYUuZQMRmCSXKBjsAe68q4hw2A+XJPZC1GGPkTMKir4z5sLfRb4iy3D9IqdPxfunqIN+RvhP7Dr2F8MG4hn5NBP2hDBnjUQYi0g6hE5BzWYqe9ujWW2+tHc9LGv3czgtl10/85qa0q3Sh6lwKg0H1JanntzneZf/UZixQ9Q5147ZB2/tVeW1zrqRvLG13SzCjD2bcTx9Mn43fRyJftnlvu9cQMAQMAUNgbCBgBK7McjICVyZQdpkhYAgYAoaAIWAIGAKGgP8qmmglSO5XcQabIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAobAvImAEbgyy90IXJlA2WWGgCFgCBgChoAhYAjMowjw9SiRFgiTPmXKFB8dKVyiaB6Fxl7bEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUOgAgEjcFWAo08ZgUujYfuGgCFgCBgChoAhYAgYAiECe++9t19Oh5D+IpdffrmbNm2a/LStIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAITAHAkbgmgOS+AEjcMVxsaOGgCFgCBgChoAhYAgYAs8hAIFrvfXW68HxwAMPuGOOOab323YMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMgRgCRuCKoRI5ZgSuCCh2yBAwBAwBQ8AQMAQMAUOgh8Byyy3nJk6c6Fg28e6773YzZ87snbMdQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQyCFgBG4UsgEx43AFQBiPw0BQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDoDUCRuDKhNAIXJlA2WWGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIZCNgBG4MqEyAlcmUHaZIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAhkI2AErkyojMCVCZRdZggYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgC2QgYgSsTKiNwZQJllxkChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGQDYCRuDKhMoIXJlA2WWGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIZCNgBG4MqEyAlcmUHaZIWAIGAKGgCFgCAw1AltssYVbYokl3GOPPeYuu+yyoc6rZW50EXjta1/r1ltvPfeKV7zCPfzww+60004b3Qy1eDrv8MY3vtGncN1117lHHnmkRWrOvf71r3fjxo1zf/7zn92VV17ZKq1hvHn8+PFuww039Fk788wz3X/+8x8333zzube97W3uRS96kbvnnnvcHXfcMYxZH6o8DQNmg9bVQT9vtAp8scUWc5tssknR42lPr7/++qJ7w5uop/TpTzzxhJsxY0Z42n4PEQLYXyuttJJbdNFFfflfddVVQ5S7sZUV/HObbbaZ75duuOEG99BDD/kX6Lo+TJ482S2wwALud7/7nbv99tvHFkiW23kWAdoY7DVk+vTp3obvCozVVlvN8Yecd9557t///nfrpBdccEG3zTbbuKWXXtottNBC7qyzznL33Xdf63QHmUA/cKnKf8xOr7q+7bkuy2gYbOO2eFTdT9mss846/pKbbrrJ3X///VWX157regxb+8CCC7rUj4LH2y2zEdh6663dIoss4mbOnOluvvnmMYkJY8iVV17ZPf744+7iiy8ek+8QZjpml9JfrLvuuv7S733ve+Et9tsQMAQMAUPAEOghYASuHhTVO0bgqsbHzhoChoAhYAgYAobA8CDAZNMyyyzjM/Tggw+6Z555ppe5z3/+8945/pe//MV98pOf7B23nTgCTILgOIXAMmvWrPhFAzg6YcIE94IXvMA7tJ588sm+P3Hbbbf1k6PyIAYNn/rUp+TnmNtC3tppp518vn/4wx+6q6++utU7fOYzn/FOUurWYYcd1iqtYbx59913d5MmTfJZ+/CHP+yeffZZ9+IXv9h98Ytf9Md+85vfuOOPP34Ysz4qeUq1E8OA2aB1ddDPG5UCn/1QJhn22GOPosdD/Pz0pz9ddG94k/Tpf/vb39zhhx/eO73UUku5hRde2D399NNzEFarzvUSGIKdVL0agqw1zsJHPvIRt+yyy/bugwB7yimn9H4Pemes6EAKFwjm++67rz8NiUSI1Kn6UKVLVee++tWv+mdA4Dr22GNT2bHjQ4jAoO3mYYLgNa95jdt///19ls4++2x37bXXdpa9gw46yK2yyio+vY9+9KPun//8Z6u0559/fnfUUUc5tiKnn366+8UvfiE/x8S2a1zqXjpmp9fdU3q+tIxSbesw2MalWOTch22IjYjwsdwFF1zQu62k7+16DNvLTEc7VfqR0oGOHm3JKAS+8pWveFL7r3/9a3fCCSeoM+ndYesn8U0uvvjifuxyxBFHpDM+hs7E7NK9997bfyTJa3zwgx8cQ29jWTUEDAFDwBAYNAJG4MpE3AhcmUDZZYaAIWAIGAKGgCEw6gi89a1vdfwhp556qrvtttt6eRInghG4epBU7nzoQx9yyy+/vCdwHXrooZXX9uskBDIhT91yyy3uO9/5Tr8e1Uv3s5/9rBP79+9//7v/evi4447rnR9rO107v+d2kkpsYmhun3Bpo9OpdmIYMBu0rg76eW3Krc29w07ggmyJ/j311FNuypQpI1616tyIC0f5R6pejXK2Gj+eqIXHHHNM776//vWvnlAxbdq03rFB74wVHUjh0pTAVaVLVeeMwJUqgeE+Php28zAhMpYIXET6ZTIb4WMZxqdnnHGGjyIzTJjW5WVuJnCVllGqbR0G27iuPNucryJwlfS9XY9h27xb7N4q/UjpQCwdO9YOgaYErmHsJ43A1U4H7G5DwBAwBAyBuQ8BI3BllqlMYGVebpcZAoaAIWAIGAKGgCEwaghoAte3vvWtEcu+GIGrWbEMg+NxNBxs4gTsMlJMM+S7vXrJJZd0LF+FsGzVAw880OoBcztJJUbgYskTjr/whS/0E2tjLTpCqwKvuTnVTgwDZoPW1UE/r6Zo+nYaUg5LlYSy4oorur322ssf/uMf/xiNVEfEEnHEhPc3/U27Rvv2pz/9yV100UW926smCavO9RIYgp1UvRqCrDXKwhprrOEOOOAAfw+Eeoj1oy1jRQdSOKUIXKn6UKVLVeeMwJUqgeE+Php28zAh0k8C19prr+1IH2Hpp7ZLKNJfvu51r/Pp8YEKH6qMRekalzoMYnZ63T2l50vLKNW2DoNtXIpFzn1dE7i6HsPmvEOTa6r0I6UDTdK3a/MQEN9NbgSuYewn50YCV8wutQhceTptVxkChoAhYAi4nt+Qj1zwxWNHszoLf+zL71ys5pu91MZ/ci8eS9cZgWsslZbl1RAwBAwBQ8AQmLcRMAJXd+U/DI7H0XCwyaTl73//e/flL3+5O0DnkpTmdpLKICeG5gaVGIZ2IoXjoHV10M9LvfdoHYfAdfDBB/vH/+EPf3CQpkdDqgg6VedGI6+pZw5zvUrlOXZ8nXXWcfvss48/dfHFFzv+RlvGig6kcEoRuFLXV+lS1TmxhWwJxRSyw3l8NOzmYUKinwSurt/zne98p4P8hBD9FzKyST0Cg7TTS8uoqm2tf8Oxe0XXBK5hR6JKP+ZVHRiNMjMC12igXvZMI3CV4WZ3GQKGgCEwLyIgH34agaum9I3AVQOQnTYEDAFDwBAwBAyBUUfgpS99qdt1113d0ksv7ZZYYgmfn4cfftg99thj7sILL3SPPPKIn0xeaKGF/BIVRx55pGMZKBznyyyzjHv66acdhJ0rrrjCPfTQQ9H3geX/5je/2b3qVa/yzyCSyG9/+1tHNB4muEqEZ2+11VY+DwsvvLD7xz/+4R34P/nJT5JpbrbZZv4L8Fe+8pWOSChEiiLv55xzjmPJPy3ked999/WHrr32WnfPPfe4t7zlLW7llVd2iy++uHviiSfcvffe66OXsLQRstpqq7kNNtjArbLKKm7BBRf0x26//XbH+/LF+bPPPuuP8W/RRRd1m2++uZswYYJbbLHFfF5mzpzprrvuOkcEFpFll13Wbbnllv4nX6yfeeaZ7plnnpHTfssyBEy8/Otf//LvM27cOPeyl73Mrbrqqv48+SP/t956q7v55ptH3Jv6wVIV22+/vc/fy1/+cn8ZEyR33XWXu+CCC0bcBvmPd5j9UYY/DpY8C2x//vOfj7g29YOlHrifdMD+ySefdA8++KD78Y9/7ATf8F709W1ve5vXAaLagDN6+7Of/czddNNN4eW937m6A44bb7yxv4/Jc/KkZbnllvPPR5+oH+ggAyWePWPGjDkiDDQlqWgdBEf0DYyIOsA4A325+uqre5HB0D0mqKnL1EvRzzDf8g65OijXyxZd4znoJrp45513uhtuuMHrqejAhz/8Ya/vvAPtC4JOkGctvMc222zjVlhhBfeSl7zEY0Z50z4QGShV9jqNnP03velNHpf777/f6yT5px1jqdO//e1v7r777nPnnntu70ulMM2m+VxqqaV8+0Q65513nm9vmDhDZ3k3cEm1EywHVIUZbcuOO+7o6yZlSLuC3v30pz9111xzzYiscx116tFHH3WxpdfWWmstr0+0HbRRbJEqXeULLr4Mpg+g3vH1FvpGv0HbQJ+hpQqL66+/3uMTPo+2esLsthG55JJLfFvgf6h/lMkuu+zij9C+UedypEnbRnpveMMb3Ktf/Wr3+OOPex1ZffXVfT2k/oMFkfnoy9pEAKkjcEnfQn5o337zm9+w25N3vOMdXsdoA0877bTecXbQOeoYcvnll/t2mfKDrECfjd5ssskmjjxQtylP+ppf/vKXviwp29Q5bAQtlBntEPWKfhkyGulQzmEfm6MXOu26fcEoVa90/0t0K9p22krabt4RvT3//POjulb1bEhW6AHtPu3bnnvu6XWXfvzYY4/t3Uqdz7GD0M+ddtrJ2xnYGwg4zpo1y+vZ3Xff3ThNuWHy5Mn+nelbqVdbb721W3fddd0CCyzgPvGJT8hlc2yr9CPUgSZ21hwPihxo2t5EkugdShG4wvpQpUuUM/1clZ7VEbjow8WGJnP0S9iK2Ge0/7kibRP6e9ZZZ3m9w5bC1qbfp1+m7tEuUN95T/padB59p83Cvo1JaTvZj7yQvyaYCS5N2mz650HazbwTJE3+6L8Zh7DPOIO+jXY8tJe4JybUYWwc8k+5YdNQ/ozLfvWrX8VuiR4LCVyMc0gXW4/2C/sanYn1tXXtIO/GmASbEbs+lFx7lH53ww039Gnxvgh5Il1sHCYpUiJ1gPNE1aVekNaaa67p7U/a2RtvvDFZJ7gvN59ci7TFpbSvamKnP5fT+v+5eSkto6p2F/s0ZRvzPNo2zv/whz/0thDtIPYttgwRLLFzGPMz9mfMSvvN2JY2Apsc2zUVFa6JXVOPYvqKGIGrSd8bplw1hgWHHXbYwfcVtDe8OzYL9gX1qEk/JM/tQj9o82jPqvpXeV6Tukj7tW9Dn448J7Wdf/75vb2HzUz/HYswjf+G9jM2BmNsxhgGO5TxLzYy15EO/XZKmvSFkgZlTFsHruwzbqHdvOOOO/wHd7xDTgSu3H6yqf0g+azaUoYbbbSR97UxrqF/om5jk33kIx/xNjP2xxFHHDFHMiV1GPsbHxP9BjYy/TR2E+Mo/CsxyfUv5Yx/QruU54UELto56gvR9ujP6MOuvPLKStuhBIvYu+pjXdkgOk3bNwQMAUPAEGiHAP0WYgSuGhwxjEwMAUPAEDAEDAFDwBAYZgRw8H3sYx+LZhEnHo4KWUIRxwgT/LEloJgcZdKKiSgtOFkOOeQQB1EsJjiQWP6iiWy66aZu22239RPNsfsgQTGZJcJE8gc+8AHvJJNjekvezz77bD/ZJcdxPhFtAiGPTDyQTigQTD73uc/5CWDyxORlTD71qU/1yCFMIuKEYWIyFJymP/jBD3pEDJyspI/zCGHi/vjjj+/dhgNzypQpfnKFgxC8IH/E0mYCeOrUqb17Uzs4w9/3vvf1SGjhdQwCTjzxxB5x6P/+7/96+dPXMmjgvauEfDJxLCSx8Fr07Zvf/OYczihIAjg+cTrGBELOCSecMAdpoInu4BhjIh1hUgCylMh2223nSCslTCYdddRRvTLnupCkkrpXjmsdpOxwAnMsFIhH6CfEklDIB/rDxIWWJjqo79tvv/38pIg+xj56i2MTRzQiBC79DqHuMsl80EEHJcuQNL/xjW/MUfb+AQ3//c///I8nMtGGUWdpl0KhHaAtwhGvpSSfmigA+Y9JWdFViJA4hGNCfcGJL21PiBlEmgMPPNAxcRATJlgpb5EvfOEL/lrqLEtMhKInjNBPJrOQlK5CHuNc6vnci/MY0ppIFRaQ/0466aQ5nkfdZrIBgQBEGxCKbm9z+5GmbRvPREeZ9EA/IE8xoR0TJgAvu+yy2KnaY3UELiZsZIlFJhuPO+64Xprjx493hx56aO/3l770pRH9MG0YbRkC1mAufToT/Ycffrg77LDDenW3l9DsHSFGS70Oz2nbgcnAFDakA5lJL0Oboxf6eXX7Wh/Ca3X/C8Fq4sSJ4SX+N20OhJbvf//70fOxg0LWYSKJCWjpq2lnhBTVxA7CVqLviAmTbaeccoo/1SRNSUvqNRNf9AnojsgHP/hB2Z1jW6UfogMldtYcDwoOlLQ3QRIjfmqdo42irULC+lClS7QBTCLGRPRMdCKMwEW7SXvCBFpMmEA/5phj5rBbYtdyTNom9iFrQUoOhT6PZdHf+973Ru3C6dOnOz580NKmnSSdLvNSgpng0qTNpt0cpN0MThAK0EnanUsvvdTbCRxHsOWrJvGfu8q5j370o/4jBvkdbmlz0akc0QQuPsCAiBsTCKBf+9rXRuip6HyqHZQyIT3yzGSzCBjkjon48EUIyXK/bE8//fQoiULO6/ejLYW4JXaZXMOWess4Swjtcq5JPuWeNriU9lVN7XTJa9W2SV5Ky6iq3a2yjfVYDTIHZIKwXKlj6Cx1LubDQKepJ1ynpaldo+9tuq/tcWxJbMqcvjf1HI2LHsNCHmGMFhtPktZTTz3liMqU+vgn9ryu9AP7FCJYTKR/5VzTuqjHork+nVge9DEIpPhfEIiyJ598sj7t9z/96U97/wY2sNhKnKjzYTD2w8+iSYUlfSHPoj/HD4c/KRQ+vEEfqC85BK6cfrLEfgjzFf7GFgS/mK8IPwf9CbZzjMBVUoch3kI6jgltBB9AhFFxm/iX0F/aIiT0Eci4OLRLuVYTuBgL4p+ICbYDNkQoJViEaYS/u7RBwrTttyFgCBgChkA5AkbgysTOCFyZQNllhoAhYAgYAoaAITBqCOCAghCA44MvABGcd5AIIHDh5BYngmQS5wVEAa5hYlecgPz++Mc/Lpf59Fjagq/mEM7jLMJGgsAgDtZcYhFp4FDBsSICMYEJCt6DL9BEcKTJl+dC3uAcTh5IZjh8IMSIvcY7QXYgf4h29vkDs/9xDSQVnGBE4QrzzyS7fDEnjjK+hkNwhvL+RF1g2QARHKSQJkhPO5Vx6IhzCAcNhCoRmYTnt3bu8uXs17/+db8UF2mRJsK78hzKkjKtEnRAlxn4ghfOM74sFOIGaTLxj3MR/dH446hkghgsv/3tb1c9zr3rXe/ykyhcRJpMGuGAwwEojjom4XgW6SJ8Vc0EhQiTnkSSYUIbHRB9C/Wqqe6knN+QLSAEUv7oBGQx3hcyJJjLJGA4cSaT57wn5VYnMR1kQokvdPlyN0YoZKCGjlMeop+iF/K8Eh3k3v33399H6JF0IABAUiACmbyznKsjcEFyQM9En3gnJlAoc5zJQoLgXXAOthXdBkha1An0mzZM8k95MknAcaQ0n9o5K89jS/roOPWJNjDWTqDvMQIXbTSYSblSH0iLNCBiyHHtuO2awKUdtZQ9E51ghO6TP5H//d//9W0lv6uwICobpLmwbkBMJdojEk58+IOz/+ky1c+T8+G2pG0jDT3xLGmiH+gP9Z2yFKF8SpZyqiNwoZ9MLlLGQrqSZxJlTSIFcoxIEz/60Y/ktCcR0TbSVrMsDSJ9uqQFyQuSGnWZZ3AtfTztGtvUOSaYED15h45Tn0mb9phoPwjHmUCVqJs5euFvzPyX0//qyQ/yQ79BedG30fZIHSISF9FrckQm6MNr6dfR61Dv6uwg9AlCHriJTUC7Tr8IMYZ8NU1T8ib1TH7LVuuGHNPbKv0QHdD1MdfO0s+I7Ze0N7F05JjWuSoCV5UuEblnk9kR61LtN+UrOhESuGQyl/zQzqN/1G2JCstx7uc6sXc4lpJY24Tegb/UZX0v5Yy+075K38t5yL3S54W6xfE6G5A0+pEX0i3BLJaXujabJWwHaTfzbkzgopMI7ZG0P+xDRoKMUiVE0COqH4I+YeuhN7Rn2gaEHBZGyoulqwlOch57FT0Nx21EypG6z7Wi83KfbKUd1GVCvUZHkab2KNEwIfpgA4udKOMsPsSR/kWer7ex98Om5v1IT9pb7oGIduqpp/Zub5pPubEUl9K+qsROl7ymtk3zUlpGVe0uY+iUbazHavIO9JfYaLptlXPUL/QS/dF2K0RWCK0iJXaN3FuyjRG4cvre1LM0LprABUFc3ptxHOMI+iHsUKlT4ccgqWdwvEv9oN5R16r615K6GBtPowdVPp2qd+YcbWwJgYu2FFuM9p480CfTx1ImK620Us+HQURlIjOKlPSF2LWMaaVvoQ7hM9H2paSfQ+Cq6ydL7QfJQ2rL2Er766i/YKd9idwbErhK6jBEeOoiAl74shjT0JbgNxThQxpIVEhT/5K2RSU9tryTjIvDcRrndV3jN4LPFh0K/Yjojo7KXYLFc09I/+/aBkk/yc4YAoaAIWAINEXACFyZiImBkXm5XWYIGAKGgCFgCBgChsCoIcCSAvwhfK2vo9CIE4FzTA58+ctf7k3OM9nIRLs4/Yj+guMP0Q770BkOSYOQ50Jg4BnihPc3J/4dffTRPeIK0cE0IUk7EoS0wlIqb3/7231qGLHkjwkyEe0MkYkGzoXOPhwkRBDBSYIQ7eg973mP32cCgHcRYZKcL9dxxOjIKJzX+Q+dK+SV8PQ42kLSgs4nEypE9iBCDV/TIVzPJBxbBKcWTjskNzoN1x5wwAG9r1/DCRocvDpaFkuNnHHGGdzmRSYqcpyAcg9EFbDGSQb5TyZ1OK8nt4hKRuQLhDLESYiEDnccWJC9yCtCtBKIa4jGPkd3Us5vIpVMeD6CBl/p86WsCPUBxyyTo2EZyuR5KYELghMYC0Z6wpznE3WO6HMI4xDqJWQ2Ji519CWNQ64OMuHL88ThzMSWtBG8K4QtTaCsI3BBdGRpAiSMmMcxvrSFFISAG3WzjWisqJd8RayjAaHXkFwQHeGoNJ+hc5ZJEoiX+pmpdkK3PToCF0RJIvwg6IKQvPgN4RHMEa1fXRK4mNykLBAmAKiHYCmioydoAmoOFrG6oScriBwIIVME/cahj4T6LdeE29K2TfdjvC+REfRkOO2N6H7YroR5SP2uI3Bxn9ZRlghhogLRz+c3/Sj9qQgTn9RbrTPSpwuBS66V9pj+Tiam6s7R19C+8Az6QsqKySgRlnckIgbCRCH2A5KjF/7Chv9S9Uq3YfQ35JP8iBCViwkOBKIL/SckxTqRfo/rSJc+kT5X6obWnyZ2kJ4QCtvp0jSlnpFX8sdkNXVV7AaOV0lKP0rtrKpnlbY3VWlqnasicEkaKV3ifNU50QlN4CJq18477+yTpj+jDxL9os2nHgtpXbefkpfYVusB5cmHC0RvQEKyCmRMoqWK/attSpYngxyIdNVOdpGXUsxCXHLb7EHbzdrGBXtsDwjN9Gk5Iv077Q79pegT9zIGQUcR3e76A4l/oc7Qjn33u9/ttWVExnn3u9/t23qS0MRp0XmO17WDmsBVYo/yDD6EgciBiL3pf1T8C98v/MiD90P/EeoTRBcZ85XmswSX0r5K30f+c+30Csg82VNs/6b9ZkkZkZdU25qyjfVYjfsh2RKZGIEUQ78nH/bwDtgg2EOI1gkdRanUrvGJFv6LEbgkqVTfK+djW42LELgYI8t4MCRpgRF2H3qE8KER44kq0TrXpX6kdIC8lNRFrTuk0cSnw/UxKSVwaX/bz3/+8xERX9FX+VBHk5FK+0I9bsQux+7FTkfWW289H9kX2x3J9d1U9ZOl9oPPQOIfvi6iMiNggu0k7TK+RPxs4oPUmJXWYdE9nqdJWvzWNqQen2udDMeBMd+kTod0Yz6C2DhN223cR7RgdEhEj8Opu/jV6AtKsZB0U9uubZDUc+y4IWAIGAKGQHMEjMCViZkRuDKBsssMAUPAEDAEDAFDYNQR0A6lKgIXEZX4OkwLS7OwhBsi9+LUkwgqGI9CJtL3aacMaZJ2legJ1tDxyH04H3FyQt4REoP+ak9PNujnaEIQk+RMgmhnH84PJkfECJZ79X16+SFx/nCfJnCxvJQQriAVyVJIkh5b7WzTWONgYzJBiEtM9OgvNfkSHsKVSJWDTa4Jt7wzzhieBUmISZZQICiBIyIYyzUyUZHrBOQ+IRcwgQz2Qk7iHF/EC0mOpY4gjOFwxIGFhBGu/MHZ/yZNmuSJcPyWCdQS3Yk5v0mT8oVchANYSCQcF5GoaGH5y+R5iJvcF25DHcTprifotCMvNkEnhA+dj1Id1F/168kRyTPRwKgP4giWCTX9DtrZuf322/tlqMgb7UQ4Ycn5TWZHOUGIKgchs41oAteMGTPcj3/84xHJ6UkeJiGEDFWaT+2cxVnO5LzWbR6eaidimEGSo24i5A8CD3qkBUctzmwEXQFTca6mSE56wgj9FKJcTFdXXnll3z6RPoRKIQvyG+HLZCGy6kmJHCxiz9PkOciC9C0i2kkdK0+5TrZt2jZNBoi12+uvv77bbbfd/KOakGUlb2xzCFz6nTWhR5Y0oS5R/7T+EjmL/CN6icfYxADXVE0Sps4RHVKW8UgtYSWTG+QRPUV3c/SCPDWVVL3SbVgqIg39jSxFqwk+VXmQfo9rIHnrCDBt7KAUgatNmlLPyGusHed4laR0oNTOqnpWaXtTlabWOV2+qfqQ0iWeUXVOdELsD67HbsJ+og7QH2E/aKEPpZ4gTOZhc9aJbptuvvlmT7bR94h9xTHeUX8kodscidrYVTvZRV7IcylmGpcmbfag7WZN4CLKBwS7JsL1jHdYZjg2fpK+AcJ1allW/TxNZknZDHppKT6qwBZAROfZD9tBjukyEQJXqT1KeiXkIP1+2GXYyKEdpe1qiWbZJp8luJT2Vfq+WPuestPBMyU6zab9ZkkZkY9U2xqzjblej9Vi5HNt/8fIsRC68B9ACpExXaldQ35KRdvjsoSipJXqe+V8bKtxEQKXrgOxcSNke2xvhKXLhSgTS59j/dKPlA6U1kWtO/TBTXw6qXcvJXDpjxpi+oifCNud8T7jX6SkL9T1nXEBH0ax1aLb81zfTaqfbGM/6DyF+9q+xKYh6rkWHZFNE7hK6zA+CexsBNyJ0qaFdPEZ4JegXpX4l7QtmvIRxOxS3T9pwqnOn84//c+9997ro/iXjNN0urH9rm2Q2DPsmCFgCBgChkAZAjJ3xZiO8SJ+Quxd/tiX37mpzzd7kuW/n+7m3jUGrjMC1xgoJMuiIWAIGAKGgCFgCHgEcghcOL00IUmg01+XycQySyHstdde/hLt5Jd72PLFnExYsNQBTtYq2XXXXd0GG2zgLwlDy8t9TEoRDQUnLpEIZAIr5tSVe97//vc7JgsRidiknX04b3DihCIORo6zT9QO2Y9F4NKOTnGq+BvUPx26neVTTjvttN5ZouzwHIxtLeF1nEs52PR94b6e0KsiIlBmlF2oDzJRkesE5PmaBMdEyi9+8Qt39dVXz+Ewk7xqHRCyoJyTLSQ30kWk3PV9uboTc37LM8ItzjzKB+c3uk8ZhfjI5Dnv2XQJRT2xIM9m6RzqHgLBjQlpLZBpJNy/EAxLdVAcgrwT6YZOYJ6rI5PVEbh0PmUfnWLpHxzXTCBIFLWuCVy8C9FIQoFkSr1BcHSnotLk5FM7Z1N1SdqPUE902yOkN103Y+RV8syX9ThoaYeIpkK6XRK4eEYoOABYgpL2bquttuotcZEicKWwiNUNSGu0NdSlsM7oCTkhq4V50781fqk8cH2sbdMTz0yI6+hS3EPUHCFbhFELOZ8jOn9hBC25X7fp8hwdeQ1CHeRVRPKpJwSFnMz52MQAx6smCVPnZHIFvUOnY0LUFpZVQqSPzakjsbTqjqXqlSa0Ur9DQiXpbjKbNAppE4mRUPyJ4J/0e9LX6NNt7KAUgatNmlLPaBvAiW0TSelAqZ3V5Nlcm9PeVKWpdW6QBC7yLZHnWIpHiFphXmVpq7BPCK+T37ptCqOBco20/5rUKfcScZI6gdx1113uG9/4xggiaZt2sou8tMFM4yJtobw321SbrdvYqvfXaem2u+qeWN+iCVxESyJCX1vhHegXaMsmPB8ptoTAFbMpyZvGSOwTjle1g5zXZSIErlJ7lPRKyEGavAIJgHYrFL1kvbxfm3yW4FLaV5Xa6SEG+ndpXkijpIy4L9WHx2xjrtdjNSGjclzkkEMOcSussIL/SZvIkplaWJ6aaOAQZqRNLLVrdLpN97W91i8Cl/4Ii/wxFsJex36Uib4m+e6XfqR0oLQuat1p6tNJ4VFK4NIffZA2YwrIonzIGLNLS/tC3d6l7Fkd6TTXd6P7AN3ntekLUxhzPMe+FFtHE7hK67D+kINxDfYRHwrRH8SkxL+kbVGNoU4/Nk7TBC5sCojfoUyePNm95S1v8YeFuFmKRZh2zu82NkhO+naNIWAIGAKGQB4CYtcZgasGLyNw1QBkpw0BQ8AQMAQMAUNgaBDIIXCFk+iSeU0kEQLXjjvu6DbeeGO5JDlRKGQklnUhMkeVaKIVy4zg9KgSTeTRy6KF97CUm3xxKqHPtbNPlmMM79Nf9wlhhWtSjke91FXVxKlgEsvzTjvt5J3VkhcczhAYwvRSDja5L7bddNNN3XbbbedPEaEIh1VMNDFIf50oExW5TkDS5stFnNbyzvI8vkjk62CcyrK0D+e0DqAvshyQ3CdbcciTDvnV9+XoDunoSQFxgkn6fJ3J0pxMkEHoCfPPdeEkrEyep+qRpC1brYM6koec32yzzRxReZAwjD7HdDkJgatUB2XivirvLD9GeSJSH/Q7yGSYv2D2PyZM+NJ4rbXW8pHlcFLHpEsCV1gm+nl66Qf9zJJ8auesjnykn5dqJ2KYMRkrxJJwGTadZrgvTu1UNA09YYR+VkXgkrSJnLj55pv7L5SFZCfnZJsicKWwSNUNrcMQIGgTIEtKNJHciek2bZueeI61OXpCrEnbJ1ix1RMfKQIX1wkJQCYaqT9EJICgAQFRiCESOUSIbjhwdBTM2MQA6Us9j5GRUuckygv3h/0QxxDdPkrfklNHnru72f9UvRLsqmwNTWrRS05W5UD6vXtnf+Uu+3J9GzsoReBqk6bUs3B5X8lv3TamA23srLrncb5pe1OVpta5QRK49HJ25C+nnuQsX6XbJibSaRe0iM7rSU05r3VdCFxdtZNd5KUNZhqXJm32oO1mTeCCBEzktaZC38G4gPKE9ByT3H5ST/inoimSvkyoazK5tH2xdpB7dJkIgavUHiW9EnKQfj/GFSwdGgr2HuMHRHBrk88SXKTeNu2rpH1uaqeHGOjfpXkhjZIy4r5UHx6zjblej9V0u8455OCDD/Y2Fvt8hBa2vzJeFLuK60rtGu4tFW2P94vARd74uA4ieCgsqY3fAfsxN+pxv/QjpQOldVHrTlOfToiT/C4lcDFuIooy/U0o2N5EVIRAK9HPSvtCfARC5InVC3m2tOe545dUP9nGfpC8hFv9rJgfRK6n/4SYrW2d0jpMX8pHHuiMFtoNxmeQ4agj4n8q8S9pWzQ1Lo6N0zSBK9aWkV/dz8lYvBQL/f6p/S5tkNQz7LghYAgYAoZAcwSMwJWJmRG4MoGyywwBQ8AQMAQMAUNg1BHIIXClSAAxApeOuJHzclUOZ7lfvvTkd07UFaJq4VhBUl8fck47f6+44gp3/vnnj1hCMRWmvCmBS0eb4rl1EnMyEoGMr/1EUl+SaqdX6us+SUO2+itCiZIi5/RWT8S0JXCRLl+8MynN8m96ol+eid7x9SA6QrkTaQhHWiwanNwjji+JPNFUd0hH64UmcOGkhDgV5pU84TyE0IGDNsyjTJ7n6DrP1w7nmA5qAtdZZ501x5J2mvwiBK5SHRQHr57gII9aNLGgjsBFhAi+imeyLBTwQSDGIZpM5Q8U/BMiSxX2EPLe8IY3+NRPOukkH8GvNJ/aOZtynKcmKHS5C+lN181p06Y5iKY5Ukfg0hNsdQQuJoepR7K0hH4+REnqKcueIuI0Zj8Hi1Td0BEJZZldHfExhS3P1aLxa9q26fZO6pFOe5AELv1lOBMMtIFEQBPSl9RvflP2TEzSToVRGqV9ZMKOCTERmQDOJXCRNm1DE/nJT37ipk+fnqUXTdKVa1P1Stqw2LvJvbruNSVwxSaX2thBKQJXmzRT9Uzev24b0482dlbV80rbm6o0U21Rqj6kdIlnVJ0T0obohG7HqvKnz8VIUPo8+3Vtk0yux/rtGIGrn+1k07y0wawOl1SbPWi7WRO4cso7LP9UW4DtSWQdJrQRISKF94e/9cRv1YcO0pbqdEOdD9PWZSIELumvwmtTv/WYSNsuYm+m7pPj+v2qbAd5PwjttJlt8lmCizy/aV8l98Xqu2AQs9PlXGwraTbNC2mVlBH3pdpW3T+Lbcz1eqwGKU9/9MN5TeCK2XAhgauNXcPzSmVQBC7yt9FGG7ktt9zSEYEpJhrf2Hk51i/9SOlAaV3UuhMbT/M+KZ+OvGu4zSFwic0VkublwyCIdDLWDdPHF4VPqrQvpK6vt956PtkzzzzTEZ0uJkLuaUvgamM/xPLFMSL30s8hRNVnbB4TIfYJgattHaZsd9ttN++fin2ohG9p6tSp/qOiEv9SyhbV7xazS3MIXEQxp79FGIvzYS31tInIOK3unq5tkLrn2XlDwBAwBAyBfASMwJWJlRG4MoGyywwBQ8AQMAQMAUNg1BHomsClI0XxpVroUA1fmMl/llGsEhyvRDxCNHHIH4j8wxaD+IPEolnJLRLFhN/f/e53PdmrjbMv5XgUBxPPwakP6aFKmDzAGSXC+0BECZ1JMUdLyUSUJuJJlBR5tt4S0QCyFaK/AJSJilwnoE6TfRyaRGNi4pov+ojsISLOZL0URiyyglwvDkkiGvBlZlPdIR09KSAELqL/oFM4B5EHHnjA6wtRkeRrWYgV48aNG0oCV6kOysR96IT2IDz/Ty9tIRNquh5JGXK5LBXFPmUEwQSSJaQJliyAIAdBDemSwFW1zJueYOR9yUtpPnOcs6l2IoYZEQKJFIhIlED/o+ZfHYFLfzmMXosOy6SDJrxpRy3LfKDzLHk6a9YsT66ESCRLg3ZF4KKeMblGmyd5kS+tmaROLYUXwtKmbdN6EZv8S5EBwjxU/c6NwKX1iolKJhjA5qqrrnLnnnuun+hgwoPJBZbf3Xffff1jjz32WAeZRCQ2McA5qeexydrUOSKjET2PtoHn1AnLF1EP9btUTabXpReeT9UrqQtVUU30kpS33367Y5neOpF+T8g6+vo2dlCKwNUmzVi91vmt24/pQBs7q+p5pe1NVZopnUvVh5Qu8Yyqc6FOjB8/vkc4p19hMrVKsA0h59dJXdvUlDTVz3ayaV7aYFaHS6rNHrTdTPuMTiJNCVx6Up++ELub/pjldRlPIWIHa6KVP5H4pwlOkGwZW4Si7RM9rgp1PrxPl4kQuErtUdIuIQfp9/vZz37mfvCDH4TZ9FHM6CsQlqdCb9vkswSX0r5K2uemdvocIKgDpXkhiZIy4r5U26p1T48n9FitCwIXeSi1a7i3VAZJ4JI8Ms6GRETdoM0FYxHqB/WkSvqlHykdKK2LWncGSeCSfo8xE+1eTJZcckkfvXqNNdbwH2SIf4FriZwLEVc+WGtiP7CsPSQ9JBXlSROdcn03qX6yjf3gMxn5xwd7fLiHpKI7co7xK7aoELg41lUd5iNDiHAQo4QUTfryrBL/UsoWJV2RmF2qCVy09+hDKJAzd955Z3/4Rz/6kY8W1hUW+ln9sEF0+rZvCBgChoAh0A4BI3Bl4ocBYWIIGAKGgCFgCBgChsBYQKBrApce2KeWqsBxtMsuu3jiDktj1TkKdYScFMFICBcy2S9fhzKhIU6gsDy0Q5D7mexo4+xLOR71hOQ3vvENx7I1oeCsEofbjTfe6Ilnco0mTjHBLpFwmLxhYpYv7kVSDjY5H9vipOILVIQQ/t/85jfnuEwTe4PLFwAAQABJREFUKkKHpExU5DoBl1122d4ymxdeeOEcBD4cZjirEHGU7b777m7SpEn+2BlnnOHAKBQ98QfBCgJIie7oSQEhcOml7FLvKQ5bykWcruSx6eR5nQ6WROAq1UGJYMV7pJb6EWIN11QRuPQSeJQr9RJChxZNLOqSwMUzpkyZ4qg/oYgDWEhebfKZ45xNtRO63GWSSn9RqydN9Tvo6EzSjsnkirSH+nr29dfsdQQurddhe0NaevmMrghcpKsnfk888UR34IEHcth/+YxTOkfatG36+aNN4IIoxaQ87TBtG20oIo583Wbi2F9mmWU8mYv6qCU2MSDpoH9NCFzS59Le8ZywLpMuk4P8IZC16I9z6oi/oeG/VL1iIox+kXxClqDtCUWToy699FJHv1Qn0u/FCFxt7KAUgatNmk37oPDdhSAQ6kepnRWmr3+Xtjc6jXA/pXOp+pDSJdKtOhfqBCRL6i0iUX38j+AfS9MymUtkPGzcOqlrmwTDWESeWASufraTTfPSBrM6XLokcLXBrA2BS9tIsnS91hdNrCwhcIntodNkH/sbOxy55pprfHQP9kOd55gWXSZC4Cq1R0m3hBykCVwpMoDuQ+X92uSzBJfSvqrUTtflFO6X5qW0jLgv1bbGbGOu12O1rghcpXYN+SmVQRC40O9VV13VE+5jBEZdryCDYnNXSb/0I6UDpXVR605XBC4iDlPnEB0d0B+Y/Y/IWvR7iPaXTJ482RHhCRt9xowZ/rz8o99jLC3+ne9///s+claJ/aDbu9S4cbXVVuuNqVI+DcmbbFP+pTZ9oaQd24p9GUYMlmspW8a6jIvEV8S5kjrMBx5EEkP4qIw6oIUyp3wYiyF8RMS4QaJ35/omU7aoflbMLtUELvFL6XvY174AooTxkVUJFkIED9OX3/2wQSRt2xoChoAhYAi0R8AIXJkYGoErEyi7zBAwBAwBQ8AQMARGHQFN4Dr11FN9dBXJlDgRUiQo/dWdTCTo0PI4rnByMHGqhYgyRJZBrrvuOscycFWiQ6kTrUscZ3IPDhSIOgiEMCb3ZcKSY8cff7xjUkILjigIJDh+NMmhjbNPHI88R0/4a/KPEIt0XtjXy94dd9xxPQKXjsDD19VHHHGE+8AHPuC/mOU+IudAwBBJOdjkfGyLs5HIZmDBJDzRdCg7LRtvvLFf7pBjocNSJipynYBrr722n4QhrdTX8DLpJ1FT9KR2+M6kg+hlECQ6WYnu6EkBcZRtt912nqjCc1iOIIykoR2Yw0jgKtVB7TSPLcmpSXNgU0XgglQiXyLHyhDiFHVbllfsmsAVy7/WD1mOrk0+c5yzqXZCtz0yicpyYtQF6iZ6BZmT9kqEcSdtHed19IUYqU7uCZdjrSNwSTQPnh8SdXgubRKkAKRLApeuU7yzLDdStcSTvKNs27RteuJZt+eSdooMIOdztrkRuEhLE475LcvEsk+dgWRDeYjcd999cyyfIX16OCGSIuiQVuqcduJfdtll/kt/eTZbJqToV9Br+hX6OLY5dUSnk7ufqld6YjJmbzAhwyTHwgsv7B+l60PVs6XfixG42thBuq8Tu4p8tElT7CFt61S9W3gupQOSLtc3sbPC9PXv0vZGpxHup3QuVR9SukS6VediOiG2DPeCYxg5QUd/yyXc1LVN8sxcAlc/28mmeQEnuacpZnW4pNrsQdvNbQhc2OdEvUTQNwhJWvg4ZsMNN/SHcvVJT/jTRh999NH+gxadrhA2OMakOREVkZjO+xPP/9NlIgSuUnuUJHV7Lvamfl5sX78f/Sbtvf7whXsg+NPGIhAH6D/b5LMEF/1uTfqqUjvdv2ziX2leSE7fm1tG3JdqW2O2MdfrsVpXBK5Su4b8lMogCFz77LOPj3JNHmPtBvYP9R5JkX78yef/6TJuoqvcru8N9SOlA6V1UetOVwQu3kHIRUyS0jZqOeCAAxxRtRBN4DryyCM9QYs2lvcOfWP6YxghBJX0hURXo42TcSNjB3wpWvSHgbm+m1Q/2cZ+0HkK93kH/AIIUXGJjqtFR9HXBK6SOky62J5g9uijj/bqgn6eTpePQSbMXhmAthfJ9U2mbFH9nJhdqglc6Bz9FXokgm1D/WU8o32vOs9NxmmSbmzbDxsk9hw7ZggYAoaAIVCGgBG4MnEzAlcmUHaZIWAIGAKGgCFgCIw6AtoBCmkAJ4k4esSJ0ITAxQvpr8BwhECokjQhfUHgEscSE4ChIz0Gip48wLl4yimn+DSXX355h4MCogNy9tlnu2uvvXbEJDEOeyJfCYkLJxTEMiEE6GWc2jj79EQFy1pBTpLlEnXEG5Y7IYoUzjsmuXFmrrnmmj7/esKNfArJjJNCsMM5B27ci+il1bQTlrQoQxxb2tHjbwr+aecuRj8TjSxxh7C8IXkUcgBONSaGRGSiItcJqEkp4EPZUKYIRAQikUk0MianJH1xfnId1/OFMGWLw2r77bfvRfUKJ6mb6o6uE0Lg0qQziDLoNEsdgTfXQ4QUfChXnNDinJVJ7jBfvEdM6nSwJAIXzynRQU0QIo0rr7zSTZs2zesTzkvqnhCuOC/OeP0OQkYCH3ATnMQhix6jY5AwpR6TFnX8jjvuYNfjTP65F52BDJIjEML4claEusIf7RFlit7LF7Wy5ECbfOY4Z1PtRAwz8q2jyFE3qQ8Q4GgHIBYJeYrIftQlRE+QQBg46aSTfF2B7Lr11lv3yoBrNWElpqu63jFRA6mEerfCCiu4Pffc0xE9UIQvl1mOFsnBIvY8SYutnrzgd5Oy53qktG3T5TQMBC7Kjf5ThAl0JtJFdDlxTPRZzrOVPj0kcMlx2izafto26TNS51hShLaV+sJ96AUEPoQ6B9GYPgzRulmnF9p+oL2hf84RXV66/9WTWaRz0UUXuUsuucQnyaQHbZhENIuRsVLPln4pdY9+jyZ2UIrART5K06yrZ6l3lOMpHdBl2cTOknRjW63HTdqbWFpyTOdT23vyXmF9SOkS6VWdi+mEJsxiA0BMvv/++33WiLD4rne9q9fvEXVD6pDkPbbVeYi1TdJuantS0olF4OJcv9rJkryUYlaHS4rANWi7uQ2BS5cTpHOWr6WcaWv50ABbSiSMmCfHw60mOHGO9Piwhoi8tJ/77befX96cc9gT9BEiMZ2Xc2x1mQiBi+Ml9ij3adtG7E2OV0n4fthRRBqGpMU48L3vfa+bMNueRcK2oDSfJbiU9lWldrp/4cS/0ryQXEkZcZ/WFd2Hp2xjPVbrisBVateQf913EQGaD7ZypIrAJX1UzDZLpa1xkTEsyyXutdde/hbIJhCQJCIxZU2bxEcTSGrZPX/y+X/90o+UDvDYkrqodadLApfYVOSLiH3YFRBAGZ8TLVVEE7j233//XkRabOKTTz65Z2cTgRObWQhLjF0pp9K+kDZb+gJ8QJCTGDcy3qV+0iaK5PpuqvpJ3S819SFJPsKtjorIeAQfmPgEIPTR34k/QRO4SuswY2GZyz3//PPdFVdc0csSeIEb+OmPpZr6l1K2aO9Bs3ekzuu+SBO4uJaPVYmkx3appZbyukOdRMg3+UdKsfA3J/7psu7KBkk8yg4bAoaAIWAIFCBAP4wwh4efnb6S/os/9uV3btLzzQ7DPDIcQ+6dQ36ddPpDnk3LniFgCBgChoAhYAgYAt45j1NMi0z8ihOhKYGLSRIcITjORHBiQfQQZwvHQweJXBvb4lw/+OCDveEp53HoYIiKhBFHdFQrruF6nKBCfOJYGAmojbNPO01JG+ErPSZDCFfP5Ku8P/kAE01Y4RhONnH66ig64SSxdupxH5NkTLojUm7+x+x/OV/TMolBmen8MCEbGvixZTFloiLXCUi+QmcUDjHeg+drjCAoPPzww/5VcC7j3JXzHBQClxwjDchxkOREmuqOLkdxfqMzfN0opD/ShkyiyUtMzsp58oVDl2USxNE72gSuEh3kPXX0MX6DMXVJ6hG/BX+ZUNP1SAhc3EsdJuqQSFh+OCwXWmghf5p077zzTu/k1ssacg/PyZGQwCX36DxzLIwqV5rPHOes1i/JD+0EdUAmRjVmYMnEEM5zkbDtQ7doL9gifP3NV+Ap0e9fR+Biea9tttmmlxT3IlLmYdsuXyLnYFFXN8J24rbbbvNO/F5mMnZK2zY9kRQjSaTIABlZ6l3SJAKXjsBEAtpJz+8QK76SFn3gPCJ9g54Y4Lh+V37r81Xn9Bfw3Idu0C7qvh+yJJMc6AlSpxeHHHKIJwdyLe1nzpJyXJuqV/S/2267rZ9Y4zqEfOo2jGPkj4hh2AQ5Iv1e2DfLvaV2UBWBqzTNunomeU5tq3SgxM5KPYfjpe1NVZopnUvVhypdqjqX0gkdsYB80och0oeyn9IjzoWiyyPWNpWQpvrVTpbkhfctwawOl6o2W3RBsO6n3QxZAp1EZIwgz63bEiGUOif9L9ejT6JLYRtMu8a7sYRnSkKCk1xHWuFzGKNI9C2uS+m8pKHLRBO4Su3REnJQ7vvxvhBvdJS80nyW4lLaV5XY6VJGqW1pXkrKiDyk2taUbayv74rART5K7Bru0wQj6okQdTlXJVUELl1/SEPbZqk0NS4yhuVaPR5C17EPaTe0vQYRBntN+qjUMzjeD/3QeZdnSxtZUhf1WLRLApeOdCj51FtpO2l/JfI041hsMfFbcQ26TRloX0LY95T0hfR1PCv06cjkreSPPDfx3aT6yVL7QWMW2+djy3HjxvVOkW9E+iV5D03g4nxJHdYf6pEGdYDy493keRyX6GjsN/UvpWxR0hIRjHVdD8d3cq28v/zmw0v8Bbr+lmAh6cW2/bBBYs+xY4aAIWAIGAJlCBiBKxM3I3BlAmWXGQKGgCFgCBgChsBQIKCjOpAhIuwQpQbCCqQBHAKQA0KJLaEo13AfjkeJbCHH2eKwIpQ3f02ELxQhVshXZnIvDgy+yjvttNN6k8RyTjtG5ZhsY2QAnGg48JHYeY7rpfqEsMJxHGY4m/jiTQRHqBjRLJXDl96aiCHXgTEO6JkzZ/pDRKDaaqut/D6OGNLBQaVFT5xqIhrLLk6ePLnnJJw1a5abOnWqvjW6T774AlSWZ9EXgTFfRDOhHopMVDRxAuKwPPDAA3tf/IZp4liGiBWGzAdD9EpIPvo+9IplnO4NlpThmia6w/IzOGcR7fxOPZvyoc4QrYClCsQ5e9VVV3nM5Kts7YzziSf+1emgjsB1+umnjyCrkaTWi3Byt4kO6uyxtAPOeu3E5DzkOiLNCV5SH/Q7aDISjlCik1EeWtAv8CMqFxHepI6TPhPAbQlc6AZkMBy04TvgLCdyFcQTkdJ88sUzXz4jQoSVNGWbaifQD2l7NGbcxyQEdZOIg6EwmQpuOioe18z+WMrttttuc7wv9YOJDCIhIprAldLVlPP4scce8xO4tLMsR4lQH9CDHCxSz/MJzf6HnlCnRJgg45lNpaRto31iwggJ6xHHqsgAnM8R6qOQEWUJz6r7pE/mGshGQtrlN3klz0gq8orcH7ZFkGOJBITeI9QX+jKk6hzn119/fV//pd3jmAgT4UQZgNQnUqcXmsA1ffp0x3K4OZKqV9L/VuWTukNECsheuSJL6ISTbfr+EjuoisBF2iVp1tUznefYfp0ONLWzYs/Qx0raG31/uJ/SuVR9qNKlqnNVOqEjKer80fdh57A0LKTCHKlrm4Q0he5jO2rREbiwm4l0KdKPdrI0L+SpKWZ1uFS12YO0m3UEC5aykgjFUg51W2zUnXbaqWdryvW067S32PBik3GuLhqQJnwTeYuItmClhbaRCKohwbVK57lfl4kmcHGuxB4tIQdpAhcfoWAHSl9HPhD6PMZgt95663MH1P+SfLbBpbSvamqnq1dM7pbkpaSMyECqbU3ZxnqsFltaW3+IEbPhqBeMVWLtZNV7x+wa8q+XfAvtM86nZPfdd/f2Oud1hFB+1/W9XBOKxkWPYYnSByaLLrpoeIv/jR3EWKiJjV2FU8quqtKPlA6IHde0LuqxaFOfThQkdVC343KY/pwo8PSzlJ0mcHENbRH3ka+Y4AMiOqAej3Jd076Qexg3M97WPimOk8dvf/vbbtddd/Vj7Sa+m6p+ssR+ID9VwnidMTW4hYLvAZ1effXVvX/siCOOGHFJlW6m6jCRjvmIIPQTkDD2mUQh1w9q4l9K2aI6vZhdSvQ8ouiJv4l8Cmlb7iUaF74+rgmlBIswDf27axtEp237hoAhYAgYAu0QEJvJInDV4GgErhqA7LQhYAgYAoaAIWAIDB0COEH4qgqDj0hWuRNJdS9CaG8mByByYURCHmCZrTZCXllyEJuLL1yJnlM18brIIot4B89KK63knWlcj8MqJES1yZO+FwcjS0jhvJToUXIepxBLj4EJTjW+iodYQvSFLgUnKMvzMCkB5k0miCgzHGKQRZisgUxyzz33RJ1CbfPM8msTJ070E04QAFhqighkt9xyi3cyxtLnOr56hLCAk5TrKU+JXBa7R4411R25T7Y8e9111/VliP6gy7qM0Un0DMINk6NNcJdn9HtbqoM4Z3E+ot/oLRNd4WRebt5xbKNjfB3M5DUkCJzKCNhRtpD00Dv9DBybOLZjhNLYs+WLcyGk4DRn8o66AWEGIhPblJTmM5WePl7VTujr9D5lAFGKCVrIOyE++lr2KWvqCu9L+8vkgMYzvL7qN+0CTnTqLG0C+g2uIpQZEwakD2m0C0E/IADwHugcX5S3kUG2bW3yOVr3Uidp42hLZXkdyUvVOcpp1VVX9fWW+osO3HjjjZVRXyTd2FYmIGOTsrHr9bGqesWkPXWBP9oX9JSlbKqi0+i0S/f7YQf1I82696vSga7trNFob8L3r9KlqnNhOvIb+4D2m8h79GP0PUz26nZUrh3N7TC1k4PEbKzYzegGJHdsUcZWkLixmWXZc85jIzCm42MI7JymYzpsZSZ9uR9brwmpg+fnSqk9mps+12kClywPx6Q7S53RX9EPQO6owmgQ+dTvVNpXdWmnS35K8yL3N92WtK1Nn5Fzfdd2Tc4zU9dU9b2pe1LH6X8gihPVlX5IxhKMw0qkH/pRpQODrotVmMh4lXaY/hxfRJU/irQg3tAm4QdizMTYjDKoG4+W9oXjx4/vfbyE/4txYIzkU/We+lxdP9kP+4H+SHQWvx9+gxwfXkkdlr6VeoJuP/HEE76PpR+sKtu2/iWNcc4++SOiF7pD/1WVN9IrwaIqH4JTv2yQqmfbOUPAEDAEDIE0AkbgSmMz4gyGlYkhYAgYAoaAIWAIGAKGgCFgCBgChkA3COCEPvTQQ32kLr5QzpGQwJVzj10zPAjwlTGRHpFzzjnHXXPNNcOTOctJ3xCQZVPCiC19e6AlbAgYAoaAITBXIhAjcM2VL2ovZQgYAoaAIWAIGAKGgCFgCBgC8ywCRuDKLHojcGUCZZcZAoaAIWAIGAKGgCFgCBgChoAhUIMAUQVYPo8vr1kmkwhQOWIErhyUhusavu4m6gdlPmXKFP/VsERQG66cWm76gcD+++/voxOwHAhLdpkYAoaAIWAIGAKlCBiBqxQ5u88QMAQMAUPAEDAEDAFDwBAwBMYKAkbgyiwpI3BlAmWXGQKGgCFgCBgChoAhYAgYAoaAIVCDAKSeQw45xF188cV+qYCay3unjcDVg2LM7Oy9995+aSjIeiKXX365mzZtmvy07VyMwI477uiXR2b5RBNDwBAwBAwBQ6ANAkbgaoOe3WsIGAKGgCFgCBgChoAhYAgYAmMBASNwZZaSEbgygbLLDAFDwBAwBAwBQ8AQMAQMAUPAEOgTAkbg6hOwfUwWAtd6663Xe8IDDzzgjjnmmN5v2zEEDAFDwBAwBAwBQyAHASNw5aBk1xgChoAhYAgYAoaAIWAIGAKGwFhGwAhcmaVnBK5MoOwyQ8AQMAQMAUPAEDAEDAFDwBAwBPqEwMorr+yWWGIJ9/TTTzeK3NWn7FiyGQgst9xybuLEiY5lE++++243c+bMjLvsEkPAEDAEDAFDwBAwBEYisMACC/ionhxl+e0//elPIy+wX4aAIWAIGAKGgCFgCBgChoAhYAiMcQSMwJVZgEbgygTKLjMEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMgWwEjMCVCZURuDKBsssMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQyAbASNwZUJlBK5MoOwyQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBDIRsAIXJlQGYErEyi7zBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEshEwAlcmVEbgygTKLjMEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMgWwEjMCVCZURuDKBsssMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQyAbASNwZUJlBK5MoMboZVtssYVbYokl3GOPPeYuu+yyMfoWlu3RROAVr3iFe+Mb3+j+85//uCuvvNJtttlmbr755nM33HCDe+ihh3zWVlxxRbfWWmv5/WnTprl//etfo5lle/ZciMBYacs23HBD3+Y+8cQTbsaMGXNhSdgrpRCgXdxtt93cC17wAnfnnXe6W2+9NXXpmDiOfRhr78eCjm+99dZukUUWcTNnznQ333zzmMC7y0yOHz/eUU7ImWee6fvvLtPvKq0XvvCFbp111nGvf/3rHfr2ve99z91///1u22239fXonnvucXfccUdXj+ssHbGLSPC6665zjzzySGdpd50QfedKK63kFl10UXf99de7q666qutH1KZH+Y4bN879+c9/9nYkN9Bevu1tb3MvetGL3LCWc+2LzQUXoBeUAzJ9+nT38MMPd/ZWq622muMPOe+889y///3v1mkvuOCCbptttnFLL720W2ihhdxZZ53l7rvvvtbpzo0JWB3LL1UbY+RjNexXDkNZLrnkkm7zzTf3UOE7GQYbYSzY7sOoW/Q5O+ywg7dZsJ/E9zWMeZ2X8jTM/RtjUGxbbO6ubKp+pKn1RdLXx6r2//KXv7jLL7+86hI7N0QITJ482S2wwALud7/7nbv99tuHKGeWlXkVgS5tkrnNBzuv6oS9tyFgCMybCBiBK7Pc53UCF4bsMsss49F68MEH3TPPPJOJ3PBcVvUOn//8572Dm0HWJz/5yeHJtOUkG4EJEyb4iczHH3/cPfnkk9n3dXXh3nvv7dZbbz33j3/8w0+u7rvvvj5pJmNwSiIce+1rX+v3P/GJT7i//vWvft/+GQJdITDotqyqXa16J8nn3/72N3f44YdXXWrn5jIEXvziF7svfvGL/q1+9atfuZNPPnko3rC0D6FNj7X3Y0HHv/KVr/jJll//+tfuhBNOyCqHUpyyEh/wRbvvvrubNGmSf+qHP/xh9+yzzw44B/WPw9mGLjE5JnL66ae7X/ziF+6rX/2qP/Tb3/7WHXfccXJ64NulllrKLbzwwu7pp58eMQELqX2nnXby+fnhD3/orr766oHnLeeBH/nIR9yyyy7buxQy3CmnnNL7Paidz3zmM55QyRjrsMMO84/V7eVvfvMbd/zxxw8qO/YchcBrXvMat//++/sjZ599trv22mvV2Xa7Bx10kFtllVV8Ih/96EfdP//5z1YJzj///O6oo45ybEWkzZDftv0vAlbH/osFe1V2vdg1g/KXVOVlZK5H/pJ8zutjjCr8BKNBleXIEnrul7YRIKbz0dtoi+Ayr+tO03J41ate5d73vvf528455xx3zTXXNE3Crm+BACRzPprgQ9JZs2b1Uhrm/k3qGvmdOnVqL89tdvqRps6PjLv0sap97DnsOpOxgYCULwSuY489dmxk2nI5VyMgbVoXNonuD5r6YFN9zFwNvr2cIWAIGAJDhIARuDILY14ncL31rW91/CGnnnqqu+222zKRG57Lqt5BDKPRdGIND1JjLyc4LD71qU/5jN9yyy3uO9/5zsBf4uijj/YTmHfddZf/kiw2oW8EroEXyzz3wEG3ZVXtahX4ks8uBqNVz7Fzw4dAG+dBv96mTR8yLxG42uDUr7Jrk+5YIHARJUMi//CuEL9PO+00HzVNHM2jTeCCkEm9fuqpp9yUKVN6RaInZ4eVwMXX/8ccc0wvz+ALOYcoqYMWI3ANGvH8540lAhcfk/BRCcJkLmPbM844w7cZ+W8871ypbRIjSTrv70n5fMR2H5S/xMYY7ephFX6DLsvYm2gbwQhcMYTGzjEjcI1uWX3oQx9yyy+/vO/zDz300F5mhrl/23HHHd3GG2/s88xHC3wE21b6kabOk4y79LGqfSNwVaEzfOekfI3ANXxlM6/mSGy1Lnzmuj9oSuBK9THzarnYexsChoAhMGgEjMCVibgRuP5L4PrWt741JkPKaidW+A5iGA3KIZmpdnZZJgKjPamsn3/SSSf5r96NwJVZeHZZpwgMui2ralerXgxSAktn/OlPf3IXXXRR1aV2bi5DoI3zoF9Q6Da8KQk4ReAaCzreNAJXG5z6VXZt0h0LBK53vvOdbu211/avCTkd/RQRR/OwErho46kHCEvqPPDAA5L1odmuscYa7oADDvD54eMUPlIZLYkRuIjAhp6yjCZLnRJ5zWTwCPSTwEX9Jn0EEkPbJRT32msv97rXvc6nF7YZ/qD9G4GA1bERcIwgcI22v8TGGCPLpumvKvwGPV6M5X0YCVxjwXaPYTnax4zANbolkJpcH+b+7SUveYmPFkoeL774Yv/XFsV+pKnzJOOuv//97+7EE0/Up6L7ELhYvcRkbCAg5WsErrFRXvNCLru0Sdr4YFN9zLxQBvaOhoAhYAgMAwJG4MosBSNwGYErU1XsslFAYLQnlXfeeWe30UYb+eWXWIYpNaFvEbhGQTnmsUcO2iFfNTkwj0Fvr5uJQBvnQeYjGl/Wpg9JtfeNMzEKNxiBa/iXUNQErnB5NXE0j3bUmFQErlFQ6caPXGedddw+++zj7+tqAqlxJp6/IUbgKk3L7usWgX4SuLrNqXO6zfjsZz/rifJdP8PSm3sRqLLrbYwxtsp9mMoyhtwwErhi+bRj9QgYgaseo35eMVYn1xnXLLPMMu7Pf/6z+/SnP90JRP1IUzIm4y6i9X7iE5+Qw7adSxCQ8jUC11xSoPYaIxBo44Mdq33MCADshyFgCBgCYxgBI3BlFt68SuB66Utf6nbddVe39NJLuyWWWMKj9fDDD7vHHnvMXXjhhe6RRx7pIch5lnlhELbIIos4vjjhup/97Gfupptu6l0X7iy11FI+fPIKK6zgCTC//OUv/fIl//rXv9ymm27qwypfccUVji9dtLzgBS9wb37zmx0OA57N84hCwNfpGN0iOe+gHZJHHnmke/3rX+8jHvAuTz/9tPv973/vyMNDDz0kyY7Y5uZFbpo8ebLHlC9yLrnkErf11lu7dddd1y2wwAJzDAZxvq255pru5S9/uf8CnyhhLNN37rnnOjAKpWk5bLfddo7J61mzZvkICUxGT5o0yY0bN85HS/j5z3/u7rjjDv8YsIaoNH78eP+FONEUfvrTn7q77757RDZ22mknv5wO+fz1r3/t02OSjLw98cQT/tj555/vy1bfSNjrxRZbzD366KPR5WvWWmst/1U5781X6myJTPCyl73Mrbrqqj4pBtT33HOPu/XWW93NN9+sk/f38qU75Yrcf//9Ppoc17HMiBbyyx+N5DnnnOP33/KWtzjaAlm+SK5HZ1gXXAZ7qQn9OgIXOvCmN73Jvw8GNqFyn3zySa97hLnVgs6RHkIZ3XvvvR5nvronj0RquPrqq3sRLzbYYANPLKMuo9NcT+Qj0o9Jk7zE7rdjeQisttpqjrJBaCuZiNfyjne8w7HEE+0beqeF+rTNNtv4Q5dffrlvp9q2Zcstt5xvx1/5yle6hRZayIezpw7Qhs+YMaMXGSKnXdV5Dff5moh2hzaVNkQLbcAuu+zi20i+pKTtp02gbb/++uv1pbY/JAigozvssIPvj2l/iCDyxz/+0fcNLEWm29fQeXD22We79ddf30cgoZ97/PHHffvEfWG/L69L1ByWXaA9Q09p07BH6FdiX7pC0CCCDXpM27jnnnu6CRMm+DyiW7l9iDxfb1Ptfajj0qdwL7p8++2362R6+xCCsaF4p7POOqt3nB3a99w+TG6kPDbccEO3yiqr+L6BfpsoTvTrX/7ylx1fPtNPn3DCCXJLdFvX18r75fSZ9Jebb765LwPqO457yuW6667zZaIzoPs6lrajf6cvXnnlld3iiy/ubQrpz+j/YwL5gnJadtll3TPPPOPuvPNOd8MNN7gtt9zS95vcA/n62Wefjd3e+FgX/SdtMeWGbYN+IuSZukXdwBYUR3MqAhc6Tv/CsioLL7yw+8Mf/uCwsWlHU3WL5+Tet8kmm7gVV1zR1130iLyRPnWRMQJ2JPUUgRwl9obYnZTblVde6ZjEpYwYD1A+2NxE7MJGi0npuEGnRTuErYoOoUsI+GALUz+1XdvUxpfnNL0vRuAiDcZhCHYt9QR5wxve4F796lf79pLxwOqrr+51Gb2hraOe8x46Ypu/8fl/Cy64YG8MxT71CluS+xhbcYx7Y+2pTid3H7y33357r1u08wgROBknXHDBBXMk06Q9meNmdaCLukhyIYGLsQz2Om0KZcR4F7ywk0Kp6n+OPfZYP76gnqP7P/7xj8Pb/fgip72MtRnkiXSlzZgj8ciB3PaZd6cNRaj7Z555pn+WTpLlHMGO8Rr1GnuP/pvxHGOcn/zkJ47xPzaAjOOwCy+77LIRY3mdJvu5bRTX0l5stdVW7LrzzjvP29T0Z9jQtIVgE6tj/obn/zV5HuVA/09/Qh9OH4c/g/ekHUZXqMeXXnqpfsSI/aZj/6ZtzYiHPf8jx663McZzYKE/lCX98Pe///0RcErbxUH8C7TbWihb+kbaP/Rfy2abbebrC2Mw7GpsI+oNfoiwz67Sa55LPa7y37UtS51vvd9kPBASuOhvpC0QmwV7jTFuTOjrqGvYxdjN2CHYzvgq6Ve0n5L7qzCjLaB9CG137uuiTmML0WfT/5E/nse7oS/oA+WcGheQh5h0pS/y7rFnhMdo97E/sP1od/B9YZvOP//87n3ve5+/HH295pprRtza1D8pN+Ozo/1mi06wzJ7UHe3vpX2lnUWwN2M+W5YWxhYBZ+wjRMqWcSpLfaO/6CV9FP0akVnpt6iLnKP+Mp6S8Sr2N7rGtTFp0neA577P+/ZyxzviRyJP2G0I74ffCF8p7xXr34bFhkT/5SMKljPHBtVSks9+pCl5knFXGwJXlz4MbMgcaWIj4FukviLY6OiiFtrdPfbYw7e32Hbo/1NPPdW7pIndrXW+K382/kXacYSxJHWB8TTzKfgVGevdeOONc7wX10v5ik+fY1pK/DD6/ib7tEPMyWBDghNjaPpIxgkpf0fTdlbG413OA8k7NtE5uYdt6TuU+hRK86nzLPv9wDNmk8jz2DaxLUp8sHV9TFe+M/1Otm8IGAKGgCEwJwJG4JoTk+gRJsDmRcGh8LGPfSz66jg8xaHCBBET7jhOYnLffff5ycHQ+YRhCmkndh+Tizh9kTCEP4bdIYcc4nA2xgSHNctGIDnvIE4snD4MRHAChYJxglEfTig1yYukKZM0TADgEIAQJfLBD37Q7/Juhx56qCc5yDm9ZTBy1FFHjRgwlZSDvDuNAU5+JrNCwTmBkwmjNBQcAywbyABPRKJ6MAjHCYlTJxSeN3Xq1BGTtV/4whf8tamlLBksQq5DwJCJ/i996Ut+sipMn4EI6SM8/6CDDnI4UWIC0QCngdZPHChM+PJ+ONmZMBZhkgJnF0I5UQ7IGWec4QeEqQl9SZNr+WpLD77kazHOxQQsyaOINsB5VxxqHAuFiT0mRZjcC4Xy/tznPud1UJ9rmhd9r+03QwBHAMvdIOEkPO0CbYAIuq7bHyafaUMR6iB1VOpzSVtG/RbnpzxTb9EXdJ26m9Ou6nvDfcknE3iHH35473RdGwbBjZD1KUdpLyHbGRgCTIhAfom1P2QCpx59gpA3dNuFI4r7cQaGgp6hb+idFshXEydO1Id6+7TXOBrDCTVxyOGIxxEBWRqhDcbhHXu+7kN6D4jspNr7UMdxHmInIfRd9GGh0P9+8pOf9IexDeRr5JI+jESYmMBWYtIhFCaQwB77K4fAVdfXSv9W12eCF5MoMcy59wc/+MGIiR+tL9h29GdM4oRCWdKf0fZp2W+//fxEjD7GPs9ico8JTqQrAldX/SeEDSHo+gyqf6effrqffBK9DvsOLoVQCcEkJkyA4fQPJ0qa3nfYYYf18NPPIX3GDvRP9FMIE2OQyhGpG5AnsbsgmIVCG//1r399DhJH6bghTF/bbuE5xh+nnHKKP1xi43NjyX0yNqDNA1tE67+OtIZNy4QdYxPI36myZmIFMowW8H7/+98ftc0Zr9Fu4NDWYyl9f9N90mNiVyYXw/ux+enXtT7mtidhWvp3V3WRNDWBi4n2mM5yHX3a1772tRHjCamnsf6HsYCUJfeTZ8Z3Ik3ay5w2Q9JNbZs8j36FNlf6U62fpA+BZMqUKV6X+C1jJ9p/iF0IE/8yrvMHnv9H+zx9+nRPqNDH2W/atuk+mol9xnPid/h/9t4D3LKiSv+ubhqRIEERBBUaUAFhAMUAioqoKIiAGBDDYCQY8HHMOA4yjhhGnI8RUAQFCaIzioyoCIiALSISBgVB/wqIoJIkDdBkvvMreI/vra69zz51zrl9u7vW89y799mhwtpVq9Za9dYq9Ga2TSWKIJTWgWvD5udyj/wAudGfUspNEJbY/iWyJi0Lv7vo9ZLdS7qNQWQ7+QZTex6bhu2DISaK4ZkTwHnag+uB6DTvec97suMp7yLnWexAfxG1tWvaFgCbHMl/N8q3zKXLtWHtAe8rgBrRz3OEj+mwww6bcovxhDEz52PSgwDEAWWJ2niGLMCWFl/cPvVyDtun+bbY8vTTlOhH6EtM0GMrIS+70Ljbi+o+KG8WcaLTSX76897mUgDXINu+yU+MT2LHHXfM5kfeLPrQQhfkOgtkIYBL3lfixd4/+ScZw+mHkH9bFvkAPknrx3jEuI5ekvMRM+7jo+M5p2HHDtf3uto78AcgX44+/vGPR10kN75J75gJOqT8v7l+XlrOSaQJj6XPYXOWROAatw+jSxmG1REAjLAoH6JNo+fhKxdJP+c3+jvtTG1/WL3b2/y4/Nmur2PPAdxK+zRlR2YdeuihUxbE6/um+lmpH4Z8hiX8I3xXLXZJ32eu6ogjjugv6NH9Ejmr8W6c80CUZ9g2N446lPgUSsup8qbHSfBTabpOQr4luoX3t64+2EFjDG2nUuVA5UDlQOXA5DkgeYvuha6AboNPgT/O9btrSWb1UOJTLZeub87w5+SkmeHFHHvxWPW/1157RZAKKxYgHAxMuOAAQilmhRITZCIUfFY9oRDhwJLTMp0MxUAmuowIZRRAEw7f1BnjAC7KgeNM6VIWJiL5Rqy4kIKu/LrUQYqRyoIRwgpZ0mZyD2UH4vdHPvIRPRZXcQxTFr2oSRr91pEJK8KTQv4MxjV1RHGb2wMhqTysQCN/qPQ75OqOAoxjHn6mhKOJNoBRoQlYn2zieTlI9C78JE2+GRPU+kY+Qc2zMraHAXDts88+0Zki4BllAShA26SNQkyCywiCl7RPyq4VrjzDt+U56ge9ueekwdEHUX6VmXMmTxVVDgMXQ5frTADzDd1BiOMQByLkabrDlwhsRDyAKB+RHygHqw1pv8obIBlRLSBXwOOF3j/6EHzGEYhSnxICn4khT5O8mCQVlZRF79bj8BygHeL04xunhhngVkUvIWVWgJ5wwgn9TGhDyFmXG7n+3EWW4ehn8oBy0JZxpiKPmdChb6mvC0jYRa72C5o5UTm9zshw5J7KgHwDsMaE1nrrrdeX+azqxTlcaWZwAJAV3whCpuMYp73QpjSxSxvEAQjlZJfGON4TqIhnU2CKT/zSTpHltBNkJTJespJIXETNFMkhp986MoHGeIFDvm0M0fO5Y5O8T9s44x99XbrLvvvuuwDYaLfddosr28kH0AXgC6hkDIMfOFXFE3iMfkbEsnQCoguAa9BY6+Mb30b5cq4xkygJbO8lgvd8A3jvZWKCjD8o115IE/AVegrvKi/pfUr/bW97W38hANdonzjdGfsl0/TsOABc4xw/idSA04zxXP2I8Z26M5GLc1ntOu0nPkHA87yHrEUn5/tDXGdCinREw77HRB4gIvjJN2Asoq8zdgDI8YmxHIBL+XKkbaL3UEZ9G377IpJSu8Hz0TmTv0yqen9AR2JClYk/5Mew9obSLn1Per/r1N7+HVyiSS3lyRHdmT5Fn3CwFHYCchJCVvNbfYZ2QB/kedl58cHev3EAuFJeUEbGdfJDbsveo84AH2hDUBd5Eh9s+DfOvkgWPiGkLCkzY1BqfwJEoP2L1E/1W0cBOPxbOoBrWHmZkxkASCDJDOWdOw6bH2n4tln81mICzh3g6baGj+M8B9HvkB2M/5J3XGcxFu1QNKyM4j0fo5UOR9o+0ZGY7M9NcPNMSX4u90gDwvbiW9BW8HOIBKbRb8kAfksvarP90/41yCeifHLHLnq99Bq9Dw+XRBvD27CPbfDFwe58Q3QLEXoxuhQE8BbAOvSJT3wijvWc01aQkcgXFmfRZiB4jZ8EfxDU1q4B5DAOIO8l11P/Xem3jJk3/BvWHsj1FeqNfkfZ3Q9F1BGPcOhAAfQ6dBlkB3arbBKK+ZnPfKYfiauNZ8gC5I344vZprpwlfZryoIcyziEHNA5zfRgA17jbi+pOOZqIyGi77rpr/zb8Qd9Abri+wQMO4Cr1T3oUJdJEd8AHQX4CSHL98MMPD0SpHweAi/QgxiPq5n7Ch+481A8Zu2lr3s6IpgfoWFQydri+p3To9232DgsBFaFFi3U07uOPRQblxjfXO5TXwtIhJU8pK/1aeiDlKi3nJNKkPNLnSgBcKhPp8F3H4cNAZ2ijUh3Bgcg+30DEYhaAQOmYVKJ359r8qP7snL5OmvAbm9r9DSysAMQv0vdNAVwlfhilOezx7W9/ewSd8R7jIX41ZBKLYXw+g2+keYtSOavxTmXkm+IzGGUeqLTNjasO1KWLT6G0nOJV7jgJfipN10nI2+0FfnfRLXL9TbZGkw920BiD3VGpcqByoHKgcmDyHKgAro48lvOk4+OL3WOEi+YPcjAVv5mUlXMoNV5xHqFcajKGFe0Aa6ADDjigDzIBlOJbCTDBhyNZ5Hm6IZcq3Th+P/CBD/SjTaDwyIhtq4MUI/JDEWZVFsYyhFOCLfLkUKa+OCuh0rK4woWijMHPZKWUcI/UwTWMBilHTHbgINOkBxO0dOTS7+B1x0jAOaiJHjc0qS/fTlEJ+O0OJPiOcQQ5gIs0AZroHm0FfsrJACBFW6iVALjIDwcffIDSCSdWC7IlFYTDBccqRjeEEkv7lDHkk8Y+ecSzTJDi2MO54US0FNq5gC3ccwdhFwCX6o0CzbdW+UiLVf4C9fmKwVQBZxUFRicORci/Db9ZochKRQh5xjcASEB9FPGFeyVl4b1K5RwQEIsUHNThzhPuIcvoryL1M769nHPen4eRZUT+m/twhDpWozFRLEIGIrOQOaTpE+ptclXv544qpxujnhZh1D2KEk5STTrjxIBPlRY+B5B9kh+MiwJpUTLkC6vJNVnJZC7jQSq7aNeMudyDkHkAK5jYQJ7hzIVIB7nGdWQlERaRiSKicuEwh3D+MiZIlsohxz3ePa4XLZGxgvEXahtD4gMt/5rkfa6Nu87g442Sl15Euej/jPulYxjge0UzYDIQfmkcJvoJkf80edQFwEUZ2/jUZcxU/UjLJ3j4zTYZbEtEmVzOpO2FSS6iR0lPIbrknnvuSRKxfugikLcX+ImDVtvTIMuYVPVJoHEAuCYxfro+nJZR7doBXHwj+iR85Hvz3fn+IhZOAISCXKcofY90GH/4TukEpE96+iS3+gbvUka+jbaJRtYjN0gP8slXbz/D2A0xoYZ/PkmYtknvr8PYG6XvyTZAFnaNwEW1aN8APgXw55rrD0RL1oKGvffeu79VHdtN0D4kB9PoGqk+TbrD0h577BG3zOO9FNiEbYj+Ix2cLU2QzVAXeRIfbPg37r6YTgjBm6OPPrrPO7bEecc73tGXqd5u1U8pam788fbiAC5v72nbbJKX5NEmM7jfRKX5ua0omw87lugjEPIcmcQR8uf5nUaJI1Il70OuI5bKKB+jSZNJDoBmivjmY4yDJEvzc7lHfoyvREuQfebRc11/L7H9ve0MI6MoVxO5Lu7+F5532c337OovWdxsDAdieZtxm1389ejJbGVGlAxIMoK+DHgfkj9HPh+ueX8R6JPrg9o1z0ziW5JujkrsgbSveEQl8vDyu5+FCXjGS4iJWmwPjWNc84gRrmt34ZnauMuetJwlfZpvig2jb0tbYeGUdJ1Uf6IeOZpke8nlp2s+PpzZ25aMCO8ib9dc8/Gq1D/p+bkOQ/oOFBE4eFwALq8b+ijtTAtvGL+RechtyPUCdFjAZFDp2OFjEel0tXd4Fl8hbYp+4NHbPU2XVT52LGwdEh0Q/yfkfmF+l5ZzEmlSHtfnZIdyPUf4zdVP3CYdtw8jl7euOf+G0RGQsdhimsM5+eSTo51Bf9ZCIF9oRn4lere3T9IYhz/b+yVppou70NexTSDaPvMq+pb6vg7gKvXDxAwK/smmpp0QOEC6I0m5bQQAHCA4VCpnNd6RBvo731C8cN2D+13ngUrb3DjqMIxPobSc8KKJJsFPpek6idsLw+gWaX/r6oOlvk1jTBMv6vXKgcqByoHKgfFyoAK4OvKzArjyAC4mAVHuIHesOFvZu5tJOUjKsG8b5hNI/p4rcXIgYvwAPIFovALt+Huu0LCi7Kijjoq33Qmk9PSeFCN+8zzvOfmEh94dpSyapCEPdxQoT1/ByEQlfHPykNyA5nBGl3wH0vS6q27Kyw0gHLVMKLmDzJ3srDZncgYSsITzXPk9XQelyPBLQUWkA+W2UOQ6jhK1hXTCCQcpABTKjeMMZ4gTkapwEkHwUQ4EN5BY7frZz37WX4vngPpIn4lSXxnqDsIuAC7SxkC+5JJL+u3VM9NqXpyUtA3IFXDqxuSIwArcd6Mr18cEGuJdd/SUlIX8KpVzwB3N7vTUd+cb0cYw5LWSm6gnGJ6QT355f+4qy0gDo4xVy/QPRfXjukiRFNL20iZX9W7uqHK6MerAAne4633kHvWmnXvUON2vx+nngMvynJwBKMLKYIgJTJxCqexCdqfAWDnuvb15NCWPRui1Bsij7WJd9sohx7O5MaltDPH0c+dN8j7Xxn3lqstz0vXti1yfKhnDGNfQoSQ3AF0iP5x23333AHgFGjeAKzdmssWbJvRTJ6DK5aAzya+0vTBGy3jRe64vahtqby85Pct5RDopOEppD3OcxPjZBsZQu3YAF1vVERUH0laLaR28f+Ecxmlb+h5py9mcTkD6pGcTgCtXRu/HgLuYdCi1G9K6p7+bAFylOn7pe5RLtsGwAK5cf/JoGdKLvS+RB3qjTwxQBue93uN6CZEfej1yyMG4nhb6OTIO8np30cE9nfR83H3Rx7omG8VlqkfXUT+ljLnxxycRBOAqlZfk0SYzuJ+jUfLj+2KbaDEX8sijprp9SN5un7BYSt/fy4W9RqQESACYUhnlYzSTO4Abvd17v/AJ7tL8XO7lfBUAiOkXkMvMYW3/Cy+8sNgnEjNv+Nem10uv4VWN0Z5Mzl/C/cXRxpCN5otKFDlZdht19+jJ6Lvom/hUtBgGm0t+RvqCFhDyrsh1HOx37KBB7Zp3J/EtVab06DKyqz3gfcV9MEobPwuyHPK+gi6Nvggxia0FavFC7x+RywTm9wVBXXimNu72qZdzmD6tb0t7APjjYHrK6uO010/1yB2VJvfG3V5y+XHN/b05/Z5n8M+i/0DyZfh7btfEhx7+l/MT+4KcdHEQrwGoQu/Edya9YRwArtw38LEo55vgu1Ie7Fv5T0rHDh+LaDNd7R140jS57mn6+OZ6x0zQISUbnY/Uq6ScvAdNIk3X5x7Kpfm/9xW3Scftw2gqwSj2CGn6nArtEV+BFobl+mWJ3u3tkzzG4c/2sQidjzETOeHkeqiP0fq+mrPiHeRsyVyC5zfMueZR0BMou+ur6MVatMYOHyx8KZWzlEnjHefjmAcqbXPjqkNXn0JpOeFTG42bn+SlNF0nkR4wrG6R9reuPljK0TTGcK9S5UDlQOVA5cDkOaA5EHyR2ED44LBB+ONcv7uWpG6h2JVTi9hzTQ4gX3GVKn2qIg5dHE+QDGMH/sjI1/M67rTTTuGFL3xh/Km0fQLHneN6h6M7ewh/jeENNdWBe1KMUIIczMI9yFd6qbyjlEWTNOSHMsTRSYo7UUS4nxLOXxRdiBVf1E0rOsWr9J3cd+CZtrrjUJQzwo1Qpa3tA/nN9hznnHNOvKXyu4NS7+ioZ9zxOW4AF4IMxwpEOGABtVQGHeU09+/vk0eaQNTzOtI+aae8hyNWBpY7CB1E4GnKAau00iMOXkIlb7311mHuw5GRfMLfFfDU2UFabMlIu4Uw8CiHE45NHJyQJrz9vp8PKgvOt/XXX99fied8W/pLpXYOOHhEESr49gJr4ZiGx5AmshzM6G2prT/zfk6WcT1HrDylHIBwkHcoBN5HeKdNrubS1DWV041Rd2TzHI5unCoAatW39H49zgwO+OQ7JWLrNCZLaLNSMNOSuuzyaAL+HJPYbK8FMQYyFtLO2TI0lbf+HvJy5513jpeY3CRCCiSHnHSQeNH+eR900AL1Q19hLEmJ8Q5neJO8z7Vx0sDBiZ5CPRRli+sOYNTEaOkY5s5L5wP5iDyCwbgBXLkx053XfI8/9qL/pPSsZz0rArW5zjbFxxxzzBTAX9Nkv5xKvKf2oskk+Mx4lwLYeNajgowDwEWaKQ0aP9Pn099tYAy1awdwyanXpD+SPpGC2G4NUmTc0vdIoxTA5aBk0hH5uCKdu9RuUJpNxyYAV6mOX/oe5ZNtoAlJrrm8bJp8k17A8yKPPiC9wqPGNPV55K6iHrosVLrDHD2/trRcJsoGc305J0+GKYeeHaUvukzN6dTk4eOIfyv106bxxycoBeAqlZeUo01mcD9Ho+RHeuiKyF70RCfJcb/mE2dEdTjllFP8djxn4Zf0XkUZKJVRPkbn2mFTHyvNz8EebPnGAp+UZO+6DSy7uEl2p7Y/CxqIpAkN6xNJy+O/Xf6mPgXpNaktoPeXJBvD9Q6A0ERBUORDFqcxQYgOqYVq9A18EhxdPui7N8kHeMuWVYCWII3Zg9o1z07XtySvEnvA+wpbF7OdaUpsPU7Udu8r6TP8Rl9eY401YhSi7bffvg+KawJw5WQB6aiNu33q5RymTwvc01Z2AQHbvj/lEk2yvSiP9Oj2idpf+gyLdfALQtLbSv3E/t7ZZ5+d9SWhXxBFF76xZeg4AFznnntuOP7446dU7b3vfW9YZ5114jXkNtuwOal9AqrEToVKxw4fi4axd8hT8iiVzZ6myx3XO2aCDul9DP7h24RKyhlf7P2bRJrS5+BzCgpSvjoSHY7Fa9AkfRjKLz2OYo8oLXTytddeWz/jEdsNGxvQ7SAapHd7+yYdwGsAAEAASURBVByXP9v19auuuiouLE/L6dt/e7/Q9xWAq9QPk+Y3zG8HbNPGzj///DBv3rwsuJt0XV6mOpvyna55oNI2N446DONTKC2n+Nl0lP6QymGeL51XU5quk5TqFt7fhvHBUv6mMYZ7lSoHKgcqByoHJs8Bza9VANcAXmtl3IDHFtvbTQ4gdyjJeZVjgoxbVkEwkeYrk5j4UShqf9cn8qSMamWjnkM5ypEcyDjTKBfUVAfuSTHySROuixwMI6fEKGXRJE0O4NQUEUplyR1LvwNptdXdlX1NAHn+gwBcTYohacg4cQVXDu2m1e0OWoGHpA/5pIk75NKtDAa1F9LSNl8+ecQKuJyRqqhE6Sokd6p2BXDhiGJbDQAKOOlz1ATgkpHp72yzzTZxCwGusRUdDkynNgDXsGVxsIXn0WRI+TP1/CEOaAJTDkBF2YOHrI4R+FCrxLQiNF0J3NafySkny/QNmGxgG4+5PcAgABPJUd3n6P2V321ylftNpHK6MQqKnK0R6c8p4aAlQh0Tp3Lqpc/U3wuHA0wg4ghJiW8LsIQ2ixNT5M6DnOziOXeYC1ijPuLjutLUEfkpB7pvTSSHHKAhnesdjk1jCNsyS4fw5zk/88yHthBpkve5Ns579DG2QIE8ep7qB1hRAIrSMQz5z2QG5GNQvGD/NBHUBOawR+NpE5+4OWjM1MQmzzaNxdyT3BEoyduLtkrhOSfXKdVeBCpq0ut4ny03WekP6b34Y4R/w46fg7JqA2OoLYtXpKXJQM6b+Cwe84yih5a+RxridToB6ZMnuQhcTd/GowFJ5/ZvPIzdQPnaqAnAVarjl75HGWUbOF+8/fskg09q5ewvn0xX/3a+snU7EXxT8skK16fT57r81iIHnlU7y73n+qgiigySJ7l00mvj7Is+IZRb4a28JVPdLlA/bRp//FsKwFUqLylHm8xQOdPjKPkpLd8akGvos0RTSOWQA7gOO+ywcNlllymJ/tG3CRMAo1RG+RjtY64ya+pjpfm53GP7JHSFlBTRQcCOEtt/FFmTlsd/t+n10mtcRvm7S5KN4QsGvvnNb8aFC2oz6L2AS1jcJFsY0DTgaUgyxP0sPo47Tzl3f4u2kxvUrnlvUt+StHM0rD3gfaVJV5VvSH3F8yVCzIte9KIIlsOGzJHkB/e68Ext3O1TL2fXPu1jcG4RpMqqRYSp/qT7fpx0e/G8/Bz9GJAuhI2FbE/JtyWT3lbqn/T3vv71r8ct79P80t/jAHDl2uA+++wT0CUgwCzpeCYft/w3PCc5wHn6PNegnB7uY9Ew9g7pNU2ue5ozWYdE90S3px878Nv1o666LvyAJpGm9Dn/3g/l1v5fNv4kfBhNOY9DR0COIaMA0Yp80bau6Tis3u3tM+cTKvFnu77eBAymPvRdyP3r+r4qS6kfBh2plPBLMOfhMoK0mEcjuiWLJKmXyOVlro/oOckqzcdxXeNdTqfz8abrPFBpm5tUHaij274am0rLSXptNG5+kpfSlE4yim4xqL+RX84Hy/WmMYZ7lSoHKgcqByoHJs+BCuDqyOMK4MpvoYhjdtVVV43GqVZN51gqxUNOLHcU54xh0vCQ2gJwedSAXD7pNVdGuzixmoBDOYfkKGXJTdKo7Bg+OAugdM92PZMeS78D6ejb5OruijsOdhztTu5QdGNOExgyfvwdnSusNY4NtR056XJl4T2flOgC4HIQoPIddJRjyiePdM3fxcGAIYRxlYZUdwehO4Oa0mxqS/CGiDZEUoDcwHQFnChshx9+uBcvuMErx7I/4BNmHoGrpCwVwOWcLTv3bYuI5kafYCWxVm4L8Mhv+onanju5yLmtP3M/J8u4zkQrWzmmzgLaIE5zjEXavPdX3muTq9xvIpVTxqiew6ECeA1AECCyHJ100knh9NNPz92q1xYSB7baaquw7bbbBqI65cidxoNkF+/nnAcaV9omOTztHICraUxqAiZNAsC1yiqrhP322y+ySZEhWd2qcdBBE6VjGI4pRelkNTmrynOkSQYBPHLP+LUmPvFM0/im9yXD9HvQUZMX/k1zYx3pOLhHQCy1lzYnuzvw9N6gcrXdLxk/29Ljnus9aRnlaNbEL/Kbeg9DgHgAGJS8BwgIKgVwNel6OWdrqd0wiBdNAK6mb9mUnuyN0vdIN2cbePt3OeqTWq7DqXzu4FX/9vZOpBOfANB74wRw+Yrqpqgd5Ot1yQG4cjq4ytt0bPoObXp1U1pc9wmhtgllyR3X19VPm8Yfr78AXKXykrK2yQzu52iU/JQekaD55qKmCCIO4Gr6tr5NOBFYiMJVKqOabDKVM9fHSmUpMtHBHk39LAVwldj+TW1c9UqPklHp9fR3m14v3b1Jdi9JNobLWCIA/fCHP+xv24duRZTrXXfdNbKX39jljDfIIGxwfGK+rXZTtFQS8DaF/YMdNKhd894kviXpttEw9oDXq6mvyDfkAC4WuiE7WHiUEpPTtE9twdoE4HL/jKehNu72aZdypn3aJ/6bdFfylexts21Uvkm3F+WTHrVojOs5fYPrHjlck+Sl/km+LQtyINLgew6iLgAujcW+xeegb+sArlzdBYqQrTHK2OFjUVObydk78KZpct3TnOk65B577BEAAvpCJtePcvx3OSxd19vKuNNUG9L39rzazqUbtvVz/1bD+DCa8h2HjkB7RkYBMhfltgLnXlN+bXq31znX5kv82a6vN8l5yqtv4gvO9X2lr5f6YWgfoxARwrDZ8EPxDVJCJhKpDr2uVM6Spsa7nE5XMg/U1AbS8uu39NJJ1YF8cj6F0nKq3E3HcfOTfJSmdJJRdItB/Y38cj5YrjeNMdyrVDlQOVA5UDkweQ5UAFdHHlcAVx7A5QN8G+Jfk4REMSKakRufRFzBKZOSb58gAJev7mV1Y27ywdNBGWUbRWjcTqxRypKbpFG5fU9uN950P3cs/Q6kJaVwXIo7acog0uQ011JSvgL1cV9OulxZuO+rMzy8dtOksk+Kw8s0LDppOuH0Y8IBGjQZ7avD0wmIJqdqLk03DDFwcT4QKpmVLvABUv/xCaFBCniJwVtaFgAJGHkpOT/Te/X3VA54m8GJjdMfwBQT66z2laFJf2FbMdoSlDpR1K+a+lBucoWtEulPchBcc801gYmEX/3qV/1oV4DK1lxzzYkDuGKlHv7H6nVWoeHMA8ym8nEbAKgUGH+nni9cDiAHAN/hPEP+IqdE2gZpkOzieR/TBFrR+NC2etUnEH79618HdAcodcjFi/avaQyhD9L2cgS4Fged9113FKovyuHiaaAHAcxF5hPJkQlteAbxHkBNqHQMY/sYAHVQLuII1+lPmhDPOb15JqUmPvEcMgleQOmYyDUH4PA9GB/aCIcqumGX9uI6pdqLQEW5SKfK17cN03u6N+yxdPwclE8bGEPtWgAu0lJYferN+DCI2A6GLbtK3yN98TqdmPCJsVwErqZxKuds9W88jN0wqP5NAK5SHb/0PcqZsw28/Y86+eZ8bYoi4tuKOJh0EB9z913faIvAhf3IJAWkRT2D5EkuP12bRF/0CaGm6GX+rbxPqp9qQkjl1NEnKAXgKpWXpNkmM5RnehwlP9LCT8IkP2OmEwBRAT113QFc8IbIZCm57BCwo1RGNY3RytO/m/ex0vxyZVdeOqZgjxLbfxRZo3LkjuP2lyzONoZkNmMZtjtRueTXYBEKUVcgfFa0QxY5eHQ++g32F+QyI16wf4rKzCW2BsdGG9SueXbc35I0u1IXe6BLX5Hu7wAu2cSUBbAH9ir8Z+EjE8Lo7ejW0MICcDmwRAsSYoGSf/KZpfpT8lj8Oen2ksuTa77ojwUolDUl35JKAC635YbxEwPUmduLBg4J1B1/tPwbBODy7zFJABdFLB07fCzKgVlI23Vht1uaJtc9TR/fXO/oCoyatA4J6JW2BgkoX1LOmMDD/8adpvS5YQFckmOT8GF4ff18HDpCLqoifg0AN4x1olK929tnrs2X+LNdX2/aXhoQMN8E8giJ+r7S10v9MOLLqEcWtm6yySYR/A3Q3/3t6s+lcpayyVeVs8fJC/Ae1HUhf2mbm1QdKLvLLY1NpeUkvTYaNz/JS2nKn+hj2bC6xaD+Rn7+LbqMMbxTqXKgcqByoHJg8hzQ/CdjNj435nNYfMsf5/rdtSSzelGT8nvadU1hhj5XAVx5AJeDrI477rhw3nnnLfAFXfEFGMBKpde85jWBsOfQkUceGR0v6Yse1UcALjcOmkLi0mhJH4UXAx3FHRq3E2uUssjhp1UHad3lzGky8uAddYSI/sQqQyKWQcN8B56XUjguxZ00Vf6miVNW8WAQ8K3ckSnjtokvWqVIHl0AXAg1wE+Qr66JF+wf4fcBi6AYM8kEvXnAZLScSwAEmbRwanKq5tJ0UJqMCk/LnXWTBnCVlsXLW8/LOMCgS1ulTyAncThBTIwDPiSaDpNeEL8f//jH9ycJ4sWH/7X1Zx7xCVW1NyYcdt5555hCE5BDod8BnChSEC+0ydWYYMM/lVPGKI9tt912ATAg9TvrrLOmvElfBvSi1da5bUGnvFB/TAsHaJdsEYOsB6CVkk8kK/x6qfPAozYCEMoBv90hc+qpp8aICJQpdcil5WwDJqXPpr+b5H2ujevdl7zkJXFLHH4D+qIf4UxMJ3BKxzB3XjZNCm6wwQZhr732ikVq6vcqr45tfMqNb3qPo0+4NW2bRURXAc/QJyl7l/aSm9BgXFYEhqZtkAWko3zupOL3sDSp8dP7UFpGtWv/xtqOB1nN84CzUqJ98AfR/tD/St8jjekAcJXaDZSvjZoAXKU6ful7lDFnG3j7l7OeZ0smtdZZZ53omOX9JnDWDjvsEPWEtme414UcDMYWyEccccQCr6HzYBci6zzqwiB5skBCdmESfdFlqn8Hy3ZK5GiiRqFjQeqnmhDydzj3bykAV6m8JL02mcH9HI2SH+k5CI9xTLoacoh2DeBZ5ACuJoCxg2uRTbJ/AAMNK9uaxmiVp6mPlcrELqCUFMBFWWQ7d7X9sWvZZgca1icSX2r416bXS6/J+QxIbkmzMXxLbBatrbbaalMmg/FVYMfj60C/Qd6xFTzjrkjfvYmnPOcAS/WHQe2a98b9LUmziUrsgS59Rb4hB3C5TZrKF8rn2/cuLAAX5RCQJ9XtuQd5lN+mZx568u//J9le/p7L1DOX2SeccEIEJE59IkRZhP4Dyb9Q6if2ftUE/pZ8ls8Q24HFK1AOoO4LLycN4FLZhh2rfCzKgVmoW87e4fp0ALimQ4eUHsziEvq+60ddgWbww2mcaUqfGxbANUkfhtfVz0exR0jHdXjaMn0N8Ah08cUXByLrikr17kFtflQAFwsE9M1UVo7uV23T10v9MJ7XMOf4f5///OfHV4jqqUAESsPLrTGxVM6SZptOVwLgKm1zk6oDdcwBuErLSXptNG5+kpfSdJ95qW4xqL+RXwVwwYVKlQOVA5UDM48DFcDV8ZtUANffAVwOuPKJj7/97W/9VYTOVjc0tRLXV2k5gEfvsdLgrW99q37GKBpE0/Cth3D0ExUGg8LJt/Vjj3C2j4PcieV14J4UoybnWc4hOUpZZEjK6UAZnHzika3xMOKdUiAToVR33333+Mgw34EX2upeoriTppxLnGsFFeciX80zb968QGQGqG0yNd2aowuAizTl5ONcYBjORR6xRRME3Bs0eaSoWKyqxZnl1ORUzaXJClFFeMHATFei+6Sll2+QAl5i8JaWxetez8s54A56UtEqbs4BpNJ+cf6Lrrrqqn4EHV1r6888k5NlO+20U3R2c5+t1tJIdakDpwnAlcpV0msildON0f333z9O+gE4AHiQynZ3yDc5dJvyq9cnwwHGHfQAKCe/AOsecMAB8b5AJoNkFw/nnAc+Ke1je0y89w8QJE5zhfn3MULOu6YJ9DZgktJvOjbJ+1wbVxrwAMc0/ZnJWjlE0+14eb5kDGPshhekTz9CtpCPk0+4TweAy4GiAvN7eTj3Vf6HHHLISAAuByTkwCq+uIC8U3AU14ahSY2f3u7TMqpdq29RXnein3baaTECm9cDZzTgAdogshaecyx9j7SnA8BVajd43XPnbsdo0pHnSnX80vfIM2cbuLx04FDJpJanhX6BDMCWEgG6YZIJfQPK9Rs92+VIBBraGnKINkYf8fxIg4kKtgmBfCVxTl+OD3X4N4m+6AAu6sK4hk7upAk6riHfmYCE1E+bxh//lgJwlcpL8muTGdzP0Sj5OSAZMDcR8t7znvfECJLkldqlDgZgcgrdz/U9xkPGT41f0jlLZVTTGC0+eL/wPlaaXxdQSg7ANaztTzvUdszD+kRU99xx3P6SxdnGSPUI+OkALe+L4jVyHvCdSHKf34ceemigDTqhnxLthP7gfqNB7Zo0xv0tvVzpeYk90KWv5ABc8sEgN1KgOnxCBmkLvoUJ4BKAD14dfPDBcYxzvvmY0RXANcn24mXzc5/szpUTvZLvxBGSLuX6VToOKP2cn3ijjTaKiz54JrdQMgfG8kUpKbgE+xC+AQCGJg3gKh07fCwqBXBRPwc6eZo+vrne4c/zPuRRXmQjelqT0iEls+jbyD3kClsqQ13LGR+2f+NMU/rcsAAuHwvG7cOwqk45HcUewQ5gvgE9HjrllFMCCzEACiJjoa985SuB7YOhUr3b21SuzZf4s11fp53iD/FFBJQX3Qn+QMxb4FOF9H1dXy/xw8TECv5tuummUYfn1aboYSqPwP6lcpY85KvKzYGVzAOVtrlJ1YE65gBcpeUkvTYaNz/JS2m6z7xUtxjU38gv54PlukDCnOdkMdcrVQ5UDlQOVA5MjgMVwNWRt+MCcOFMmDt3bjTgU0WyY1EWymPuYMH4IyKWJgM16U7BmET68pe/HIEHGMtEddEqAnc68aw7IAh5/q1vfStuqUMkKZR13hcpAhe/99xzz7DhhhvGW6x2BIGusgBOAMAlpy95iM9tdZBilFNeySgHeuB6aVlU95QnpAmxZdgee+wRzzE8WLVO6Frq9YY3vCFuUcVNNx5Lv0Nb3UsUd8rlAC4czIT7B4CHMf6KV7wifl+e4zeTIID4IDduicKDYUj9cdQQFUAGI8/65LyDBOAJdWJVCnk78At+40C7+uqrSSI6BN7+9rfHyCf89qg+bZNHbhjyLd0RSzpNTtVcmu7wZNsstjyiDjhscXoDZhS502yQAl5i8JaWReWrx9E44JEvSEmrD5Wq93Gu5VbBtvVn3snJMncWMPmGTGUrUfoVchPHl/oefRYDjiPUJlfjAw3/VE43Rj3iAvIO8Cp9GCJCHhOCcr4y0ZWuSmvIql6eIAccVMH3QPYjpyDGD2QeAEBIkTYGyS6ezTkPHJTEMyeffHJ0KHKOoxmHuSLXueON+zmHHNdFbWOInmk6Nsn7XBv3NJiUZvJPRJ/CAUofdCodwwDBa/xgPGTCiwkUdCvGWsYxkZzz+t10bONTbnxL03HwOdvdEDGUejPpQ5k23njj+IrrNl3ai08ACeSE3s74LNnFpOr3v//9KFPQw2kvAqqQqd7j3HU7n4zlXhNNavx0vcjLSDnUrh3AxdacTApSb3jLRBoTmBARyZCj6BeQb4lQ+h7pqK2TH2Auxg9kt48Po26hSD7SnTkfxm7g+SZyp7EmHfWst4Nh7I3S91Q/tw28/Y86+Ua9XC7Qz4466qioayDLiWCBnBU5gIu+T/+lXbH9qba50bNNR+8XOB1oH7fddlt8HPlE+1YfZZJFoKgu8qRLnsPo1U3pcd31fn7DOxYIMaEFz+Ar25tA2C/UU6R+mo5Luu8TqQJwca9EXvJem8zgfhOV5OfgEtIVkB+e0J41of/jH/84yl+ecQAXvwHusU0ithR+EiYG1A59UUGpjGoao8kbaupjpfm53NP2jw/l9Pf/OQBXie1fKmv+XpIFz7z8qc9Hsn4Yf8nibmMQQdB1CSZX2ZIJShckus0jznv7xOdBhFL4DtG/WKyoSXTfptvf8+tKl+O4v6WnnZ6X2ANevqa+kgNwuT0MGIKxG94RIQg/GdHORGw3iR8K6sIztXH/Vl3KmevTHi2F8hGpmHEVPybR32WzUDb38fC7ibwO424vTXlyXfoJ5yw2/NKXvhQBhchq9AGBIbjvupR/q2H8xA5u4z2i/eDvZdEq+jtRiyH8x+ecc04ca7BzIHRR2tMFF1wQnvSkJ0V/GpHLRZMGcJWOHT4W5cAslD9n73Dd9QgikAH+QFfzNBemDkkZ5R9u0yH5rshQ9EIW+eL/GRXANc40pc+5nUrdBhH9RAureHacPoy2vEt1BOYh0EkgB1F6pCT8FYDsOLquP4ze7e0z1+ZL/Nmpvo7twVwKIC3G0r333jvM7fkBIJfz/Nb3dX291A9TYjd5W6WfoA8g/yD0DOw0RSpHDqu8pXJW411Op6PNYhdA7i+IF3r/PGiC5DD3StvcJOpAeXIArlHKybtNNAl+Kk1vq6W6xaD+Rr1yPliuN40x3KtUOVA5UDlQOTB5DlQAV0cejwLgQnljwgQDxEFJGN0YkABKOJ/J5AqcyinwAJOzDOhywHOf+lBXXcOQZqKOCTsRjm74omd0XUfe0T0HcDFRC3gHBUTEylOUWj3P9ZNOOimcfvrpeiQ6g6WE6qLqIMUop7zybA70wPXSssgJ4pM0pOfkBjrX4QekOvIbpV0rRkq/Q1vd/bsPo7jLQI8FfvhfWn4upw5Hd177uzr3NuEALu6rHnrWJzR9VRD31d80scA1N9T43TZ5hOHHtmFNxrs717yOuTRxKOH40nclb8qnslFnOWC4R1unrpoI49q4DN7SsqQANspUaXgO+Gog3kZ+IcdE6aQX7RoZ4qR+MIwso60BpNQEAenR5nxCgnx0n/ZJu2abQ5cRKofkqn7njiqnG6OAs5CNGidp+ziFKJ+Xxft2Lu16bXo5AJhOW9XxzWgrfDMfowEQ4Yyn7YziPNhxxx37AGBqSX4ARSQvuYaMZCIFsJJIDq5Uzus+R7VJXevazprkvdLzNq60OfqkJr9zEfW4DpWMYegn9CdNcpAO/JduBu807nQFcJGG6sU5JD7lxreHnvj7f1bI49hTvpSB7+Vl5BqTMETpgrq0F9eXHOTkkT9IK20vzgN/z51XyDltrUwaTTSp8bMNjKF2rW+gsr3yla+MDkv9pp6uR3CdiTD6JPwXlb7nTj3SUpv3Sc9xALhK7QbVL3dsA3CV6vil7+VsA2//45h8o6+hP2s8T3ni+qcDuBifmXyCeIb+0oXIh/y8j/M+MkBjPemkW8B1kSdN+U+iL6YTQsrbZQjXqBvyS9G3uKZ+2jT+eP9xAFeJvCS/NpnB/SYqyc8jJ6f180kv+MSkLODOVJdVeVJeYucgo5xKZFTTGK10m/oY90vyc7nXBErJgT3Iz8cyfsMTyMdMt/1LZU1MtOFfm16v8b/aGH9nnusLqWxExiEP9P1y9jop4QdwMA96Ld/edds0gtGgdk264/6WpNlGw9oDXfpKDsAF+IkFgaK0n6DXuF9QAIQuPFMblx5DHl3K2dSnAaQKLKDy5o5dAVy8O6n2kiuXrqXjA9fdV+DnDuAq9U/Cs3322WeKnkC/cL0htZs+9rGP9SOvqdw6+vgyaQAXeZaMHT4WNckKHyPcbvE2qjrDD/woApQvTB2SMkkXSuWkyqujZCp9kEW3owK4SHdcaaoOTT5g1SF3nKQPI5cf10p0BAce02+QwQIlM5ZhCzC2QLI/S/XuSfizu+rr1A0ANosuRPq+qT5b4ocptZtSHZk+TFmxo6RLpN+lVM5qvMvpdK4/DDMPVNLm4P8k6kC6TQCu0nKSZhNNgp9K03US8i/RLbqMMZKV5NFljEEWVppeDrD9tvww+I4E6siVgnb+2te+Ns5dco5+jI8CHYNFqsiSEkLnIAI3cgJbhXQInAKgnh0AcsTiCoCfAPGZO0CfQ76hozO3RJlSIpDH1ltvnV5e4DcRjFM+sKiDgDLkCyYDHZL2ysLWU089Nf5eIKHeBXyDW265ZVhzzTVj/dBZmMu4/PLL4wIFzpuohC9NaY3zOmMH3+u5z31uDIgALwiiwhwA7SDlXZe8S9OczvdYGOg7uzXVi/aA33MQCQdAe1eE+Nw76CHofOxuQP+gjdN+WJigYDK59wZd03dizCZdeEk/4o9z/R6Uju7P6kVPKpMASmGGHksBXLzH1jUIjCYC/IDRj/CayeRoespJJANW1EJsQ4cDetlll42//R/1QqCySiAlVhaywmO11VbrK6QISBo1EZto9FAadhx+kp87u5Q2+TFo5AaOpjoAXiBNVoXjjE6pCcDFcyVl0QqDVBFL833Vq14VnUbpdQYenMLpIFfyHdrq7op7Go6cMvnKi2OPPbYP0BOAiwlYJm8Uxl71QOCdccYZcaDWNR2JwLbrrrv224Ou036oL3lCKYCLAWm77bbrO3aIznDQQQfp9bDbbrvFSF79Cw+fUBbaGls9MpCJfCUR28woyhv3Fbo4nXDSu274OpilKU0GUr61O6VIi/ZBFCK2WGQrRRHGJpMgHKFf/epXcdW77nP0FUv+bfSMO/88DG5JWTTRrrTrsZwD6o+kwLjAdxbhPN1rr73izyZHr94fVpY1yQ7kMbKeKBOMZWqjZ555ZmCFJ9QkV+PNhn8qZyoDUXboJw7Y8iR++9vfxlV0OIorzQwOMI7jZEdBzRERVVhBqEiLfNs22UUaTc4D7mFQIQ/VFrkmIi/Gn9SZoDFJjkY978dBY4g/6+dN8r6pjfu7ONVxpkCsLmecbaJhxzDSwXlIxDxWgzsx7hF5B4OecX4YAFcTn5rGN8+Xc2QNxk9OL0ZuodvQz0Vd2kvThAZpsPUquiQGjRPOaFaoa2x1J5W3P3QVtv/uQpMYP9vAGHI0575fWz/BWY1ugTGZUsl7OF2JZipQEHo4UUPgh/jrAC71jaZxqsnZSllL7Ya0nvrdBuDimRIdv/S9nG3g7d8n39AF0Akg1+Hihd4/nHToEFDaPkiTdkUUEMlRZALtgQUzcgr61lOlExHkDw9ZsKPtwrkmIl90CYCSTl3lib/j5+Pui77AhMhbOAThsRPjDhFMHTzM/UHjj39LB3Dx7rDyknfaZAb322iY/IgCsP3228fk0BUBW6VOTbc1BEDxySmiULz0pS/tt0OVjWeJ6C29Qdc5DiujmsZopdnUx3R/2Pxc7mFb5hyCTWAP8hzW9i+VUapf7tik1w+S3U3+ksXZxthqq63Cq1/96sjGNPoeFx3k2Kbnvf71rw9sVZejnJ0/qF0rnXF/S6WbOw5rD3TpKzkAF3m7HPGyIDPQjeAnW/FByCfGtS48Uxt3+7RLOdv6NDIO0JmAzIx7pE9ELvRwxhLsF4GkvT5N55NqL035cR0QPfq77BY9y8Idxj78BJADuPjd1P+51+YnZvEz9iV2ihP8w1465phjpixCgL/onh6BjfeYsKRMbNcMrx3ANejbkr+ia+Z0LexZxhAmVVLA8bBjh49FuT5PXZrsHepF3d3eozy0M9ncC1OHpOyyWfDhoBs0EYAt/PwQ/t+5D0dLyvGfejfpup7+uNJUHdD3AMgNS21tYhQfRls5htURJHNJM7fdY7p1sLZSpC/NBH+2A7jw1bNgTvap+ITcwd9w0UUX6VI8tunrw/phSu0mJmWxC7Ctc4S8JTAC8xdOJXJW413OHi+dB6JMw7Y51WPcdSDdNp9CaTlV3vQ4CX4qTddJlO+wukWXMcZ9YO4baxpjBChQmepx8hxwHwkLJ3L+PEqBTs48ZqqzqYSuC+lal2O6i0T6Tg7wiQ+I9kQbbCJfOKhnBJjR76YjQEeiP4rYQYb+kfp/dR8MBnO6yFOndIGI3+Mc/fOLX/xiDP6R3ivhS5rGJH7j60NfoT3kiHlu5tbT+ZPcs7pWmuZ0v+c6vsqeO3qk+Nx9rqG/gVMQ5XRS7rXZZ7QfsDHMSZWQ5G0FcA3gXimAC0MK5OYgApjCZMpMJzo9KyxoOKx6csALnREDiwkFADuAOpg06ArugE8IDW2/hDMMpxjEpIa2QnQegT7GqQ6Qi0aMkUeo9DZqq0Pbe4PulZRlUJrcR4kn2tN6660XAWYAkxhsm2jU79CU7jDXZfzwPTB0MQBwSuKIAZyHsdQ2QDDQ0pYwtvmuTOamEyK58qBY8g5GGXk76Irn6cc483DE4DxgkAclzvMLm+DR05/+9NiWQcWjwGBMiVB66HsoGcgL73t6ZlzHmVSWcdWppjOYA8gO2iCrFZiEQ5ZqxR1v03+QQxgBOE69f41TriLzcL4g23F+IAMAstHuXTEfXKP6xHRyALkKGIJIcshXvhkrDUoV1Lay44hD1vMHcJxxEWNx1GiAg8aQtjJNx73SMQxnKw5Mxla2q2JMZUKrlEblE+VAztDHmWzgu1166aVZY7i0jP4eDjom7nAMkhc6yCCdQgZf0wS8p+/nM2n8ZDIL/RG9HLlNnc8777yB/aT0PQGC0Pulyztvxn1eYjeUlqFUxy99r7Scw75H/+N7CyyO7i2HhG/xonRxJCPfc4td9EzTEV7QRlh5SVtkMpExYhRZ1JQX1yfdF9F7cOaglyNTcmCjtvJ1vTfd8nLS+TnwYr/99ouygokqouOge+I7YJxqo1IZ1ZZm273pzm9Y25+yj1vWjFOvp3zVxoAL7bTiiitGGYmtBeiEfkB/SIGR7akseHfc33LBHKZemS57gDaPvQhgh/EE29R9Oug+2JGMN9gJC5vQnVm0io4kPw7AJ/pGGk2qS1kn1V7a8kY2obvTRhm7ARFQ9kE0in+S9ssW69hA+BHpF21+RL45/j7sUXym9KFJ6RmD6j3dYwd2DpEtAAO5D2dQOUvvD6tDluazOL03KR/GIB6NW0fI5TdpvTuXZ3rNAVw/+MEP4qJ+5iCYi8AWZiwAICkZnL7f9rvED1NqNzGuPfOZz4yLX5Cf119/fRw7mCNgEjhHo8jZXHqjXitpcwujDiXlHJU343x/3LpFW9mme4xpK8uSeE/+SdW9DcAl8B/P4rthfhQAFd8QXQ5qW9wcH0j+sU05cglCDvE+OiD+Iwedsngb3zyETgKgXHniN0E3Y64Te8G34Aaciq9SxDa9yELywi5qIoDcmgcgTcD3Iq6TH+MTuisyBkojLfrWpNzHdkCPYvygDPhHIOwMFiy4XlnCl5jYNPxzEBx8ZK4ZPAffDNkBoVMTzS0FtDUVrzTN6X6PRTJEhIfcPkzrha6QLib1Z2ijtGG1He7JX+rPpdFwaUP4WMGrKNgBugcYiTZMh6fp5xXA5dxoOUdZG5ZQzhEk6ui8z3ZYrGRgxSrCV4RAbVuJoucWlyPC4o1vfGOsDiu8CduXkgtrIkg0Karpe/X3wudACuBa+CWqJagcqByoHKgcqByoHKgc6MYBLcBII+J0e7s+NW4OVLthPBzF4YQDjUgxRx555AKJvu51rwtEwoUOOeSQKQBcrbonIucRRxyxwLv1QuVAFw7kAFxd3qvPVA5UDlQOLEocIBoii++YdCIKkk/2UA8mu4ioBOUi3cQb9V/lwAziwCg65AyqRi3KYsyBHIBrYVW32k0Li/OLd75Vt1i8v29aOyIbsgAOEIlAUHqmCcC1dW/bQaKFQ4ByCIiixSAASViIp7TQT7sEXQEARaRY8A3MzaeRhNilQbslAU5hgQLk0QsBtHzqU5+aAqD1+5Rx3333je/xTzsfsdCXaI9dyCMfpzspsPABnAHgeojdMLSN7rvf/e7wpCc9KV4X+Df+6P0D6ASvBXhyH1kpX5T2JI8sgFB0WL4Z5fZFcm28aipXaZrT/R7lVxRdAGolUVPFA+eTruUAXIrGzDNpVC8H+QEMpG0PSxXA1ZFjJQAuQvaxpZsoXVmlrTJ0H0c6qwGWBELwKdwx4DV4ocZI/T3sHCutDjzwwCWBLYtNHSuAa7H5lLUilQOVA5UDlQOVA0sUB972trfFqA7ufFiiGDADK1vthvF8FJxmOJogtqjz7Up9O0kmnAEviohix7s47XDYEe2kUuVACQcqgKuEa/WdyoHKgUWNA759pW9JTD2YyAAMo+1k0u1fFrW61vIuGRwo1SGXDO7UWs4EDswUAFe1m2ZCa1g8y1B1i8XzuzbVCtAUkUVz1ATgcsBJzm/jWwXmti7M5eXRjH7605+GE044Ycpj+IgAgwEMc8AMkZ0ATkGAiXzeXwnwnvRhgrcoQqLmlU877bQAqGoQkbewA+Qj8JK/51HMzj333HD88cfH2wLfpCAyvcu2jPxBp556avjhD38Yz0v5El+e8D8HDXldlS2gQKKhQ+yeoHPdzx1L05zu9yg79aGOABRpYyXk3xcwpLZkTgFcRCfeY489YhYAFdmWMiUBEgHTffjDH26NCpa+y2/1nbqFYo47dq0EwOX7B5NUuhUFyGm2tBGdf/754dhjj9XPxf7IxACr0iAaMI2csLYMTkID00EI/TgdW7Es9gyfxgpqoCVMJ+EBK1UOVA5UDlQOVA5UDlQOLAoc2GWXXeJWLWyfWGnmcKDaDaN/i1e84hUBp50I+4qoINheWo2ITXb44YfHbU31HAA67Nof/ehHS8xiI9W9HsfLgQrgGi8/a2qVA5UDM5MDrOQnwpZ2Y2BSaP78+XHrdyb3RWeeeWY48cQT9bMeKwdmLAdKdcgZW6FasMWOAzMFwFXtpsWuac2YClXdYsZ8imkpyI477hi3YFNmRKWXzyYH4ELnJPoVR3w873//+/Vq/8hiPgDZUBNgqf/wwyeAoVh8gJ+IBQh33XVX+kj2N1GumOdv23XMg9tQrhtuuCGwRTNANAi/1G9+85ts+n7Rt09ki++vfe1rfjue4/MCFAfdcsstMbIW/Pzc5z4XrzVtK7nTTjuFF77whfEZwFuAuKBSvsSXJ/zPgXyAh3Jbawu4RlE+8pGPDPyupWlO93vUh2/Kt73gggvCMcccw6WhyCMls+iVcZ3+B6UALg9CdPrpp8fd9+KD9g+bUFuNfvvb3w4/+9nP7O7g0wrgGsyj+EQJgIsQbY997GP7ORx88MFTwtW9+MUvDjvssEP/fpOg6D+wmJ0gxBlM2HM6R4C3WI2GUK20aHGgArgWre9VS1s5UDlQOVA5UDlQOVA5MJM5UO2G8XwdtrDffPPN+5PKnipOOZxdF198sV+u55UDY+NABXCNjZU1ocqByoEZzoEtt9wyvPrVr+4vTk2Lm4tikD5Tf1cOzCQOVB1yJn2NWpaUAzMFwJWWq/6uHBgnB6puMU5uLlppfeADH+gDunIALgcxte1mpWhARLsi6tUgAhQ2e/bsPuiJiEMbbbRRWHPNNcONN94YLr/88rgdeJoOaRO1iB3JvvGNb6S3Y5qAvAjigh+K5zl6ZHgAXJtsskkMAENa1157bVxoeM4550xJz99p2prco/rfc8894UMf+lBMA75Rhuuvv36BADIsugDfoSj28IKdGqBSvsSXJ/xP8/JpZH3PVtsMco3n+U5tVJrmdL8HgJE8oZ/85Cfx26611loxGhzBg6688sq4zSG8yRHtjChaAMAAK7L1JttsNgG4HP/De+SR0jbbbBMAZELz5s0L3/nOd9JHWn9XAFcre/5+swTAlYY6FJJUqW6xxRbhda97nX5GNGTJPpj9BBbRkyc+8YkBXgjsBpDtkksuyaJDF9EqLnHFZuBkYGRgRTBWqhyoHKgcqByoHKgcqByoHKgcGJUD1W4YlYMhRpjbeuut4yKaZZddNjprLr300rjQCKdZpcqBSXEAe5/V49B5550XV+ROKq+abuVA5UDlwMLmABNCL3jBCwK6y8orrxwnhy677LI4+cTkUaXKgUWNA8yNVB1yUftqS0Z5mXR9+tOfHivLdu+5SdQlgxO1los7B6pusbh/4Xz9BgG4nvWsZwWiAUFErQL8lCPf2jCNJpQ+76AnQGGAWlZfffX0sRjNi0hH6Lhd6R3veEcEgvH8bbfd1o+6BdAFwEsbEezlsMMO62MH0LEBtUF/bNgNykG+beC1ZzzjGREwhu4O8Ic6p+lOki8xsxH/aTcsttwDgJSjN7/5zWGzzTaLt44++uhw4YUX5h7rXytNc7rfW2ONNeI2hf2CZ06wwb75zW9m60yfAKyFX/Sggw6K7UnXSCrtM47/Se8pa/gMv6GmCHHxZsO/CuBqYEx6uQTApXBtSgtEHnvBitg+kW0URV33HNXz9Vg5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQObD4cGAQgMt3+jr33HPD8ccfn638vvvuG1ZbbbV4j20ABQ7JPcyuWWyvlxJbNAKCIkq+CMALYJZB4FmiWu25555h7bXX1qvhP//zP8MVV1wRf3Nvww037N8jXQBe5OXbkAPCofzs4AUdeOCB/WheBNEhOpiTA9dI833ve5/f7p87IEcXHWDGtUnwRXmNegTDQiQoiG+h8zRdogRvtdVW8fL3v//9GJUqfUa/S9OkHSr/YcpS+t6Pf/zjGCTIAybxrdkulDZLPbS9PXXzdsdvtspky0zII2W1AbiE/2lrUwDCBO4i0A3AsGFIfRRAHgBe6kBUPP441++uac7qRVJaLJftlgC4tNermIeApMGIHPnJtVQY6Ll6rByoHKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgcWPw5MAjAtcMOOwRAXBBbx33ve9/LMuWjH/1oP4rWfvvtt8C2gf7SBhtsEPbaa6/+JUAwbId4wQUXxGsrrrhioFwcoRtuuCEAnmqil7zkJeFlL3tZBKHwDKCXk08+OZx66qn9Vxxglm6HSCTvPfbYox8Vi128DjnkkPguwBsAOBDgMt5lW8BHP/rR4XnPe94U8FdbBC74s+qqq/bLGBPs/SPwDoAfgEjj5ovyGMeRCGnUAfrrX/8amnZ722WXXcLzn//8+NwPfvCDcNppp8Xz3L/SNIk2VVKW0veogwPTrrvuuvD5z38+aLvElVZaKW6HqF3g2CJRAEWu0fYAQ910000BwJ+oDcBF+nPmzIkAsfe///16ZcqRSG7arrQpQtyUF5IfFcCVMKTpZwmAC4GlPVJJN90H00Mbcp9G9elPf5rTSpUDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA0sYBwYBuAApvepVr4pc+fnPfx7+67/+K8uh/fffPwBkgRQVKPtg7+JTnvKU8M53vrN/+2tf+1rcAq5/oXcCQOpf/uVf4qWmKERPfvKTw+677x5WWGGF/qvsUvbVr361H3lLN9i6kAhX3L/00kt1uX9cZ511wnvf+974G2DOBz/4wf49B6f1Lz58QtkgADpE7frwhz/88J38ged6gYrCzjvvHNiiF7rkkkvCEUccMTa+5HMe7SpbPhIVCmoD1PE9nva0p8XnjjvuuHDeeefF89y/0jR/9atfFZWl9D3qQHskuhsR2375y18uUB3qQpQ1bY0JXgfAFv0CICLgPu5zTdQG4ALLs+yyy0YwYlNUt0022SS89a1vjcmpDSntLscK4OrCpd4zJQAuwvg95jGP6efAnp+g7EQelo1rf/jDH8LBBx+s2/VYOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDmwBHFgEICLbQfZfhACAHPkkUdmudMFcKIXAVwBZoFSsJSe4XjAAQf0I1ylAWzYzg4glIgoXmeccUZg275S8vyIoEQkJdHLX/7y8NznPjeCagBhkR/Rs4455piwzz77RADXMLugge3453/+5/je/PnzY0SpcfBF5Z3E8T/+4z8G1vNd73pXAFgHpVsJ5spUmuZ0v5cre3qNqHJEUYO+853vREDf61//+vgbcN+f/vSneK5/AMIE+CLqG/S73/0ubjsJeBHQGNQEiCTSGRHPoDZwZXwg868CuDJMyV0qAXCBuvP9XAldSAhD0Zve9Kaw+eab62f43//93/D1r3+9/7ueVA5UDlQOVA5UDlQOVA5UDlQOVA5UDlQOVA5UDoyPA7NiUg/2HFu9VZi987ges/fvwd6Fhxdnji+zmlLlwCLKgdmzHgzP3vj+sN3mc8IyD9zLkuVw9wOzw7V3zAkXXn5fuPj3D4Y758/u9ZmHetQiWs1a7MqByoHKgcqByoHKgcqByoHKgcqByoEZy4FBAC6ARgSTgdoiL33hC18Is2fP7hSFirQEwLn99tsjkIlrKb373e8ObG8IeQCbt73tbeEf/uEf+o9fdtll4aijjop59y8WnHjQnBTA5cmxMxqRvCDfBlBRkNjOccstt4z3tT1i/JH8A5QmbIjyG4UvSfJj//nv//7vMQJV21aRvlXlxz72sT6fmgpTmuZ0v9dUfr/uoMLvfve7EZz42te+1h8ZeA7Ii77EtolEjYMOOuigcOWVVy7wrm/reOKJJ4YzzzxzgWfaLlQAVxt37J46qV0aePqa17wmIj71IMi8L33pS/oZ6Bzac5OLIP7mzZvXv19PKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgcqB0bnAICURy51f1jzUXeFDVa9I6z5mHvC8svPCvfd82C44eZZ4XfXLx8uv6Xn6Lt3Trj/wdmjZ1hTqBxYhDmw0doPhNc/59bw88uXDZdeu3RYatac8PiV7w8bP/6OsOHqc8L82Q+G0y6dE86+aOlwZ2/hcwVyLcIfuxa9cqByoHKgcqByoHKgcqByoHKgcmBGcmAQgItCH3jggWGppZaK28DxPAAep8022yy8+c1vjpd++9vfhi9/+ct+O3vOdnxEHyKSFWCVHDnGgWd4lihYYCMgysF2ib/5zW9yr/evUT7ANdDpp58eTjvttP49PxEoSGUi0hYRpQCmARLLvbfbbruFZz/72TEZQGQXXXRRALTznOc8J1777//+73D22Wd7Nv3zT33qUz2f0fJTtskr5Us/0QmesD3kGmusEXPIRdfie372s5+NUbruvPPOAJhrEJWmOZ3vsW3iv/7rv8Z6sQteU/v+p3/6p7DWWmvFKtOW2Dpx1113bWQBuCD6FXTLLbfEI9t7sk3pTjvtFNhlD6L90I5SokzkAcFreD4MVQBXR26VALhWXnnl8IlPfKKfA3utIkivueaasOmmm4a3vOUtU+4hWBE8lSoHKgcqByoHKgcqByoHKgcqByoHKgcqByoHKgdG5wDxgR71iHvD0x97U3jZZjeFjZ+xSnjE2k8NYYV1Q5izcu/u3SHMv7a3XPWycM1FV4TTz18m/OSK1cJ18x8ZHqjRhUb/ADWFRZIDuz337nDd/y0VfvLrOQuUf81Hzw7bbTonPOcpd4W/3DM/fPcXS4ff/GHpcPe9NRrXAsyqFyoHKgcqByoHKgcqByoHKgcqByoHKgcKOdAFwOVbw11wwQVx20DP7oMf/GB4/OMfHy+xxSJbLQ6ivffeO6y//vrxsZNPPjmccsopU17xyFbaYpAHvLyHHnpo+H//7/9NeS/3w6OINW1zuM0224Qdd9wxvu6RxhQRKwdIAtjzmc98JoJwwF7AB0BlALoAdkF/+ctfAqCslJ7ylKeEd77znfGyp13KlzT9SfwGlKaIUgCOHJ9Cfq94xSvCi170oph11x3hStOc7vcE7gOH89GPfnTK9ppUmGBKgKgA/bVFKIvMefifbzeZbpMIsA+AH0R6bKlItDoR0bkEfGQrz/3220+3Oh8rgKsjq0oAXCTNR1lllVWm5MKesQgOJ1CBhBisVDlQObBoc+BVr3pVWGaZZSLim0FwYRJA0Y022igW4Rvf+MbCLErNewAHtttuu9hurrjiivDrX/86Po3i+rznPS+e/+IXvwjXXtubWJwQPetZzwprrrlmQEH2rX5z5ZpQEfrJzpkzJ4awXWeddcKqq64a9yonNOlZZ50V7rnnnv5z9WTx4QAKrbaUxiCkDbzkJS+JFTz33HPDX//6135lt91227DccsvF8L65VTX9B+1E7ftvf/vbIhXpdCbK8B122CGuHGG11oUXXmhcHv50JtWPPemf8IQnBNpI6pAYvmZL9hs4hFiBhCMHo/jzn/98nyGsqKOvM77Rr4855pj+vcXhZHGsX04PGKSfIL/XW2+9sNJKKwVk+LAhwtO2QDj+jTfeOF4+77zzwp///Of0kdbfvU0Rw1or3B7+cbM/hee9pLfy7ak9h99KjDHr9f5W6v1hl7M6lZVw1/WwXOeFcO3/hD+fcWE4/MzVwwXXPSbcff9w0bi8zOecc0647rpeuh2Ilao4lW6++eao93R4pfER/Bc4OOmHv/zlL6NTsvHheqNyIMOBVz/z7nDeH5cJV92QufnwpXUft3R43bMfDBvPvT38/JoQjv3xI8Ktt7GtYvM7fge7GTkDnX/++XGx40tf+tKw7LLLBreL/J16XjmQ4wB6hyYjsGcnaTuT/6CxMFfGRfkaY8nLX/7yaKddfvnl4eKLL16Uq7PYlb3aMsN/0g022CA8/elPjy+O0186qb7SZmPlaj+p+uXyWpSuPe5xjwtbbLFFLPLPfvazcOONNy4SxceuQgZDZ5xxxhQf2aAK5Oy5Qe/U+8NxYEnTCYbjTn16HBxwQBSAHEUC8rTXXXfdsM8++/Qv/ehHP4q+GHAIb3rTmwJgJAigFSAWQC4iAaDuu+++CL7SdXwKimrE8/hLTz311AhWWXvttSO4CXsO8ihWAtIwj4NPaBCxlR0AGCJDKT38PocffnisK9G10PO333776N8gPY8uBU8IpgOR3/HHHx/PV1hhhQigET7jpz/9aTjhhBPiPaIqffrTn44Rxrjwhz/8IRx33HHRF0N+W2+9dZS7ir6EffG9730vvlvKl/jyhP9RdvivchP5jDoz54evjnkViO95wAEHxC03VaT9998/+vH4jS+XQERQaZrT/R7tn34A3XXXXTEKF7gbaMMNNwxvfetb+7gcbwvxgYZ/bQAuXgEohh0KMadAG8KPwfahRLzTd6CNM7c5LFUAV0eOlQK4mIBmH1h9qFx2CDKEEx+4UuXAks4BJv2Y1F1ttdWi45bBm/2K2UP22GOPjfvSduUR4BeFwuz6zsEHHzxl31+MO8IoEnqSEJOUBTAJDuYcQEsKDyE7DzvssK7ZTuQ5EOJSzlKE8EQy7CU6d+7cOKjfdNNNWWVyUvku6ukKwMsAjwIK0X4BBEK+xS6KrFZMoMzefXcvasSIpHCepEV4U1GuXLo3iSPtlT3Spax7HhgRhCdlEnJRIhwtGPMoxrm9sBeluqis426DrJ5hkhliVQwALRmd6QofydjUqFTZckcMMiYCWQXxz//8z7lHJn6tRDYuDBk+iBHif7oteNt7TXWfSfWTsc8YT/jvmUjj7neTqCMTBXvuuWffoUIe0j+8n3MdI/TjH/84p4sFLa71y+kBTfoJH9Idi/xmgpdw+aMQegHOD+gHP/hBNiR+U/qAt9Zf+dbw3hdcFdbbftMQ1npTCEv10iKq1oP3P/wa5z3n5SxAXL3z2QC6bgrh1h+GB875r/Cf310+/Piqx4V7HliqKZsFrnuZ3ZG5wIPJBY1XvnqVR7BHGBtZ9ZmCEpru+RYJOBodoJ9kW38uIRxoGo+bqv+Sje8LV/5tTvjD33H02UfnzJkdXrrZI8JuW8wPv7317vCNM5cOf/wz23dkH59yERtbtgdbZZx00knhC1/4QrQnr7rqqoDe0YWGrVuXNOszM5MDTTLPxybAGJO2GT0/t9VnJtdGLxWTf0wGQURRIJqCqPY/cWLqcTr5sijYMuJOUx/W/ek6MpmtRWSyV8a9lFNDAABAAElEQVSRd1tfKU2/zcZqSnNS9WvKj+sz5du2ldFl93SMFW1lGeYethD2BfStb30rsECkK+Xsua7v1ue6ccDb1ZKgE3TjSn1qnBxwP0sTgIv8AHpqQXQuf+YmAEWxBZyT5IS2JfR7u+yySwAo7kQ6AJZFLNAE3wA9+tGPjpGIdK/LEXuQOaknP/nJERTmaTM3DBDIKfUzsYAODIaIejBvwFyu0gL09slPfnLKDmjed/VuWjeug9vgXadh+eLvTvo8x8c0z3nz5sX5Rr+u7SK5xk5yV199df92aZrT+R4+MwIq+bwi3xNSO+AcMBv9iLY1iAYBuIjChW/7kY98ZGNSLOjE11dCFcDVkWulAC6SZ6L9ve99bx/N6VkyUYQRnEPN+nP1vHJgSeDA7rvvHp72tKc1VhWw4xe/+MUpg0fjw70bPnnS9pzfA8gC+AhiNRZGrwt4f5boL0wkOWlye0kEcAFS0WRs1xCczrsl+VyKchcA18te9rLAH9Q15O0g3s4EABftB3CN+hvKNgoVihAKN4TSxaSOK5CD6raw72tvbcr+vve9b2EXZyz5j7sNsoqGEL4QbYBvzgoGKHV+SMYuSgCuUtk4kwBO8WP0/on/XQFcbXWfSfVbFCY9xt3v9E3HeXS9iz4K2IQQ0hDODtlTrIRCjh9yyCHjzH6hprW41m8Y/YToiR5xDTuXyYXvf//7I30bb1fDALhwJ679qNvDh5//h7DeK57d+/HaEJZ6VAj39yJtzeqBsR5cpvc3p/cUzsCe4wYA16xepM8He8D4Wb3rs3sOmFvOD+HnR4XPfnulcOaf1wj3d9xO0cs8DgCXVrDmwq433asArpGa3WL3ctt43FTZp63zYA8CuVS46MrBjk3SeOoTHwz77nhfuH2pWeErp8wOv758ds9x3pT6Q9dZFf1v//Zv8QfgLUBcyBHkSQoSaUqppG5NadXrM58DTTLPJ2CmY1Le80vtlZnPxeFL2ARKqf0vz8vp5suiYMuIU019WPen6zgpgFNTXxmlXq5XpjZWU7qTql9TflyfKd+2rYwuu6djrGgryzD3KoBrGG5N/7PerpYEnWD6OVxzZD6BiFcQY34bloBgAFtttVV/fkXcY/wgCn5u60T5fZp87exsQYCLXJAaj3hFXi94wQvCK1/5SmXb6SgAFw8j78gLOzEl5lb+53/+J0YWS+9tueWWMRACdmRKBEAgYEIuCALzv294wxuydSM/osADnGWeKqVh+JK+O+nf8NEjQCk/6gR4S5HIdJ0jdrn4zgL7dPFgSZqkO53vEdGeeiv4Bfk70f6//vWvdwJv8d4gABfPEP3tQx/6UFx0yW8nIoCBZ8i1H3+u6bwCuJo4k1zXhENyeaifoE9prEQWQmiAFNUHGCqh+nDlwGLIgTe+8Y3hGc94RqwZA8n1118fwzTS90DqCtQxTMSG17/+9eGZz3zmQG4pbR4k8gYTTjhcHEzCNSJv0Y/pw3onXdGuye0K4PrfOBgOZH59IHJAirIDuAi/qVUTZ555Zj9sqU/if+1rX+tvuTgKK2cCgAtlnVX4UBrG1J1PhDjXRM8odZ6udxd3ANc42iBGFsYZBL+IlqU9xI8++ugpW/VJxjYZlbnviuGITEf3IqLXdFOpA38mAZzEM/G/ArjEkek7TkL2j7v02joePQ7jlW3jRWo7AHMF6tK9xeG4uNZvGP3kqU99athjjz3i58QpAsh8HOSTVsMAuB619L3hfU/7bdhq59VD2KA3xiy9bA/ANT880NsO8cG7Z/fwWcv2cFw9ENf9PajXUqzKuz88MB9w171h9iN6vyNmpQf4uvaccPtPvh8+cNKG4crbH9UDtAwmL/MwAC70PvQ/Vnj6eNU2IdZ0rwK4Bn+nJemJEl1kjVVmhSc+dnY47/f3xyB1Xfi14dx7w97bzA4rr/JA+OpPZoWfX7xUuPe+v6/OTtPApkZ+QkTa/fnPf97fOoNt5dEzB1FJ3QalWe/PXA40yTyfPJ2OSfkmW33mcm60ktFXX/e618XJLbZSJyI9VPtfnq/TzZcK4Mp/h7ar7mMaZwSupr7SVpZB99psrKZ3J1W/pvy43iSf296Z7ntrrbVWf/soQAA33NCyT/R0F64lP+YTsS+gGoGrhVEL6daSphMsJDbXbIfgABGrNtpoo4hFIMoQkY1/8YtfxIXxQyQz5VHAW+ycwlZ0nLPlIL5hFm5OgoiqxXZ4q666agQSsYU3C0HboiYRAAA/yBOf+MS4FSDzy9iUzOm2Ee+tt956YZ3ebmrM+xLcg7xYUMSccBtNN1/aypK7x7jHnDuL5YmUhr096jcrTXM63yMq6Prrrx/WXHPNGCCCOVf+ciC+HN9KrrFVJ7wGPMb8JfZSCoIbNl3hh9hVhraGnkn/5o9z/e6a7qzePtJdfJpd05sxz40DwDVjKlMLUjkwwziA8GGlLQKHST/Q4BdeeGG/lGxBxmQfz0Fy7vYfGOGEqC9Ef4GIEPDjH/84nhN2E0UBYr9fjyjgEyKgZ9///vfH5/inCbwK4KoArn6j6HCSmyBtem0Sk/gzAcClvsMqEhyQTijShOJFRqR9zp+biecVwDX4q7gzSs5T9Ykvf/nLgUkCkdrJMAAuvbuwjqUO/Argmr4vtihMekxC9o+bwxpLAG598IMfnJK8+jSOEyIpLm60uNZP9XKAedO3I4ou0XShH/3oR/Gv6dlhrjsYqiuAqwfRCs9d/frw8Z2vDmHzbcODK/YA4sveGx68Z6lw9yW9qEB/fUSYvdodYZlNHxXmLL9c7+n7w91/uDncd8kDYdYyc8IjNr0/zFm9BzqZf18vKFfP/rh8Xjj9uzeE/+/iDcLdPQDYIPIyDwPgakq3bUKs6Z7bK+mCk6Z86vXFlwMlusicXtPfaK3Z4Y/X9XYUvbNbFC44+MQ1HwzvfenssNbqd4Yvn7pU+OmvHhHuW3DBcp/Z0u0UWVhbN6SrufsvJCcldUuSqD8XIQ40ybzpBnAtQiybaFFr/8uzd7r5sijYMuJUUx/W/ek6LgyAU2nd2myspjQXRv1myrdt4smifN19ZhXAtSh/yVr2yoHKgcqByoHKge4cqACujryqAK6OjKqPVQ4UcMAnfJomiHbYYYfw4he/OKY+rlX9oGHZQxpQyO9///v+Vj78ZnKRI6huokgAFnBiW1RQ2ZCczZzLAQ2A69vf/nbYfvvtw9y5cyPK+eabb45IX0JUloZNJA8mcTfeeOMYDQxQGwhc8jvxxBOnlDOd/Ad5DwoYRDlAOaLR4BjPhU4lH+QeADfqCUobXoA6h1dEBXAEOqsxQRmDbIa4BzL+oosumgLGizfrvwU4kJsgBSGuPcaZCKUNvva1r41RqggHCoGcZ8XYD3/4wymIblDY9Bei1/Esk+l8N5Df9LGU5BAChU4kLFGuXM9+9rPjKg7Q+t/85jdj+8ZhTl6Ar375y1/GdkWeOC6JJkEbIqoSqHO212RLJSdQ8R/5yEfipXQfcz2nMvIb0OQofUhpTvK4wQYbhOc85zlxdYr2wWbVB3y54IILQg/wHrP/2c9+Fld0eFn+8R//MW4dw7MAWp34ntpuEMCpryIZ9rsr3a7vEUa3axtU2l2OrKjZZ599olzSNpOSpYBnAdGKdH0YABdtkLZIX2FrHghQICGSkfN8F60iVz4ct9122/CEJzwhRoRMtwAjPO5rXvOa2B+Rj2wJx8oe0kGuikaRjdMpw1Vejsj+5z73ubHtcg7/6bf0TY2NXSJwDar7qPWjrIxpm266aT80Miuk+J6AwBnnupJPegAY32mnnaJMI+omxhLjJWMs43gTlZQF2YksoD3RD5Gh5PXd7363P8aOq9+h86CTcFxuueUC21IT5YeVx7lxgXpus802/ejBhCEnehYyB0AKbV60+eabR35xZNsQeE8/YJwgH+onmcd76Aakw8ovpxIe8n7pe563n1OHnXfeOY5vtAEIXqFrpVtno5N1rZ/nwfm4x1P6HN8WfY2Q4Lk+oLGJ/HPjD9edcnpAqp+g8xGmn5WRWvxw3XXXhSuvvDLKRFZmirqON3qeo4OhugK4ll3qvvAvm18cNn/JquHe5TYJD965TJj1qLvDPX+aH2Zf8cyw1L2PDvOvPivM2v7OsOL2a4X5V9wcbj/8xrDc3ZuFpR/7uHDvaheFpTb7vzDrrkeFnjUQ5ix3fXjgovPDHt97arjmjuW9eNlzL/MwAC6NV3/5y19iNNKtt946rjpl0kZ2ySWXXBL1KfoX42fuHnphFwDXMH1n3O01yzi7OMn8hunjFGmSZSH9ku/AqmDGJVZAI19ZZYxdyJiNLsK4LRo0Huu53HH1lXurO3s3/npLdwAX6aw7d+nwrhfPCms+5sbwHyc9Mpx3GWNDPhIXugWygW11sVc0Jp/Zi0BMHduoa92G/eZtefq9rt8O3V32G2NJagvx7YgeTj9Hx0XWMe4iEyB4gV2AfoYfAN0TOcuWHmlaPA+YljSxOVgIgc47t+eTYCUw24c4da2Dv9NFf/Hnu+rN/k563iYPkXkpgAt9iii/+CcYG+HXpZde2l+wl6bPb3iE/cbqcL2DzEWncb2HZ9OxUFvZbLfddtE+IP9TTjkl4Mdia5ZlllkmRnvn3S7UtSxs3UKbgCQX0vRf/vKXxzGatqU2oWe66of0UexACD2OdtW1//HOMO1sSZa58AoaxhfH85Kb6GQltsywMhL/zgtf+EKyjoB99JaUABCRrmztQX04fT/3G7mGXMT+W3HFFaPMRB/CL4bcHCbKQQpwQobQV4mkg7xFZvzkJz+ZspiMMuG7wqaCAMgjq+kLyHhkBX6DtK/Eh+1f1/7dZmORdxuV1m9hfVvq0pUvPIveQ1vAL4hfknGGNkn5kd/4fpBTp556Ko/3yWX3GWecEb9z/2bvhO+IzEQ20sZoC6SF3YT8FHHv1a9+dfyJHyj1F+k5lQm7jMXo7kenvbHVGL50+sr8+fOjPwCf1W9+8xslEY8pgAu/BO/ir0I+U0Z0v7POOmvKe/zI2XP+0DDy2d9rO++aZukYhu0Jz9Cp8E2hCzO3xPeDN1xja+6cPUy5h7VJB43t3q7w30snEI+GzQ9/FD5Mtt5CDiGfaAPIH/z2+FLQDfF/5ajr2O7vEjyBAAf0Q/oTvh/6EFGb0B8rVQ5UDlQOVA5MPwcqgKsjzzGeKlUOVA5MhgMowi996Utj4gAScobPJptsEt761rfGZ9KIWCWlQnn+5Cc/GZ2fKL9snYhDC8IRvueee8ZzHG+sIkrJ93MmnOahhx4aHxG4gIk+FF4cDClhlOGgdlBC+kzuN5O4gBtQ4nOEYcnWcrfeemu87ZPjGH8AuHLERHFq5AHG4X2cyDnCCDrssMP6zowDDzwwW1cm7w466KBcEvWacSBnULsT+jvf+U4ExAEmzBF9RtHjMFgBGGrP6vR5jHomdp0EjuoC4PJ2BVgLgzslnGhsebL33ntn2wWOEkADIoxQJhYgwIE4151witEPaY9pGf25mXS+4447RvBDrkyAO+XscfAoz7KnvUBM/KZvuVGOo4K2AX3lK1/p86rku5PGMO/hlOjSBkl3GKIMyGCPrqbJPJzR7vyQjB0GwPXpT386AggBu7I1LoSTjK0IIOTj4YcfHs/9H85vwBsAULzeTOwA3mqSj4wJRA4DRDGKbPS+NmkZrnrjCEV+0OdSwjFOX6XeXQBcg+o+Sv0A4PH+3J5zKUc4mAD/pZNtuWe5RjtjzOZbI2Nyej/fE6crDiynkrKgG9DmBQ7y9DinfR9xxBFxjB1Hv8NxjExqarPUCce3CKf3e97znv62trquI32Vlb+MAZCD2vWMjvCTScuUMEI//vGPx8slPOTF0vfSsvhv+sC73vWuIOCt3+McOUL/lg5HdMhB9UvT0G/vA+MYT2lTyFPouOOOixP7yktH367YFyDofnrsop+gVzRtbeyg7GHGGy+Hg6G6ALjQXNdb4f/C57e9JDxijbXDnVduER657GbhgSvPDXdf/oewwopPCkvdc1+4569/DvNfeGdY4UNrhFtP/1u4/z/vCasss2aY85hHhztuvSbc/5hZYZmnbRfum3V3mLXCqWG5ZX8bDjt5pfDdPz6xB+nK68cqt5d5GACXxitshY9+9KMRVK/tpZU2RwEwm+4xZrUBuEr6zrjbq9cndz6p/Ibt45RtUmUZ5Tsgh5lMxCbNkfeVQeNx7n1dm91r6sssPSvMv6c7KFrvPmnd1cK7X3xjr//9LRz0vZXCb//Yi42XSYbxGr0DeYquQdtnAh1QTjrxqrR17FK3km+u9JuOw347JpmZEIawo4ky5pNhb37zm2Of5T7jDOMjoBxkCYQs5XduHAeEjS9Cvgyel+xmoRbAXY1TTBwzVkDD1oF3htFfeB4aRm9+6I38fx+//Anp6W47M+FIvXPEZC9+jJTYcr2pP5EHwDeN/bzr+WGrz5s3LyYpu5oJZCY/se1EijSs303HYcrC5D2Lm9Q2sDORDSIAAiwOgmh7jNf4qqBh9EMmyuUXk/+rS/8raWdLsszluwzri+OdUWyZEhmJ/xY/LtS0ZansdoAG2PaD+nBMrOUfujn9izbVRAAdBgGb9K4DnPDJwPccAco6/vjj+7dcvwKoAS/U//BjoV+nfaX/cu9kmP7dZmMNkiel9VsY3xb+DMMXnncZzHdg8R1+/pTSReL+Xtp2B41XbEH2pS99KfoYyAv9hSOyFd0l53uQnOQZ+gDjCUS0akA2TZTOeziAiz4F0DhHzGF88YtfnFIW6QQpL0rkcy5PvzZsmqVjmOQLfGK7s5xcwN/AfITrW5S1xCYdNLZ7u3KdoDQ/2YSAA2lXue+Nf+rggw9eYDHeMGO7vh1yDZmRm8Oi7aJbnH322Xq8HisHKgcqByoHpokDFcDVkdG5iZyOr9bHKgcqBwZwgJWkijR09NFHx+gT6Su77LJL/xlWvaRRadLnB/1mVSqrM6DUaGO1FIYbxKpJACUpseoVxyvEyiyUa0hGRPzR+4dzHUc0Bj0T31KGAbjsu+++eqzTUQYDDytdJniYwMahBuGMA5gGueMrXuj9A9yFMxGDhTqIcPgIJIGDlzRkAGEwYAQy0Uwd5AAGMKYtkoiewwokoi9ATNiSHs7mHCBP+dbjQxzIGdSpAcjq37322isCs/Tt+J4Yc/AYXnOdbyfHBfdof4xhAP/kWEqBdWpbKTgqV65cu2K1L+0Bw1l56NtiVNIuWc2jNsU9gDRMUHQhJtPlUEsBT13eXxjPsOKMyRr4LjAMsgJiAgDnObzSJK3K6LKOaz/96U8DUftEmqCHr2zPCJV+92HfA/Q0qA2qnMMcAYsccMABU4BSOD2Ra0RJ9AkpydiFBeCiL9Ff+HY4MmjbAOwATBLdUH0P5waT9qPIxlxfm5QM53sh45kwVB9mnMHZRfQ85LtTFwDXoLqX1o9yCFzHOeVk1TfjKzJI/Q35x3NykvJsE2nSQ/f5tqxiZawGIKFxj+vIWGSeqKQsb3/72/vREpC7OGHJiwkUgbqoF45g+uko/c6jnFJm5C6OTvozE+QiQIxa6ev8QLbTxiknDlbZRPDic5/7XBxjiC6BUxndQk5sdAfkFG3W8+J7oIcwNh111FEx+xIe8mLpezHTzL9UJsIr6s6EEQ52jWHwgm9D/fg2g+qXySpeyvWBUcZTX1zAal2N4cqfviEwdFcZqjTc4Z/qJ0SAAXjssgInA20aYBqryFPedtVPKDtRaQQW7wKG6knnsN3j/xLe+eIeQOu6x4YHf/eisPKqq4UHLulF5rvx2jB7Tm9bxN7WiPfceUe4c4elwgofWyPcfPot4d7P3Boee/cKYc6jlg4PsE3i8quEpf5h03Df8muG2+4+Kay4+WXh/ItC2P+CzcL9DZGExOtxAbgAbT/lKU/p61e0OWQTfYhj0z1Ahj7BmG6hWNJ3xt1exaum4yTyS9thlz5O+SZRFtId13egHthd2GEOPmW8Qk8ZNB5TllKa3dOHVlx+6Z6udn+4/a4F90nc4Mkrhw9tf3W4+d454cBvrxD+cv2CzwgIiw6C7ACMwniYAlFyZRxUt9JvnsvLr5V8OwHTSMdtdiIXvvvd747J+9jqk7XKG9mNzgPo3HWzNEK5ZLfe05ExBj0WKqnDMPoL4/2werPKmTu2yUNkno9Nep/xmog86Om+CC5dwMZComc+85nxNb4BOgw2GnoSYxvEdSbFGQ8hz88na2VXx4fsn9tudnmB05KyuP2I/si3BayH/orPSroxC6hYSAUNqx/mAFyD+h/5lLSzJVnmlvji4LPr7vymvXaxZUplZAnIZ1Afptxt5IAX2jd9EX6x2MUXL37mM5/pFInLAU7KF7sF+cx46jLW9U/Xr/QeR3hOdDr8y00ArmH7d5uNxdjZRqX1Wxjfdli+UG+XweIDdis+N8Ye/I8iX/Dq7/lcgC8a5z38IIy32LeMBfLzuC/V/ZQOnFe+Po47IMt3GEFmE62YMRN7E7tS/hhA7IDZIU9L6TPGUUbq6/5eQMyMiyLpBG7Pca9EPivNpmNJmiVjmPyCKofGbr4T/Vc8BEhNmUSpzOtqkw4a271duU5Qmp8AXCo3R9ok7YT2qLklfvti02HHdtIlouFb3vIWTiNhU6AvpnIQoCR/lSoHKgcqByoHpo8DFcDVkdearOj4eH2scqByYIwcQGkE7CSDiQk/DONSYjUmE26QO0+V3jve8Y5+tCo36HRfRxlBbhC4EUHarDRTiGQcrThuNRGcM/CUdnoE5EbEFwgFHQMEQwNiQhEwiCYW5fx2xxfGDFGXnG+AP7SKww1DXyGcRuUgP4wDnCQQRowmsjEYyRvKRXmKN+q/LAfUltygbjIA2a6JP4goV4SkF/k3Tx35gO8Aw8iBi0EoQJGMURwArAoT5crledCumPRXxKzUqcAEI44ltVV3IrX1LeXPkch7OFMg2v7+++/f71Px4gz/p34GrzyyloBYFB/5xkQ35JM7/HaAKL8lYzw6oH+TYb576XttbZAylhAAPdqJoo0xccckJIA9J9W/K/iAd+X8YIJz1AhcXne2fyMikwjHMZOlOIv4ngLplspG/z7TIcMZFxWtgO/AakX4DLF9wxvf+Ma+I6wLgIv32upeWr+tttqqH8GO8YdVrRpnmVyiDwkE1dXJ5JMeOH4BWaotonvQjwEvQQ6MKS2LAIo4bNk+ljxFHoXDJ6+97aWyX+/mjoAjAUlCaZRTdx7jOGYFJ9vW7LbbbvF5DEUm/iTDuehy3CeCuaexxNs/1yGNJ2nbKeVh6XsPlSb/f4899ghPfepT483U8Y2TFLmttgVoiShXoqb66X7umPaBUcdTB2gxUQwIAtkhYrEEDnIIUDhR3gaR6tVFP3GHsU90kYfXdZhxine9nXrkSe7lqLcOPrxpvSvCq55zQ7jjuhXDXSevER533zKht+l4L3xML5xQD3z14G13h/+746Yw/x/nhMe+53HhpnNvCzd//Prw+BtXDsut3pvwWboH4JrV4939c8Jds5YPt/5DDzCx1U3hit/dG97/82eF+x7opdNC4wJwKQvJDCYWFT1y0D2fYHQAV2nf8W84HfrfJPIr7eOTKMu4vsNpp53Wn9yjTbge6TK/bTxWW2o7zuqNhb1OER58YCoA66lzVwwvfvpq4cLfXR9+9pvbskm8eItHhvdt95fw/YufGI486a5w191T05DOxzhE28L2ZwKTcdjHn2zivYttdSv95k15cb302wEGoP9q0o1FYnw/xlmBhPiNjwBK7SqfNOY+4xX1g+CbR1aS7OYeugbjFfa5xoTSOkgWddVfXHfpqjdT5jZSGVJ56LYz76d+DC+LT6TTfrAP0N/Re9F/pQOSDtGriGIFKZIQ556fT9ZKF+IZ+A1gCn0UO3YQlZaFdF2XJUoMNpODC7zOPD+sfpgDcJFOW/8rbWdLsswt9cX59x/GlimVkSUgH9oL1NSHH7qb/4/8pG9BgBiQm5JnXPPo513tP7dpSAO7Hjkl8jTxk2GzkafrVzyLPxj9lD4GtfWVEllDmpIrORuL+zkqrd90f9tSuecymPpjY2LbyK72yPXuO/P33CdJmwJsAznYld/oJehXGr+/+tWvxgW0vnsHgEnScHIZzDiM/QixCAr/PWMpvn35MriHj14LNX3MSXUCxnQWwKsfoBMwlyHgkgMZpRO4PVcqnyljE42SpsuwLmOY/IKUhf6JnS7fFd8Rv7F83yyIZWEs5OPLMDap+iBp5MZ2b1euE5TmJx8m+VEvIvtpsR1+R/RJZA3k33rYsZ33/Z3Uhsc3xBaxtKsULMa7lSoHKgcqByoHJsuBCuDqyN8K4OrIqPpY5cCYOcCK+1133bVvKLG6BGNnFAJkhJEIofRjHDh5mGq2IyBEfI60xZcrsTIiUOgxQHAsOvnKBlZ2oih3IRyyWlXGxDKGl5OHm5ax6YYCwC2Ab04OCnPA1c477xwd09QBowfQgxP3t95663iJyV6c7VCb8y4+UP81ciBnUDcZgO589kl8VpjxvSAGd4HpPFP/5t4mZIwOC+C68MILo9PA81Af4JqDxPi97rrrxigAnKeh6LmWkkfDw2gFDIZjZFGiJgCXOwTdSPYQ6xjJOHWYhIeItEG/hgQALf3upe+Rd1Mb5N6kSe2L9gAgsQvJ+TEOAJdP4uScw8hivhNOOOQjVCobp1OGA/DB4ag2B1CXtufkkStTEI4/5+dtdS+tHw4qwH1N46wiulEOB1h7udJzdxi6I1fPkR/tCP54BIXSsqgdoz/gbJSjmfwAe2sbZ7YBkaO3pN8RTYIVzVDOqQw4jUkUnNGS/4AQZfe4MzAm8vA/d3BTfjmdNZbkJhc0zqVtp5SHpe95PfwcByj6Jd+Y76EIo/4M7YB8IfFL95vqp/u5o/eBcY2nbKcyd+7cmB1OfdIVOQCfscYnqPVMelS93OHfpJ80AbhGGW8oj+su6AFEb2uj2T3g1VuedHnY4RnX9lY5rBSuPemBsMx5S4fHPGKZsOwqvYmZ5R4Z7vvbHeGGR1wflvqn5cNq2z4m3PHn+eGqT10bHnv20mHVtdYMs5aZE8Itd4S7bpsfblv9rjDnlQ+G5de+J1xx6b3hn85+di8CF2CWZprJAK7SvjOJ9trMwakTLOPoH6P08UnUfRzfwbcoFS+33HLLaDvz2+27tvFY7zYdkYtPWH2FsMm6K4Q/XdsD298wP9x6+11h3ccvHd72mi3DZk95Qjjyv+aF/z7j6mwSs+b0tj1+xQPhhZveEz7/3ceFc399Q3/CMfvCkBeb6jbKN28rQum3I02XZ+gxjIkCz6fjtE/Wovcy3jL2OPkEvUftlezm2ZzvoLQOw+ovJXqz1y933gT+8LEpp/+xiE4Raxz85RPsxx57bDj//PMXyFaTm3wzQBx8B8/PJ2ulC5HImWeeGU488cQF0mu6UFoW0gNgQBuhv0LUg4jQEO0HkAD6GVSiH9Kf4D2kLRQ5b+p/3CttZ0uyzC31xZXYMqPIyOkG+RCpUItwWeCSbmnPQhv5BlKwKG0xRy4/AUawkCIl/GvosRBylYU8DuCibwHscVuuqa+M0r8lV3I2Vlpm/S6t33R/21K+uAzO+T8BSGnuwGW+vye7n8Vq8AtKwa7i5xZbbBGBLPx2u0hgLB8f9I58e/hVaJ88AzEWYX+zoOaoxE/Pfb0HWJG5AMh1AvdrxZsP/3N/DVvp0lcg6QRe7lL5/HBW2cMoaQ4zhpG59BHOc3qO80uLYkexSdUHyS83tnu7kk4wSn7yYZJfTjfBXwSAEALcBRitZGz3CN45u4L0faEn7dUDA3C/UuVA5UDlQOXA5DhQAVwdeauJjI6P18cqByoHRuQAyjuRdxTpieRwxGHoaKIQA2r99ddfICeMWgAROVpnnXUCAC1I0SbS5zCsFGkDYwzQWI5kVOUAXG1AMzkdtXUaE4JE11KEMc/rnHPOiQ4yGSc+cezPYZxidEI4H+CVO74OO+ywcNlll/krcTtEHJBQGmliyoO9Hzg8CecMMIFVgVp5VAFcKafKfucM6pwBSOpNk/g4aImQA7mxHi88/M8d14RFxtEHyRhtmpB2Q9/bVQ7gKAeGA49iJr1/hB/HsQzRHmmXTeSryknr85///MBJ26a0Fub1JgCXO7vV/4hAILAWTklkHKSJdt9GSqCJ0u9e+h7laWqD3Js0SRYuLACXT4xSVwAQTJjhxHDHrfPBv7VPpvozuXPva5OW4e7gyk2UUz5f+ZyCcHLl51pb3UvqxzgJeBpqA0EL9IyT1CPfxRcz/zTpkZNbelxbOvGbSIV879KyOAAKucsk27x581q3+yjpdw6C1baeqo+OAGsJw49jm2iK6mPu6NazOrLVExMpkFYgc66xJDe5oHHO207p9yx9jzI2kQOM2/opeihjadq2cvVrykvXvQ+Mazz1hQI+dnt0rtz3UZnSo+rlaTXpJ00ArlHGG8rj+gB9LwUwpGUGwPWGuZeHl2/257DSE1YN8+98ZPjzWbeE+fPuCivduExYedZyYf7se8P8HUN4wjtXC8us2PuevXf+fMqN4bZ/vzms+X8rh1nLLxVuXeq2cO8G94cVX7xiePS6s8Pdt/wt/P7SWeGj5z97kY3ANUrfmUR7Tb+d/x53fqP08XGXZVzfQfqh840ogdoqRvol99vGY38/dw4cZIN1etss77VpmLXiuuHaa64P11z9x7D+hk8Kc5++Y7juqsvDkUccF37yyytzr8dry/WiIbz/1deEdXrjzSeOWyFc/Zeb+xOZjS91vNFUt1G+eVPWo3w7pYlesvbaa+tnPKJ/ABTAhhe5bqZoFLqnIxFsmfyGHFQj2Z0by0epw7D6S4nerLo1HeVLSevmYxPb9zIxnxL2JOOhj4MCrjf5WUjDI7RL7/H8NFnLs9KF0BOwAzVhz71BVFoWpbvtttuG7bffXj/7RyKxo9uISvTDJlBKU/8bpZ0t6TJX30nHLr64ElsG/x7bYELD6r7TDfIRL/xIG2OreyIW0e41b1MC4EK/zy0Q2G677QJ1hdTPHcCV41tTXxmlf0uuuOxyXuTOHcA1TP2m+9uW8sVl8FlnnRXYGjcl+Sadb/6eAFwuE32RrKdHVCfGQMjHHwdNecTbjTbaKI4dPO/jM79zhCzFH7h1b7G0FuM0AbhY5EVeKbk89jylE8ieG0U+p3nq9zjS7DqGkad8Fj4fo7Lo+P+zdx7wdlVV/t/pBUghCYSEEiD0CApDVyABFKQqoNhGLAM4jiKK+gdULDMyMxYEBwRBZUQkoKL0Guk1gpAgIBISWgIECGQgneR/vxvWY72dve89Z99zb97LWyufvHPuKbv8dl/rd9aWZ6T8m1mTShtMje26Xklf0Ux8QuBK6ae0bkg+CNb1uKjuR390RD2BpBoKjhXQRSP33Xefu+CCC8JH7LchYAgYAoZAixAwAldBYGUhUPBxe8wQMAQyEWDSj7cI3NTL14NMkDEo43IYxZoI3hFQOoSSmuDynH6Hr0Oee672dX4gWmEU+5JDHpfFgP76Ra6lvlzgXfmKi3yhwGVrOyFSSdhy5MsOtjcUT12xr0nl2fCo8yFkD/0M/RqLZUQbVPmNYhNPMmxdx0KVcomJEbhiqJS/Fi6oCSG2AOS6Xqhp5QLbIrE9kkhKWSztii1JpN7JYrQsgStWr8S4LYtkSQ/HMgQuvQVMjLyiw+3K5ykCF2kWrCClgqV40qMPw4OatHv5ol4Us/oLw9xyz32PdKfqIPdaLdLHrioCF+RVtkZEORYKijy+okShxha6IlqRFlPyynPhsZ19uFZOa8VjmCbBPxwzwufkd7285+RPbylAHI36OZ4pQviItS3e1aK9SDA3oA7SvkXKpIWvI1GCSX8sYRAmWyVA4MT4qCWn3WmiVWi802HLuVZMs33pmWeeKbc6HQ844ABP5uai3qJLxpJY/y/jnK47ueUJ4SwX+04ZUT8mTpzoDjnkEH8FAwCGgJhokr+eR8byF3tfX2vUBmSMiOFZbzwVA7WeD+stLcJtpXSawnPJlyj8uZ+an6QIXM2MN8Sn58l4GGsktU3e3MFjn3IffMeTbuS4NV2/ISPcgoV93IuPznev3vuaW3TvYrd8YG+34VfXdetNqvXlS2sh1rZMXPDc6+6x/5ztlt690K2xdX83aOcBbui2Q9yQdQa43ktfdfPnzHPTHh7gTn1oh5oHrjc9nKTSopXholRPPauvi7JePvKQeynCAvdT97SBUfr13DbH/LBV9VXyGB6rjq+ZNl51WqoqB+bx4RaD2lOg7m/rjcch9rHfa60x0H3qQzu4Aw+pefQZuIFbsWC+6zW4trVw7+HuhX/c4n5+bm17mgfiHz1JeBO26uVOPvApd9vM3dzZlz5TW9e/6YlC7uceU3lrpsxTaWmm7CRMygiiuWztw/WLL77Y8eGWFk3gShGSCIM+H9GGXum7McTJuYTdTB7Kzl9y5s2SztQx1efpsUn6vDCMmDFfPsrj2SJzOZkj6PjEWEsYMheqZ1TmuZjkpkWHpdfQXIecz/ZuWsrOD3k3RUpJtb9m6llP73NzdHE5axmIpLlz33aTfKT+4sVw77339l6x5KNSuSfHHAIXetlY+9f9sISr51fiFV3i5phqK820b+lXYmsCHbc+1wSuMvlrd9nm4qL7YDwdojsPRTxCadz0e0Lg0n1ibG4l4coaS+uiIBGib0DE0xPnuh9jTc3aWgskc7Z5ZE3HB9kx0eO6rosxj0zyvuhrtFdPmQfIeq6Z/lniCY9VhVlkDCNuyefLL7/sx90wPfwW0jltmzbQzJpU2mBqbNf1SuYEzcQna8JQRy/51J6zZK2p63ER3Q9habxjfaDEJ3qrejoiedaOhoAhYAgYAtUhYASuglgagasgUPaYIdAEAni9YpHD9kciLIBwBxsjWmkyljzPURus9HVtANJfzOhnONdeblKTXu2OWS+MZBER7qWu45AtHGURodOln+OcRei0adM6voybOXOmO/3008PHor/1gjFm8EoRuPjqBy9lWqksEYjXA74CRIzAJcg0dwwX1IQWWwByPWXE118G81wj0QtBWYzqa7wfS1ejeiUGZyEl6XTUMzjr5ziXcDRBMnymO/yuR+DSbq/Z2gqlAgogUfyIwoHfGBxQGLFw1l895ZZ77ntgnqqD7SgP6WO10qxRvKL80HVp+PDh7pRTTvGvprZMkHYRKmlEqc4XddIXhmm44oor3JQpU/zllGEjfCf83aitVdmHawXvRRdd5Lc4DdPDb1GwaqNw7Dm5Vi/vOfnTX/9JHI2OMaJp+I4YPeqRpD/ykY+4nXfe2b/K2IeXE/kSMQwv9VunBa8ZKPWYA4hCTL9HfYVkLeNuTrsjPvpd5Bvf+MZKWyLr+DjHqxaKPyTliY17enyinlPfEWkzWknub9T+yHii605uefJFczPYS5r0UX+xKt419H051/W2SgJXbJ4m42DZ8VT377LNgZ4z8yHBvHnzJEt1j1JuovDnYV3+oqDmeorApdPDc40knIuIISw1vw/Dg1o1Ycg892/bT3ejNurv1hhe86g1eIR7o0bAXTBvoXtx+itueY20NWbPkW7Q6EG1vaVqJJLadworlr7h5tz5svu/Jxe4kVuu6dYat5brV3un1+JX3dLXXnEvPfO6u+KB9dwfnt7ELXfdk8CV2+aog7ruV1lfw/KT31XH10wbrzotrSyHVhG4GKc22bA2d/ryzm70ZtvVvvZZt9ZoauvBXn3dkldmuwfvuMVdd/OD7rZpL9eM4FKKnY+9+/ZyJ32ij9tm5Dz3jckj3Iynl3R+IPNXaq7RTJmnktJM2UmYYMkcX+s8Yh+NaWNtipBEmDI31kbMWN8t8Tebh7Lzl7LzZkln6liEwCVG+TCMkMBFWYBfGbnsssvcTTfdlBwLZS4UjmWN4mgmLTps3V9xHX1SuI1j2fkh4chYzLn27JJqf83UM52HKsabVqal6j43VxeXs5bho9nddtuNIu3kUddfUH90ecjctwjJR/ohPk4Rr8UEm2rDKsqVTtG/Um9lO0P9AHoB1k1sQ48I0Uo/EzvX698UwYkdCMg/IuFqAlesb461lWbbt/QrsTVWLG9cy81fO8u2GVz0eiTV5xclcLFOZhcQ0dGnMBW9Urgm0d6/0evhqVvIXrGxILU2In7WaOgZkBSBK2Wn4B2ZE+h3pS3Keq6ZPpE4YlJVmLq/IZ7YGMZ1yafkiWuhhLaXFO7he/Jbl520QX1NnuOo66Osj5uJT+qa1mHq+GIErpyxXXTOOux656mdbOq9Y/cMAUPAEDAE8hEwAldB7IzAVRAoe8wQyEQAbwosdOQrKgzmuGXFk1VKMMDjLSIUFvAxwhfbC6BwRGJfSkk42hNJyu3suHHjnCiT9ARWFhG438bwFhO86qCwla/ryTOkjZiweEPpibENefbZZ73CI/ZseE0vfCSt+pmU8V8WnzyLMRuyCIZc4sYD2sEHH+wmTZrkgzICl0Y0/zxcUBNSbAHI9ZQRn6+3eAfBY1PovcXfUH9YCLKNIpJajMbS1ahe5RqcVdL8qbSlZ555puPL8vCZ7vC7HoFLK/9QOn34wx/2faAoKWTBj4KI/vCoo47yWdZGntxyz32PBKTqoE9ci/9IvWgHgUvqMgo4yA8xYezCGwFbfNGPo4QUQWHERDtl2JDnUsdGba3KPpxtJ3BZj6TGR61g1SScVPq5Xi/vOfnjS3EU7AhjEmSzepKaD4TviNEjVMbq53R6+coW4lUVaWGMx9sl5Be+xNXzGm0gy2l3jP3MVxAxuPgfiT+6TtX7ulK8BRLMr3/9az9H4FzGkphxQcYTXXdyyzP3PdKYkn322ccdeOCB/rZ414g9y1fZlD2ijT2x/MXe19d0nYrN06QPKkvg0vgwR/3Zz37WQQDW2yfrtKTOJV9aOZ6an6QIXM2MN5Iu+n7WBvKFu1xPHdfqu8R9dev73cbjl7u11xnk+q0xpLYfeM2wN6BfzQpd89HVe4XrNbBvjbgFc6vGNvF9d+24eJlbsaRGz1pe68sXvVHzzvW6W75wnnv9lUVu3jOL3alTt3UzXquF1UC6qgcuXTfK9qGtqq8pKKuOr5k2XnVaWlkOVZMJdPmMHrWWO/W4YW7sZhs4N3inWrtZy7k+tbV4rzXcG4sW1NrI391dU25wV935nJv5Qq0tRZhcB+yzifvk9je7ax/e3P3hljfcq/+HC7zmJDXXaKbMUylqpuwkTLa85yMALegFMCAzDxHRBK7bb7/d/f73v5dbHUf9UZnWP8T6bnmpijwQVtH5i8TLsci8WT8fO0+RP/TYlDLmhwQuwodYgrdxxhjWWI1k9uzZXiei4xNjLe/KXChl5K0Xfm5aJEy9jbJcox2Sb+qHSNn5Ie/FSClcT7W/ZupZT+5zc3VxOWsZ6nDu3LcRyUePRVUQuEQvQp1jbc4Hs2xBzweutDXtBUmIVjxbTzTBiX6FeVEo2oPtpZfWPE3W9Gxah1OUwEW4zbRv6Vdia6wwzfI7N3/tLttcXHQfnOrzixK4+IB544039tDV88AlH7OFH31pT0sQZl944QV39NFH+/D0x5dc0EQn+mfWxtRltr1GR4tIPJqEpecEkIghE4ei+2m9lg/nBM30z2Gc8ruKMIuOYcQpekGwlp0LJC1yFBKU6HiaWZNKG0yN7bo+ypygmfgk7WUIXDlju/bART1Bf1ZP+FiAfsjEEDAEDAFDoD0IGIGrIM5G4CoIlD1mCGQiIKQmXmcRftZZZ/mFeWZw0dfkCxgWSRjjUxNTvnr7yle+4sPQ3rV0oBApWLgjl1xyif8ai3NZRAg5i2taMMyilMEQngpbPy/nEq7e9k7uccSV+BFHHOEvXXvttY7/jRRf2lArBtU111zTp4+AmJSjSNbbVnJdu+U1AheINC/hgpoQYwtArqeM+FoRkNpmg3pHPUHhjiINYwCSWozG0tWoXuUanH1C1B/c+WOUQJFRj8ipXumSp/UIXBgLUM5QLhDV8EKIiAJxhx128F9Ocg2FIlvGivKBa0huuee+R5ypOsi9Vov0hc0SuPhCF0U3okm4/kLtD561qMuIJnBB8IU8THmEW6xBxv3Wt77V8WWwjA0pw4YPvM6fRm2tyj5cKwS1sk8nb8stt3THHnusvyRjhr4fO6+X95z8gTFtBtGeJsK42VoDIyFjMWScRiJGD55LbbkoXycyJtKuc9NCO5ftbq+++uoOIq2kUbd7raTPaXfaa1iKlCSGIlFGShtLKQtJp1b08T7KZUTGEp1uf6P2R8YTXXdyMcx9T9ISO0Lwh+iPsBXqeeedt9Jj9NXMJYlf9ws8GMvfSgEEFxq1gWbGU1H6MmZgcArniEFSkj8lX80QuJoZbyRhzNtRXJx99tlyqe6xT42g9f51n3QHbfWMW3v9AW7IsIFuzot93d0PLnXrrLeGGz1mkBu8Rj83oH9v16tPjaxV424tXfJGbavFN9wrLy92Tzz+f26TMb3dhK1qnn0XLnYvzlnkHn28vzv9se1qvK4a8auBdFUCVzNtp5X1NQZn1fE108arTksry0EbzXV/W288juEfXhs0sJ879gOD3cTNp9X2NR3v+qz5Dtd30Djn+o6otaEakcvVvHEtW+Ceue9299OL/+H++o/5YRD+9xZbrO++dcQTbsH8pe6nVw9z0x5vHYGrmTKPJr52sZmyI0ydJnQSjL2UGcJ6Bw+QInpuFtsKkef0nEF/eBbruyXc3DzkzF9y5s2SztSxagKXzIMoD3RAoe6DdFAW/EcgbDBHSq3VZS4k8yr/UsE/uWkheOoRnlsx4CN4nBdvRSHZIGd+qIkB+gODVN+SW89Ie0/tc5vRxeWsZXR/VHbuy4c/fACExLavw2sx9QypgsAlc2LaKW0s9CSrt8zNIXAJ4cInWP3RHtPZCQFddS6Bq5n2Lf1KbI2lktvpVBO4yuSv3WWbi4vug5slcB155JFul1128fhdeOGFburUqZ2w5IcmKIUfmWq9DGRZ+twtttjCh8GaDI/6IlqfLlvfyT2OOqwUgUv3wfpd8kBekHpzgmb6Zx2fPm82zDJjGPGKziL0li9p4qN59Dfa9tLMmlTaYGps1/VR2lsz8claPqWTiXngyhnbNTn2nHPOcY888ohA2HHEO5187EnbQF9oYggYAoaAIdAeBIzAVRBnJlAmhoAh0BoE9BaCeM7iK5mqZcyYMd7DF+GGi61YXJpQxhaOfOElAqmERSbHkAwmiwienTx5srv77rvlNX/UE+rrr7/eYbwtIlohc+655zq2/NIihmWuobh76aWXGiq+9MJQFPwQRMTTDGEQlhaUSqQFAhBiBC6NTv55TMkeWwASgzbi67qpt4TDsIxHO+qnlgMOOMDtu+++/hJ1kzqKpBajsXRVrVD1CViN/wiBiyzGPKxoIgTPaIIW7QwjBUoHkSeffNIrK+R3brnnvke8qTooaWrlUfrYZglcpFHCYjKMtywtfDGJVy1EEzXwhogxBOMORp6wjWnlsRBmUoYNHV/svFFbq7IP1+Ri8kS9hDCsRXsekjFD34+d18t7Tv6IQxT4nAvZkXMRTcLWSk+5Hzsyrsm2G7EtNTV5Tc8hctKivyxNedSQcDVpO6fdsdUgSjkk5nkpZlyR8YB3INOjINZCmULupl8KFZjybsy4IONJWHckr8RRpjxz39N50eeQNpl/ki/aN0Q+2r4WiHd8YY2ExM9U/vT74XmjNiB5LOuBi3i0IYJyIn+0bdoxv4uK5KsZAlcz403RdMaeG9V/kfu3zR90Yzda5tYZs4Z7bVEv97vrX6sZURa7EcP6u959eru+vXu5XjUy9Yrav+VvLHfLamW/4PVlbmlt+nToPkPcFuP6uFdeWuwWzFniTntga/fI/LXhejWUrkrgIuFSrzgv0+ZaWV9JSyhVx9dMG686LeS1VeXQCgLXwJrnug/sPswdtPblrt/wxW75yFGu79BRbsDwjV3ffrU50ZJernevQa730DE1omNfd/avH3EPP/a2tx9dtmuPWNP98At93Iil/3A/v2oNd9UDK3vT1s8XOU/NNZop83rx5pYd83rW7KQLue666zxhmLWCzPV//vOfu4cfftjf1wQu1gfoH0LSAtuB08cizGtZJyCxvtvfeOtPTh5y5i8582adztg5/RZkIghKsh06z+m1c8qYH/PApY3pN9xQ8yB31VWdosUozfyAOJkfnHDCCf6o4xNjLS/KXCicI3UKNPEjNy0Epz3IPP300w6yCTotdFaIXvvnzA/LEriIM6ee8V5P7XOb0cXlrGWa6SP12igkn/KRGu0A3SFSBYFLPBIxlw2JlvSfJ510Use28TkELvQB9CeawMl4ShsiP1q/lkvgaqZ9S78SW2N5kCN/NIGrTP7aXba5uOg+ONXnF/XApb0Ix/TgwKt39JCtdDXsers+6hFjR0ie5XnWmLIDB2P1rFmzdDD+oxs+0Ea0LkPPCQifuikfUUkAkgZ+M97hMRKJzQly+2cfYOJPM2GWGcOIXnR5nMe2lNTeTm+77TbHON3MmlTaYGps1/VR5gTNxJdD4MoZ2/faay936KGHAmPSVsa8Rz42PvPMM43A5dGyP4aAIWAItAcBI3AVxNkIXAWBsscMgQwE9JaFGE7xqlFPMJppQlW9Z+We3u6nCHFKu0Dmiw62LOBLAzx6oFAaMuTN7VNCTyV6EYFyAa8HeENigYXxFa8gCOQDCDZaQSBpjR0hEogLZhS4eIbgywiUFR/72Mc6tmHQRr5Giq+Y8Z/wcGEtSuRf/vKXbtq0aX7xyRZPENBECUg6+UpYvCPxhQuLSIR0sOBAwVA0j/7FHvontqCOLQCBR1/HsE4ZCdFCfyGIK2nKUu6xjQgELsqWuskCVIwAqcVoLF2N6pUs2nVdlGIdNWqUO/nkk/1P6i9f+MSEuokCjbSigJZ0xJ7t6tc0XnydCmFDe/9j2wLKRgQlC8oWETF8yG9x3S+/OeaWe+579ergbrvt5j70oQ/55IV50WnOPZc+tgoCl9R70sIXinxRj5KFLWL5Wk5EE7i0UZ46DKFW+jjGhy984QsdymoU6ZBmcvtGXXdi5L+q+/BPf/rTfis/8k3fjaIcBSZK60996lMdnge4H5JwuBaTennPyR9x6DqGAg0iMf0Esvnmm7vPfvazHeOUeEHzN+v80UYPHqM+3Hjjjb78GPs++clPehy4By4SX05aGEPpJ+nfqMf0g/IVI8Zdvm6ULxxR6Er/V6/dka6UaEUu8TBuMy5suOGG3qOmjOkXX3yxu+uuuzp9Vc58g/QJiQsjOXMXMT6H24ZIm4oZFyQfYd3JwZC85r6XwonrlDMKfIRFMkZiFO8I9YB2IPMjDOlaaZ7Kn3858adRG2hmPF177bW9R0AdtSYf6uv1ziVfzRC4CD93vOHdww8/3O26666cOr6KZ1vvItKr1wr3rqEvuU9s/rAbPmaAGz5ioHtqrnO33/9/btGCmtebmvctyFi1KVGtXN/8v2zZCrdsRW/3zq3Wcu/cor9bsmCJe33uQnfFI2PcVbM3ckuW9ykStdNjBbgzJ6snbJ0CaUOU9aEnX7nO/I16yQcvMvak7qUMjLltp5X1NYZNK+LLbeOtSEuryiFF4Ko3Hsfw19fWHraGO+6wUW6tV//iXuo7xi3ovabbcPjTbuNhT7jeg4a5pfOXuxX9hrsBYzdzbu2t3ayn+rlL/jjL3XZn7Qv9WjvUMmhwf3fa19d2Y5ZMd7+6ur/741+H6ttZ5/Xyllvm9RKSW3b64wBNqtaEW/QOkKQ5amMt6WFcQgcASYtx+HOf+5wbN26cT2rYZ8T6bv/gW39y8pAzf9F9YdF5s05n7DzV5+l5UsqYHyNwMV4yV5I1Mh5RIIAgkPuZ2zP/NhC49wAAQABJREFUQfT6VccnxlqekblQysjLMynJTQtj5Ic//GEfLGMDHoHx2qF1SNwUD0Kcl50fpghc9dpfTj0jbT21z6UO5urictcyuX2k9rbD3IQ2x1Zx48ePd3hSh4wmEhK4Um1Yno8dtU4EMiLtlHUK296hD8VDjAjzRLZ4bySa4MSzpBO9L0c+Mqbty9b2U6ZMcVdccYUPMjW/kvhSbSW3fROu9CuxNZbEGx5z89fuss3FRffBqT6/KIEL7HQdY82M11/qGLoQCC7iQTvVt6PrP+iggzoVQ4wUrNscnrnYuhfdKeMMbYc1p4gmKodzAt7hg1y851FP0eVssskm/lW8xbNeEInNCXL7ZwkzdswNM2cME70g6WDco81ju6A/ohzQ6SH8xk7BDihI7ppU2mCq/HV91HOC3PiknyzjgYv8lR3beUc7BGBNynob3OgL0H9MmDCBx3w9FX2+v2B/DAFDwBAwBFqOgBG4CkJsBK6CQNljhkAGAscff7x3R1z01ZA0VeQ9JpmQRxAMYrg1biTac07sWYz6kExYRIvoRYRcix1RDMj2dbH7sWv6ix/uM6FGxJjIbxZm8uVtI8VXzPhPeF/84hc7Fn78lkWrxIOCeNCgQdzyacDgBIkBkUWG/1H7k1NW8m5POsYW1KkFIItzFlhahNQTup3mGeoppAApP66hfEIJJZJajMbS1aheNWNwlvToL+lTX8DJs139qMtR0kp/hMIF0V9l8VsrB/kdKt5i27vllnvue/XqoFZctcKjovSxVRC42FJMvm4E61DoU2k3msDFl8S0FxR5CM9gZEO5QTsTCfu+nL6xUVurug+nPpA3IfSQF93/Cx5cD0k4XEtJKu+5+SMe/eUqv0knQjmIaMKJXEsdQ6OHPKfzzLXwC3Ou5aQlbNfUIeICe+mr+Y2hUeYr9dod6UgJxl3GdamzPIeSU/8OPfvpryzledKj8Y31zTKWxIwLMp7E6k4OhqQr9z3ejQkGcTyPhm2AMtF4xbYprpe/WFxca9QGmh1Pw3od+zo5lTa5LvnS7UmPa1pBrb9eD7cEyR1vSIcmAIThSjpTx/6933C7jHjBHbHJ427Ndfu5IcMHuudf7eWm/WORm/fK4hq3BMMMfXmtXdSIWwMH9XNbbjLIbTa2r1taI3ktenmRu/Hxdd0Vz27k5i9904tJKi59XadZX0+d33PPPe6iiy7qmEeHZAxdVwhD30/dq2dgzGk7Op4YqbjZ+hpi04r4ctt4K9JCfltRDrQ1DJZI2N+mxmP/cJ0/fWtj6zojBrllSxa5xctqbaVXf7fDpr3dv+z4V+cWP+deG7C+67N0QG3bn4FuwMbruf5rb+xef2M9d/Pdr7mrb5jlZs5602hGFGM3WNf94It93Bov3efOv36w++P9zRO4CDeVt9wyJ8x6UrbsMMhiXEXC8Z0xBlIw4zwic8jQWOtv1v7wvswVuMZvtvfFYCsS67vlnhzL5oH3ys5fcufNksbYUbdH7kt/qMemlDE/RuAiDP2xH7/BlLUGRAwRyO8YRlkXIDo+PRbKXChl5JXwUseyaWE9wHgvc7SQXK+Nx2AFQZB5c9n5YYqUQj5S7Y97OfVMl3GV400r0lJ1n5uriwvnfGCPhP1FuJZppo/UOtY3Y3v7r443JHDp8uUNacNvv73ymdYxcJfwEekLQ52XJsn6ByN/wv5MHtFp5xofc0DukfVmvfkVz9drK2XbN+Eh0q/E1lhvPrHy39z8EVI7y5b4cnDRfXCqzy9D4GJLUeqm1CnSpXUh/KZuQG6B5BIK5c4YI+/zbMyrOeRG1tryHOEQj/ThvKfHH+o2fSzvsb4IJayvhMUHZ+J9i+dTc4KcPjGMP/xdNszcMUz0gjp+sEA0tuGYmLsmlTaYGtt1fdRzgtz4ZFwtS+AqO7aDF173mCsIbuBIvdM6Ea5Rr/gwycQQMAQMAUOgfQgYgasg1kwoTAwBQ6A1CMiiqmjootQs+jzPMdFkQYTREmJWEWHyintucRWr36HzZBGEAVOLLCIw7vE1mJDG5BniP//88/2XIXKtzPGwww7zysLwHcggLFr11orHHnusn4jzbEzxlTL+o8QBI7zJaGHCztc9eHzSSmaMyxhtELbnw6OaGDpnzpzpv/TU4dj5yghIvdF1G2IJBBNELwD5rRWx/L7yyiu9txjO+QoWxUOs3kIS4Csw/muRr81C5VksXY3qlRjwaCMoubVoD1yh8lA/p0lNMZKAfrarn7Ngx2MNXxaKgItMwLjGF2GUG0J/CPFJRLuw11/gyX055pQ77+a+l6qDWrmq+wZJZ7NHqZPUZXAtIvWUH/oLSAmLvg5PRNRXFHkoL2RrWZ7BoMZ7mrAl73J89NFHvYcEFG8iOX1jo7bWij4cQxv9v66v5AFMGLvwroaBMTQKSz5jx1Tec/MncegtieUaR9LK15eQVRhziwhkbPod6ixelfjKUBRY8v4tt9zi2BYzJmXTwnyE/FO/YoJiEOUw+dCSanf6mdg54zkGITEOyzNgRV98wQUXdBgk5d5HP/rRTp7o5DpHvKCyhW8ootiMGRdEcZyqO2UxlLhz35P3wyN9Il/cy9YW+j544UmRuhBKo/yFz/O7URtodjzVHm7L9Jk6rdLnFpmf1CNwEWbueKO9A5YlcBHvgBqJ613D5rqDN5jhRq2zwq05bIBb1ru/mz1vhZs7742aR7o3atvA9XbDhvR1Y9bu7YYOWOYWzV/slsxf5q56Yn335xfGliJvESdfK7PdWFERApfMB8L5GH0F3gWZoyO6PFP3NGFEiP46PWXbTqvrq04b562KL6eNtyot5LPqcqhHJkiNx6SjsdTc1KkNRIcNG+KO3GmZ23HI3e6aV95dMxQ+7w7Y8iW36ah+rteQ/m7gmPWcW2sj9/KC4W763xe4f8xc4hYv6efes33Nw93wB9ySZ592/3vzEHf5Q8MaR13giXp5yynzAlGWKjshDhGu3spO4tloo40cH7WJsJUicwUx1qJboE+RPkCeoy9AB/DAAw/IJX+M9d2dHnjrR9n6lzN/yZk3x9Iq11J9nl47pwjLUg6xeQoeQFh7ix5D4uMIOY4P1iCFiOj49Fo9ta6W94ocy6RFfwypt9uSeCAV8PGXGGJlvOF+mfkh6x6IggieWdlmW6Re++OZsvWsJ/e5ubq4ZtYyuX0kdYr1uPZ+RXmzdma+xq4GjEkhgSvVhnm3nqQISXjWYR7O2oUtxBBILOhy64lsscaz6NPwjE4fp4W047mOZ0Qaza/qtRXCKNO+Jc56ayx5Jjzm5o9w2l22xFkWF90Hp/p8sTXoPl8TbULi1wYbbOB1qfLBMukSYbyl35sVbHko9zlS5wgDCT+Q8hff+kPa0e2H4w3zf8Ya1qGiC+YV+l52AZFdOfC8hVcw2pcW7AJ48UvZKfR6Tt4r2z/Le/WOZcLMHcNkngOhiH4ztL2wbr/pppu8l/0wrTlr0kZju66Pek5A3DnxyZowtg0nYe65556e+Mh5uD4uM7bzPkK9xbMraQ2FNNBW0HOaGAKGgCFgCLQXAbEfQuhlnoqtgvkD/zmX30VT1WuXXXZ5k+5c9I1u8pwRuLpJQVkyDYEWIMCiaPvtt/eGVjpNXOfzv4jwLgtR+hDIT3gwaFborLfYYgu36aab+q/BIEmhWKhaUKxstdVWfvGOEZnFHosgBEUgxBIWtjNmzOi0QCTPbGPFApfFrWzhV3X6enp4eKniKyzqJMqBkKiA23e2TYDIxSBPWRTddqinY9uK/LMgZhsOFOuQRFolueWe816qDqLww0CSInm0Ku+pcOsRuHhH+jPaCm7sIZiIh7RUmPTDGKRoY5CeaGMQ7yDREkZM2t035vbhpB1DIoZCFgJsW4yyRiuuY/mrd61VeWdsRVHPdgEQ5sAe8h3jTzPCQoj8Ey5jH2M+xoh6kpMWDB477rijV9ISJ9usoYT861//2jHehnGm2l34XOw370JOI61sA0nZ1qvrKIqZBzDfIP88T/tACd4KycGQdOS+Vy8P9Inkna0mUYJjrGS+00w7qBdfK+5pQlXMa1gr4iwSZs54UyTces/0qW3jNnrg626fdZ9224140a01pJcbsGZf17tfX/fGipoCpHa/94rlbvGCpW7ZguXukRfWdNfMHueeeH1I4W0T68Vf1T3qJH0F/QSkbi317unn9Hkr2o4Ovyufd6U23s5yqGo8Zn6w/rpD3B47berue3Se+8eMmW7EWkvdQe9c4HYau8StNajm3W6Nfq7PsNqHkIOHu159B7sVy2tegRfUtv988Tm3qOYJ7ye3reumzuxsfGymzjTKWyvKvJVlpz1wXXXVVf4DHIxybPPN3BUdAHPtcA1YFsOcPJSdv+TMmxvlI6fPaxQmaxj0LOg5wJjxf+rUqe7ll19u9Grl99uZlrLzw1hmG7W/nHoWi6eKa+1MSyNcUvlpZh1HmDlrmdw+krUw6zE+hkEvyVqhyHw5pw2TRvpG+iDm5nyIotd9tF3SQ9uljywrrP3wsMW6nv613jqpbNj6+Xa2bx1v2fy1s2xJZztwqUfgIg20nXE1D9bUJUhBzLmp0xwbif6A45xzzqlrN+DDKuwM6KDwlsX6H6KMCCQu9L182IWOKTbW03djb+AZiNyyTaCEUfTYij6xFWHq/AiBCz03BE7wZH7EPAk9B3g0ar/0J+3Umbc7vrJjO3N7HBGACR90Mvdhx5Uq7Fi67OzcEDAEDAFDoDgCRuAqiBUTDxNDwBAwBAwBQ8AQMAQMgfoIHHzwwW7SpEnegxBGj1Ut4sUm5X58VafP4jcEDAFDoBUI6K0wQ8+PrYivq4eJ/6B+vZe7kf0XugnDXnZbrPmSGzpwsevfZ0Vt+8Re7rXFfdzjr6/tps0b6WYvHOwWL+9T8zfEWyaGgCEQQwBDT58+vWuG+uUdpOO+tfa08brL3G7jF7vt1l3qRvRf5nrXrvXuV/Ne0mupW1HjV69Y0sfdO3dNd+6tQ9yipdbGYthyLUbgSj1r1w0BQ8AQMAQMAUOgMQJ77bWX917Fk6EHrsZvp59gTvSDH/zAe2nXHr/Sb9idZhAICVzNhGXvGgKGgCFgCBgCXRUBI3AVLBkjcBUEyh4zBAwBQ8AQMAQMgR6LwM477+y3ycAb0sknn+y/BlxVYKBEGzNmjIPEwHkrtnRcVXmzeA0BQ8AQiCGApwc8n+K54fOf/7x/JNy6JvZeT7vWq0bN6t3BG6l5l8XBbG2cwNEspC1+mhgChkAeArWm5KmPI4a+4baq7SS0WY3IteEay91avVe4BW/0cg/P7eOuntbfvfJ6RyPMi2g1f8sIXKt5AVv2DAFDwBAwBNqKAFvtsTU7nhYRCFdskduMyNrrkEMOcRMnTvRBXXvttY7/Jq1DwAhcrcPWQjYEDAFDwBDoOggYgatgWRiBqyBQ9pghYAgYAoaAIWAI9FgE2HZuzz33dJdcconfemBVAvGjH/3I7w8uabjzzjt9uuS3HQ0BQ8AQWN0Q+Pa3v+2GDh3qSauSN/pCtpIwMQQMAUPAEOg+CBiBq/uUlaXUEDAEDAFDoGsjcNBBB7m99967I5Eral9tHH/88R2/c08gEiF8MIgsXbrUff3rX49ueegfsD+VIGAErkpgtEAMAUPAEDAEujgCRuAqWEBG4CoIlD1mCBgChoAhYAgYAoZAF0BAE7jmz5/vfvjDHzqOJoaAIWAIrK4IQOAaNmxYR/ZuvfVWd+mll3b8thNDwBAwBAyB7oGAEbi6RzlZKg0BQ8AQMAS6PgLaQxapveaaa9x1113XdMKFSERAy5cvd+eff76bNm1a0+FaAPURENxnzZrlfvKTn9R/2O4aAoaAIWAIGALdFAEjcBUsOCNwFQTKHjMEDAFDwBAwBAwBQ6ALILDrrru6/v37uxkzZnjX+HxlaWIIGAKGwOqMwLve9S43fvx4N2/ePPfwww+72bNnr87ZtbwZAoaAIbDaIsBWT9tvv73P32OPPeZeeuml1TavljFDwBAwBAwBQ6CVCIwePdpNmDDBzZ071z3++OPu9ddfryS6ffbZx388M2fOHDd9+nT7YLASVBsHwpqX7Sufe+45N3PmzMYv2BOGgCFgCBgChkA3RMAIXAULzQhcBYGyxwwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDoDACRuAqCJURuAoCZY8ZAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChkBhBIzAVRAqI3AVBMoeMwQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAyBwggYgasgVEbgKgiUPWYIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoURMAJXQaiMwFUQKHvMEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQKI2AEroJQGYGrIFD2mCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIFEbACFwFoTICV0Gg7DFDwBAwBAwBQ8AQ6FYIrLvuum7vvff2af7zn//snnvuuW6V/u22285ts802Ps2//e1vu0TaDzzwQDdkyBD36KOPuvvvvz+ZJp7r27evu+eee9ycOXOSz5W50YowU/GvscYabq+99nLbbrutY65MXpYuXermzp3rnnzySffYY4+5Rx55xC1fvjwVRLe/vv/++7sBAwa4J554wk2bNs3nBywmTZrkevXq5e699143e/bspvI5fvx4N2HCBB/GXXfd5Z5//vlC4e2+++5u1KhRbt68ee6WW24p9E69h2J5rfd8V7q35ZZbOv4jl19++WpdJ8lj1XWQMHNk4MCB7qCDDnLrrbeeGzRokJs8ebLvG3LC4p3uXAdz89zV3ys63nX1fEj66LcPOOAAP57NmDHDTZ8+3d/aY4893Prrr+9eeukld91118njduwiCOy0005uzJgxbv78+Y65rEjZPmN1LedUvRac2nUcMWKEe8973uOju/vuu1dac+y7775u0003dUOHDvVz45tvvtm9853vdOPGjXMor6dMmdKupFo8PRyBnjZvLFLcsh5hXXfFFVe4FStWFHmt8DOpuWu99czYsWPdxIkTHfoM+rkf/vCHheNbnR/caKONHLghF110UcOyqnqM6Ir6mTLlfdhhh/n1PXqMv/71r2VetWcNgVWGQNk57ypLqEVsCBgChoAhUBgBI3AVhIqFhIkhYAgYAoaAIWAIGALdFYHRo0e7wYMHuwULFnQymGBIQUmFQICCcNKd5F//9V/d5ptv7pP8pS99qUsk/bTTTvNK5L///e/uZz/7WTJNp556qic1zJw5051++unJ58rcaEWYsfj3228/9773vc/nM3ZfrmFM/fGPf+xeeeUVubRaHX/yk5/4/EDgOuOMM/w5xsajjjrKn0MW0sZkf7Hkn8985jPuHe94h3/rd7/7nbvjjjsKhSB1YeHChe7EE0/seAfDKEZUDC/UPS317sXyqt/tyue6n/jqV7/qiYZdOb3Npq3qOpiTnv79+7t///d/dxxFfvOb37i//OUv8rP0sTvXwdKZTbwAkaF3797u5Zdf7hL9atHxLpGdLne5X79+7gc/+IFPFyTks846y59/+9vfdsOGDXOvv/66O/nkk7tcunt6gr773e964vzixYvd17/+9Q44yvYZq6Kc29GmU/W6A6g2neg1xx/+8Ad32223dcR8wgkneJKkXIA8+Ytf/MKdcsopbvjw4W7JkiXua1/7mtzuFsd2lG23AGIVJzKnHHravLFIEX3iE59wO+ywg3+0FXPp1Nw1tZ6BZHfMMcd0Wot2FV1AETxb+cyRRx7pdtllFx/FV77yFffGG2/Uja7eGNFs++mOZSJzWwhc55xzTl3s7KYh0FUQKDvn7SrptnQYAoaAIWAIpBEwAlcam053jMDVCQ77YQgYAoaAIWAIGALdDAEMkijnXn31VW8MkeRrY4oRuASV5o6i9GtE4PrgBz/o8PYAkQaDI8apZqUVYYZpgpyEkl0LX2O/9tpr3msJnnb4klcEpTFGuIcfflgurTbHmKIsZYDIzXTVBK4vf/nLbsMNN/T17vjjj++UrHr3Ynnt9HIX/tHTDHFV18GcosXIh7EPoY+jf7jwwgu9Z8Kc8HinO9fB3Dzr9yBefvOb3/SX8Ajwv//7v/r2KjkvOt6tksRlRJoyYq4KYk9G8nvsK92VwNWuNp2q1+2uMHrNoQlceHDVnnMgSuJx9Morr+y2BK52lW27y7C7xZdbDj1t3likXA8//HD37ne/2z/aClJOau6aInDp9dGyZcv8R2rf+ta3imRltX+mKgJXFe2nFXWl1QUoc1sjcLUaaQu/SgR6+jq5SiwtLEPAEDAEugoCRuAqWBJG4CoIlD1mCBgChoAhYAgYAl0SASNwta9YROnXiMDFFoR4qIHsdO211/r/zaayFWHqNLHdJluiieDR7dxzz+3kyYn87Lbbbt6zG55iRDDOPfPMM/JztTjGFGUpA0RuhrWBoowHLrYiYksRtvy65pprOqKvR9Kqdy+W145Au/gJW3mIFzOIqqvztp4URdV1MKd4P/7xj7t/+qd/8q9CNKpiC5LuXAdzMAzfyTWkheFU+bvoeFdlnK0Mi/ELw2efPn082VA8xhmBq5WoNx+2EbjqY5iq1/Xfqv4ucxLmJgjbI8qccOutt3ZHH320v/7ggw+6X/3qV/6cP2xJzfaYeB28+uqrO6539ZOu2F93dcxakb7ccuhp88Yi2LM9F96XIeWHH4AUeb/RM6m5a2o9I975SA/e+ZYuXdooih5zvyyBKzVG5LYfTYA0AlePqXaW0VWMQE9fJ69i+C16Q8AQMARagoARuArCagSugkDZY4aAIWAIGAKGgCHQJREwAlf7iqWMQZstKMaOHevYarCqr4ZbESboodz9r//6r44t0Z588km/bWBqWwaUvijUBwwY4MHHUKc9LLSvRFoXU0xRljJA5KYil8CViq8eSavevVheU3HY9VWLQNV1MCc3n/rUpxwGUOR73/ueJxLmhKPf6el1MNeQpjGs+rzMeFd13O0Mzwhc7US7fFxG4CqPWVd6413vepf75Cc/6ZNU1QcNqzp/XbG/XtWYrIr4rRyqQx2v0Xh6Zt3HtnxVS9m5q/T7ELdY+5q8jUBZAtfbb3Y+y20/RuDqjKP9MgTagUBPXye3A2OLwxAwBAyBdiNgBK6CiBuBqyBQ9pghYAgYAoaAIWAIdCkE9tprL7fJJpt47zMQcPA889BDD7nnnnvOf8mutzPBM82zzz7rdt11V7fFFlu4wYMHu+eff95vfXfjjTcm8zVu3DjvcYlt2eQd4rjnnnvcokWLOt7jK3qeRa677jofl/+h/jDnOuKII/yVGTNmuFtuuUXdXfk0VBBus8023uvLpptu6r8QJj+kgy/6Y0J8eJTaeOONHd6rwIetW/7xj394r0Wcx4T3dt99d7f55ps7ziEn4WFm+vTp7sc//rEnOzXywEW42mgV81C18847O/KE54E//elPbquttnK77LKL22CDDbyHEOLFQ4j2btOKMEkr5XfwwQdz6pX3J510klu8eLH/nfqz7bbbuk9/+tMdtyGAjR492uebi6R92rRpHff1CVt1DBkyxG+JMXnyZH3L1yO8fDWqc7xEfO9///v9+5dffrnf5hHF9qhRo3zd4JrI9ttv7/bcc083fPhwv+XowoUL3SuvvOKmTJni/va3v8ljHceYoqysAaIjsMRJLoGLL9ZRvM+ePdvdeuutbsstt/TtlDo7cOBAHxvYY/i47777fL2K3aNfwFgTy6tOMt6WIOxASESefvppX7b333+/b4v6WanXeHCjbOkX6Is222wzj/e9997ry4a0kQfyQhtle076Luo72ysVFdoEfRr19Y9//GOn14YNG+b7nPXWW8/3AfRZL7zwgq+b9B1lBc89pBcsqL/0u+Rzzpw57qqrrvLpLxomXkloQ9TDyy67zGMg/TNhULY33HCDe+KJJzoF2agO0n8ccMABbp111vGYsn0rigHqAX2ueCgj/bRDBEzYziomEydO9GnDC8Idd9zhaEfgTTtCKC+w533i+tjHPuZxof6JhyMd7nvf+163/vrrrxRnozqow6CNMx6xdeMll1yib/n+hzqBUD8feOCBTvf3228/7/EF73XgLlKmbKse78gPeIIrwtjEGEnayYMIW6Udeuihvk2tvfba/jL5YBsY6l+ONDPelekXJG30BYxztE28ONIHM5bTdlNjMuXJmEw7Zjs22jHjI14LeV+k0XhAHf3Qhz7kHwfbRx991J9rAhdk60MOOcT3V2BM2yF9jNHz5s2TqFY60s8VHbdWejlyoUx9jLwevTR+/HjfN9DvQr4mb/S5zAFnzZrV6R0w4LmZM2d6b0r0O5Qb3pLA/s477/RzIl6ib2frrY022sj3L9xnXHr88cc7hSk/ivZR8rwY8uln2JJapEyfwTu55Uzdo23SB2rvUZIOcAIvBIISfXfRNi1hxI5F2wrtKFavdZiQM5hvUqcZrxj/2PqatkWZPvXUUx3zNcqHcY65AWM4bVXGAfrduXPn+rZz/fXX6yh8OMSDgAPt+bDDDnMjR4501D2EdQd1irGB+sGcjHbNhw4xD1xF274PvPan7Lxf5itl5uHNlK3E14r5UdnxoZVpoTzKjA+SlnaVA/UqNm+Ufo/+8M9//rOfu+LhlbGF/od2gnc55sBlhXUJczPm0cy/mAPTlm6//XY/PwvDazYt9Av77LOP75+Jm/hY/9L2wnklcTO3++d//mefT93PVjWPTs1dw/UMW3QzpnCkTtPv0l/RZvSajjRXPe4SZj0ps44E/6OOOsoHx5qGOR0ezugL6ROZT1DP8KKcmvtQ98CNOTP1jz6bNRTzaMZjBLJd6mMr/0DtT2yMaKYfW1X6maJjouQ7dZSPE5g7//73v/d6BOoSuiLKhfZx6aWXJnEt27YkHbR9dBYcGUtZM8laJNYmeY/1BvWANR3zX8ZK+iHmv1oPx7MydtNm2MaY58GMtSZrP/RlzM0Ig3ush9ANMC+g76V/YC0h60TC1FJ1e6tynpu7ptb5K4N1bnz9+/dvyTq57NyHMZD/rAOoS5zTPxHOBRdc0LE+0vjYuSFgCBgChkD1CNAPI+g1GRfRLzPP4D/n8rtozL1qE8QVRR/uTs8xQJkYAoaAIWAIGAKGgCHQ3RBAwYrhIxQUQnhHQmmD8QTBUAnJIyYosM4555yVbn3gAx/wxpWVbtQuEMcZZ5zRsUUKxCyMXAgEr/POO8+f6z8QhFCOIBj92XqrnmgFIQQbjE8xwfAbksFQ/vI+E96YoNwiz2LAlWdQfh133HFesSXX5IjRCyU+YRYhcPHef//3f3uvVjGMJX8oXlHgY8iKCco0iBwirQjzO9/5jhs6dKiPgriKkgFOPvlkT5biRcoAIoiQ9FAGYngNBcX1N77xDX859E5Wps4RgDYIYDBE+SRljqL75z//uY9HPJf5H5E/GJxDD2Ix47COD0MCBp5mJJfAdeqpp3pyDuSfE0880ZPvpG2F6aFuYVyPyTe/+U2vvIvlledRNFJPx9UU2zF58cUXPW5aiSz1mucxNOy0004rvYoh5pe//KX73Oc+5xeq4QM33XRTJ3JNeF//1vFRzhipEMgU1EWpD/odzh977DF39tlnJxXV4fMQ46jPYJIS6kNoYEo9+4lPfMIbqLifwol+Cix0mPXqIAY/DO0pwQDE1q4oClAKUOc5Eg/1SJejhPGjH/3IlxHPYGgSwqTcl+NvfvMbb6Bi2xuEPpstWEOBIIOxQMYpuZ+qg3JfH/H6JWt4+iBt/CIfbOWFQBagrWiBhEuedf9UtmyrHu8EY51OziE6nH766f4yY9PnP//5DoJm+CxKH+ozfVlRyR3vcvoFFFKUFWUfk2XLlvl5gx6TabuQhGWL1PA96uT555/fQeLWbSM2HkC+wWMpQvs/66yz/LkQe6iTtBGpW/7mW38waEEWvPvuu/Vlf1523FopgOBC2foYvB79iQckDDQxAUeI92AmImMMfQWYMG6HgrGPcVbIS/o+YTL+MvfRUqaPkveqJnCVLWe2E4OchsS2i4J8f8wxx/j7YMj/Im3avxD5U7atQHKI1WuCxkhM+iFwhMI4DBYQNF599VUnfbdeO5AXyAL0maFgcGYdIKLfw3jMPJ/xJiZ8EPGLX/zCxwl5NRwPyrZ94siZ98v8ocw8vJmylfhIb2rcz5kf5YwPrUpLzvggaVkV5aDnjdLvsZZhPsSHJKEwFvzP//xPlAQVPiu/G81H8Xr8s5/9rNMcrJm00N5Zy6655pqShE7H2BocIgflwHyKsRpplO4y82g9Puv1k+RT1jOkm486YqL736rH3Vh8+lrZdaTul8EbwiD9cSjg/R//8R+eoKbvMfeBeBMKYyvEa9H/FCFw6bTI3Keqfqwd+pmyY2KIWfhbCFyQpxj/CD8U6uOZZ5650pw6p20RNmsz9GCpNSlzS/1BG3XlC1/4Qkc5h+mjr7z44ov9OCL39BjMRzsQDsP4qD8//elP3VE1ciFE0lD4YIG1Ic9pqbq9VT3PzV1Tk8ccrHPjY74jc62q1sk5cx/Knz6ZcoaMj/5M5KKLLvKkWfltR0PAEDAEDIHWIWAEroLYxhR0BV+1xwwBQ8AQMAQMAUPAEFhlCEDOQuHKV3koaFAq8zUvXw9iyNWKHEkkhjgUfyh18RogEpKg8KKy4447+tss7lFmo8zCMI6nHESUQBhxIP9AAkJCQ4y/WPsjRlJ+/+d//mdDbzWi0Jf3OWJkIn8o0PhSUoSwxRMHniUw8AvRgrSjkMJwDAFLtv2D6IFCVoT7EFpE2YVyDIIK+Q2VXEUJXKLgISzi0l81xvKHEZ58YCxFuSWitylrRZhakfv//t//62REkDTEjniD2WuvvfwtDO8YbIUUwkU8eWGM0vKRj3zE8bU9osliZesc72uDAL9FqJt4WIEkeOCBB/qv0LlHOeD1gTrKF7Ao0qS8UWBpDxAxQomOTxsgJN6yx6oIXHg6kK/Y+aoWgbyC3FzzFkAZ0d7DeyixMVLF8sq7QrbhHOzwLIKiW74E5jrv8xyYIrF6DVmG9iZ9lX/wrT+0CZTo9CHSZrkFyY/20Eh0fGKIY32H0Z+ypS4QPh4T6Pfw4CfGaDxK8eVpEdHGGwwu9Hv0JXgv0UayIn0b8Uk71nHTVujDdT/FfeoxBiAkVQfxxoiyX/KMQZC+kvTRn4iBQpMVIQSh+EVCoijXIM5QRxHeg8CFAQKDh/SjUs8wJBBf1YppH3nwR2MHUeC2227reEL3ZdRZvRUQGH3xi1/0z0Js5Mt7pGzZVj3ekSbGGCHJME4zDkBywGsUYx1jgNRb2gX1mTGCfkzaDe9BYNPjjM9g5E8z411Ov/DZz37WTZgwwaeEdOJBgPoO8UBIXZQX6Ze+RG/VSTumH6F94I1C2hzvkB7apG4bOssyHuBxMEZ00XMT3uN54iF9GEmlrnOdciAdIjnjlrybOpatj6lw5Dok2o9+9KP+J3hBRmdeRH8MliIYKyFlIWJUl3vknXcYQ/TcUe5TZszPKEvpayhn7cklp48i/KoJXJLmouWcQ+Bq1KYlDbFj2bYSM85LuIKd/IY0T/9AHyZzH+6lCFzyHuM3fT1jK++K0D+JF1+95qBfnjp1qieP6Tk0SmraFeQlvJ8yXsQIXGXbPm00Z96v5w+Sp0bz8GbKNhZfs/Oj3PGhFWkBw5zxIZaWdpWDzBtJe9jvcY11IP0b61/p2/jNh1JFJPRWTHjMpVnHEqaM65qwTbi5aQnrA3N0+nzaLn23tPswPsY6+mvmcaznq55H6/FZr58kn0LgYs3G3BN8wEbGHvotPD0jrRh3fcCJPznrSN0vS7DkBR0M4yjzvVRZ6LUh79JvM8dhzJY6KGHmEriq7sdaqZ8pOyYKNqmjELjkvsyLKA/WX4IxYxV6DJHctgV5XrYRJiz6NtZUjH3ywQnX+egFUg+i56WMv8z5mVMxZ6NtItQnPuqjfSN6DPYXan/IA+sJvW6Xe7zP+MP4KXNq7uGdmI+HRFrR3qqe5+p1oaSbvDdaU/NsDta58VVN4Mqd+wiBi/xTD6Qv4pwPsiAAmhgChoAhYAi0HgEjcBXEWCY/BR+3xwwBQ8AQMAQMAUPAEOhSCGCQRFGoDTAkMFTkhF/34Tqd/4g26qPghTzBYh6vGHgA0dtFsMUCX/UhGGHxaIJopT3voBwWYb6FcQVBcSUemOR+7KgV+igUIDHo7bC+/OUvd3wdrck3EFnYpgIJ88w1lO6QGhCMW2KMPfbYYzu8lJFf8kD+EbZy+PjHP96h4ChK4MKYCS4I7vhxXS8S5g8ykyYQaU8yGMgwlCFVh4miEtIDgoIcXIsKuKDEQjDufv/73+9E4BFvFDo8nuFrR8qUPGJcyK1z2iBAHCi5IZFRn0XEY5k29Ms9vq6X/Oq6zP0YqUnHpw0QEl7Zo1bSQySCUFREQoOHvCNtAmwxOmupdy+WV7x2yRZ7tBHqCAYEhP6GshPihS7nsF6jkMZLC6IJQfzGUIRBhjqAaIUoZAuMvI1ExyeGON23sdWX3mYPJTV9Ef1bqJxPxQW5hr4CwfjG1/JgLKK9C2os5H7sqPPK/ZBApb08iWGL51J1EM8E497ylIZnIb6yF9FfOmvDo/Ycg5KbfGnRBK8LL7zQG+S5r43r2nhUtWJap0WfayKWeBPgvm7P8jz1VsYvthnDowQiRLvcsq16vKMPhECMhN4xjj76aMd2HQhEWQjaIvTf2rMVpAnKqpHkjne5/YLMU+iHIQljkBLRhgRIdZDrqEtgTDulrUEu0lvynXDCCR3kIzE26bZB2OF4oA2qut6Exhu8CkmdwYBM3ylEp1mzZnWMDbnjluQ7dsytj7Gw5Jr0/fzWJC1+a8w0JjLG8AxGQ8ZRiLBI2HeJNyV/s/ZH40k5yTwqp48iTCEhhYSw2LglaYgddbqof0XLOYfARfz12nQsfXKtbFtJ1Wu84opHVMZY8i9jLX0lhF/eRfT6IVw7MN/Fq660WT4e4RmEjyOEFKnf08RabbwO5zkxAldO28+d9+v5A/1M0Xl4btmG8VUxP8odH1qRltzxIUxLO8tB5o3UZ93v0W/htVEIFcwdqa/SZmQOwXv1hHkVxA9Exip5HhIPc2khjOCVjv4UyU2LxpIt0/S2r5BT6JPlYw7iEBI+aWBuhSEJklHV82g91uj1k+RTz3PJv/T74Ty9FeMu8dWTnHWk7pcJmz6WMUfGUT3/pq5RLggEWfpqmftQfmxJjkDWZ86tST96Du4fivzRadHjfFX9WKv1M2XHxAgEnS5pAhflgR5L1rd8oEKbFOK+Xpvlti3Re5AIrc/htyYHMsfFux8fufGxG0J7pA+RsZtreg7G+lzWp3oM5jk+4GL7b4T+i+eEMMpcnHwzhiN6fa49Q7WivbVinqsxIT+63PidWlPnYp0bX9Xr5Ny5j153gQ8fb9CO0dGaGAKGgCFgCLQPASNwFcTaCFwFgbLHDAFDwBAwBAwBQ6BLIiCKLW2AIaFakYMhU4hEkgmUU/I1q35XG+35Cusvf/mLvNJxFGUUBg+MsRjWtBIBZSNbpIloggNb7eHxq5FoRRnErfPPP7/TK9o4pQ3eeIXCiEDa+Io4VEZor1EoylCYQShCQYbCFKUWJC+OWvQWREUJXLyPUR4FGEpCIbFxXecvNIByf9ddd3Uf/vCHOV3JoF9lmHhAQQGMhIpyf7HOH4hw8hW6EPPGjx/v/u3f/s2/BdlFb6Gjn9ekwdw6pw0CKMBRuoqRUZJNHccwwdaeYR3iGfHYE6Y1ZhzW8WkDhMRV9tiVCVwYpyD+0I4wJtBHaKHN0A8gun/R9fr+++93v/71r/VrTivOtfGIhzQx55577nG48W8kOj4xxGmSaYxQxVYQeC9EYU8f0Eio0xBeEAgm4TZqkDvE+BISxlJha+UvxjLwDgXcMSYgQkRK1UFIGrQvykn3MxKmbLlLeWpynximdF8u70jboC8kfzyDrGoCF2mQtOk+64Mf/KDbY489fDrpyxFIs5BnEek3NYktt2yrHu9ShjQMb5QR+aFvo46HQjuV+hOSXMJn+d3MeJfbL0i7B3sIZ7qfpo7LFnRsQwoJTXtciHnKY0s7qcdi9NJtIzYepIyYup3FiKPgS19FGWiSc+64FSsTuZZbH+X92FFvkUz50d9oIR8Y98AR4g0iRnXOmcuJAZnf2tBHedK3SN/AfW0og2wo22Lm9lFiyA/rdmyMJv6U5JZzuwlcZdtKql4zDqDrpGww1AopUfDRc0y9BtBrB5TKQiyV9yAR0CchqfeaIXDltP2ceT/p1/OHMvPwVH9NmPVEx1fF/KiZ8aHqtJDv3PFBp6Xd5SDzRtKv+73Y+pdxCuINArEGglQ90R+56DWPfmeXXXZxRx55pL+ktyXNSYv2Dhpru0Si186xtbWkrep5tB6f9fpJ8lmUwNWKcVfynDrmrCN1v0wfjA5GjHQSjyb3yfaQel2oCTjyjp6/cW1VE7hidUjXsWb1M+Sx7JjIO/VEwqNcmBeE69vtttvOr3MIQz5Oy21beLPHgxUS+1AGQhV6PPQUMseRsZt3YnM2ruu6w5ya9aweu/XYzPOIngPF1sbME0iP1le1or21Yp6bu6bOxTo3vqoJXLlzH03gwoOb6IPfrCn21xAwBAwBQ6BdCMjcEFsKcwH0TYzF/OdcfhdNT6/awuLtz4yLvtUNnjMCVzcoJEuiIWAIGAKGgCFgCCQRKELgwpMNhslQ2O6OL3G1EVyUGdpQGb73L//yL26bbbbxl+WLYYw6KACYZIoSSt7TSqOcrdHOOecc98gjj0hw/siXxJDHkNArib+o/kBWY6spSBsY3pkcI0Lg0gbJmFGFZ/UXg2UIXFqhBrby9a02WAg5g3hEtKetMH9VhqlxRAGIIrCoaNz4ihtFPEI9AHOUo5Cq5MtRbRCATCUe1XLrnDYIaCVxo/RjfIO4tldta8Fxb3ktMgKX89sC8oU4C0YUuYgor2OYQs7D8E85C5lC1+vQExRhCGEIUpAQByVstkyR+kd7p903Eh2fGOK0YZr3MVxD5KG+aeJIo7Dr3QcjtpzBk8n73//+ji0tcghcbE143XXXrRQdRj2Me4h4JtJ1XhvBVnq5doGyoZ7jMZFtNumbdVnxjiam6vDo3+nnEf3FPr+7AoFLexWSbV/pa/BMQF+EsQXijfRL5J06zTHMD3nSUqRsqx7vUoQATWqs18fpPlfaos6TPtf9dpnxrpl+QRubmB9ADGfry5BMJOmkH6A/QOiTMGyHgmcfyhjDOG1ct40YVtqgquuAzE9ifZLESV9FW0IgLJGH3HFLwix6LFIf64WlSQfM6+hbIdKDQUrEqB72FzyvParGDD94QGWehbC16l133eXPY3+K9FFVE7jKlnO7CVxl20qqXosxVs/vwzIQIqw29ur5ZeqDCxnHddj6vWYIXDltP8wXvxvN+3lGzx/KzMNT/TVh1hMdXxXzo2bGh6rT0sz4oNPS7nKQeSPlJv1eqo/QXqlCb3KxctdeP0MirDyPdy7aPKLbYU5amOfhLRrR20T7C2/90R9vsbUa419Mqp5H6/FZzzUln0UJXO0ad2OY6GuN1pG6X059pKHnsZwzPgvhmrGXDyeoi6Fob5armsDVav0MeS87JoZ4hb+FwMVWpkJGDp8R3ZrUy9y2pfuA2McIxEs/ztqF9o/Hakmf7g/C9PGhHEQoRPRwegyOfQR13HHHuY033ti/Q77JvxbRCWpdULvaW7PzXE2oKrOmzsU6N76qCVy6/OS8yNxHE7iKkJElbDsaAoaAIWAIVIuAEbgK4mkEroJA2WOGgCFgCBgChoAh0CURECVTqOjRihytLNWZiBlhxKDDcygQY4IRXARvWhh5EJSNstUQxiO2pcNAJ16YQpKMhBE7aoU+Bh0USlq0ETEkVEFKw8vOtttu67euQDEUEyFw7b///u5973uffySFFTdF0RPGFwtbrhG3fF153333uQsuuMDf0vkTAoK8w1F7VQnjqzpMyVfKaKHTpc/f+973evIK11A6sn0hgut/3NIj2o29kAxCTzK5dU4bBHQ8PmL1B+Uo2/5ACIB4EZOwbsa8e+j46tWTWPixa/pL6yLGIAkjZfAQY0DM6F7vXpjXcCu6Iv2AkBp0vY61W6kD2vAr+aqKwAVB86STTvKe7yRsOdJP4o0NTz9CppR7jY58Vb733nt7cpCQQMN3cghcMQMI4eqtHSTcenUQ0hJtb1yNlIgCV/fTks6wbkBCo+wQITtxrssx3HatKxC49qqRL/nyGJk8ebL3iib9CGQ9jCFbbLGFN35h4NKEtJhnjZyyrXK8SxECJk6c6A455BCfTz3W+gvqj05L6ot9eTx3vGumX8ALwUc/+tGV6iSespgn4NVOb5kq/QRpFsOmpD911G0jNh5og2qMwJXyWEJ8mnwshDKpb9wv0kfWKz/C0JJTH/X7+px+FU+Z5F8LaabNQ+KjzQjRmmdkjAnJ+NzThIOQXM79egSunD6qagJX2XJuN4GrbFuJ1Ws9f4yR7CgnREjYev2g1w5sv4QXmFDE05Iex/V7zRC4cto+6Ss77+cdPc6VmYen+mvCrCc6virmR82MD1WnpZnxQael3eUQI3DF+j3Kdc899/RrS86LzNk1ySKWL8JBhDiht9Kr1wfzTiwt4oWU+0ijcYk+n3TFpOp5tB6f9fpJ8ilEGUmL9Pu6j+FeK8ddiTt2LLuO1P2yeAkNw9XejYSIJTqdVB0kDDw6MU4g8p7/kfij06LnPq3qx6rUz5ClsmNiAoaOy6LziHn7k4c0kY45QG7b0n0AW9TxcUE90fMrtrVj/RUTPc+SbRn1GKzbmLz/xS9+0ZPF+E2ewv5B+iFN4Gple6tynqsJVUXX1KwThDxbFuuc+C655BK/PTxb8SJ6q0p/4a0/eOvjI07tLZpboa5G3smZ+2gCF/HhSd3EEDAEDAFDoP0IGIGrIOZG4CoIlD1mCBgChoAhYAgYAl0SAVH2aQMMCdWKnNi2QDwTErgw+KPYKiOXXXaZu+mmm/wrO+20kzfS8kPc6h944IFun3328fdjCiV/I/JHK/RlawH9WEpBiJcMvjJEoREKClEEcgMiBC6tiGHbNr5cjIkoskJCVexZfe3oo492W2+9dactsBrlTxvgYvFVGabUA9IcMyrpvOhzrZiEEEP5IvoLQ/HgpLe80t5RmqlzKYOATqP2Fqevo7icN2+eV5Jx3Qhcb3vg0u1YY1bvXOpNo3otxlmtIJZwqyJwEZ4oNPlqWtq7xCPHK664wk2ZMkV+Jo+Q/sgf5INQMLjh8lq2OhSiVfhc+Fv3OYJd+AweA8ETkS+3U3UeQy5b1YakLeo5BjD6Ewxy/EZxr0WM+NyD6AHBUhT5MSNSMwQuMcoVVUzrdOpz3T9CHr366qs7trGkn4ZILFvQ8nvSpEkOj03kUXs1aKZsdTtpdrxLGdL01/vylb3GQc51u2tE4NJ1r8x4p/Mr8TY66rq92WabeSMYXh/Deko4tCO+9qfOiYEtrCf14ku1DXknZcTEAwntV28FK+/IUZOSmTfMmDGjqbmShBsem6mPYVj6N2My7YEyiJFPIW+ffvrpnkzHe2JUl62RdVjawBjzlKgNi9oDV24fJX1G2BeljFk6rfo8p5wxvDcicNGv4MkQ0dsSpdq0TlPqvExbidVrTaZJGQmJWzyb6PVDkbVDKwlcOW0/Z95P/nW/GVtn6HFGz8Nzy7ZRfGXnR82MD1WnpZnxoVFaWlkOMQJXrN+jvsRIU1xPCR6nR44cGZ136Xekv9Uf0ci1MmlJrXd0XPo87FP1Pc6rnEenxmfJZxECVzPrxTBvZX6ncGU+mVpH6n451QfHCFzS/8XWSJJmTSZa1QSuWL9ZpX5G8lxmTJR3UkfBmC1Q8T4UE9lyXdZMqToQe5dr0raYA7O2RYp4oMerFroVJOUhl3t6nGYdy3pWX4vp/TSBK1Zusu6Tuteq9taKea5e1+h1B1iJhGtqdFG5WOfEB+lX68dS/YLMecP1T2zOmzv30QSuFF6Cmx0NAUPAEDAEWoeAEbgKYmsEroJA2WOGgCFgCBgChoAh0CURqJLARQZl2xUUB3iaaCS4YMftP4KyBwUQBkJRXsmXZJoc0ChM7jdS6KcUhEJGIAwMsni9Qgn27LPP+nRCcsCYjwiBiy3Q8CaFxDx3cF0rsrQhh3uNBDIBpAFEvsBslL+UwULiqjJMMTIQdj2FocTNkfSheAcXRLyS+B+1P7rc8fCDsoutuxDew/OHSG6dSxkEJFxtUKL+UW5s3YXXEIwiiJDyjMD1NoFLk+1oN5A86gkkJtkKrVG9LmugrBcv93R82hCn38MbE19wQ6LE45TUWZ5BSS8LZ/2OPteKe8hNKP2pRzNnzvT9nPZilUPgQik7a9YsHaU/jynjY3UeL4cQXyRfbClHOyad4mUMYtaYMWOihkRtDMLjCqRLCKKI9hroL9T+NEPgkvIPvfDFFNMSX+ooSm7aMuWBVy4xgELaIy4E70Lgxja4c+fO7fjimnvNlC14VzXepQgBkJ8hQSP1PDjhRQNiFBL7st7feOtP7njXTL+g48cojHdMiC94tYAQJCLeIYRUrD2SyDOpY6xt6Ge1QVXi4b4Qe6Tu6HfkXPcz4m0wd9ySMGPHZupjLLzYNQyhO+ywg99Smq/8RbSnEzGqx8gDOQSuZvooaecyp5T0lu0zcsu5EYFLewOsisAleSzSVmL1Ws8fU95fiEMM2V2JwJXT9nPm/eRft+uYQVvjqOf9qf6aMOtJo/hkfBQDug4rRnBvZnyoOi3NjA+N0tLKctDzxnr9HmVRlsCltyyr54FL1iGaRJyTFjwNM29EmPdor5b+YvCH/p1tFItIs/Po1Pgs+SxC4CKdrRh36+U/dx2p++UUUSNG4BKdTkje0GnU3pu7E4Ert5/WeS8yJurnY+cy7tXzTvn973/fDR482Em9zG1bjCvjal6RkUYfV/CM1m3V8wqFl3n6I+TXv/61X/PF1oz+gbf+lCVw8Vor2lsr5rmaUFV0Tc0HCKydkbJY58RHf1yEwCXzgCLr5Nw2ZQQuX+z2xxAwBAyBVY6A6KGZk2NDQ7/Ibiv851x+F01or1122WVF0Ye703NG4OpOpWVpNQQMAUPAEDAEDIEQAVH2aQMMzzRS5PCMGEq04U6UARBeUAwKOYvnRSDiCBkHr0tMOEW0Iv7ss892xx57rL/FNkkogoqKDidmWNFKLjGs6O0ayRPEpDD92muUELjICwpRJKXE2XLLLTvyIvEVzQvPiRESwhu4N8pfymCh46wqzFBBfeKJJ3baTknHKedsh8V7SEh+4tq+++7rt1LinDqy3377+e0Lw3rK/dw6lzIIECaiyzq23YmuQ2EeYsbhRvG9GWvxv1oJH0tfKqSUwaPeNon17oV5ZfGIQQl5+eWXfd2NpYXtBDHqoOCGXII0qteimCxqoIzFq6/p+MQQxxZxKEkhn8n2rvIOeYNcKN602NIA0lU9kTTTJ9Lm+OJei97GKIfAlSKN6vpBG6GOxuqgJhCk+iadh9ADl24HGDQwIrL9IEJd02RLrqUIXHgxgiSBxEgDmlRVRDHtA6rzRwkbe8YAAEAASURBVHtFgnS2zjrrOG2QQTFP3iBt4QUDBYj2FEjQGpecstX1r5nxLkUIgGyDkQ9h68/zzjvPn+s/5EuIZCGu+jk5zx3vcvsFyMZ77LGHjx5PaaHBGDIRxhBE5iLi/YBrse09aL/UNfIO+ZE+LNY2eF9EG1RjBC6eE3KWvCNH8VTEfIK+FMkdtyTM2LHZ+hiGibETLz0IpE7Iy1pos+CLkhCB6InhWMaYqghczfRRMs+pisBFPsuUsyZwsTUvdVSLeEPlWrMErpy2kqrXYniNzblIK4RP2T5NP1Nk7dBKD1xl2z59Im0RKTPv53ndf8fWGal5eKq/Jsx60ig+af9F50fNjA9VpyV3fACvRmlpZTnIvJF01Ov3uF+WwHXkkUe6mh2FV92FF17opk6d6s/1H018g3zPWI7kpEWv5SAL4IUnFMbMI444wnvXYl1+++23h4/431XPo1Pjs+RTiDKSGOn3ZU4g11sx7krYsWPuOlL3y2UIXMxrxKNvbO5DGrku5OvuQuDK1c/kjImxctTXhMAV1jl5BpI69Yy2Ih995LYtvU5JfYQhdVrmOJK+2PxL0oiehvU3wvusDxuN3TkELklbrk5Q0quPMs51hTV1LtaawFVmDV/lOjm3TVEWRx11lF8zcW4euEDBxBAwBAyBVYOAEbgK4o5S18QQMAQMAUPAEDAEDIHuikDVBC6trLzhhhu8RyqNDUp6jDcoJzFm4llKk6S0QQFllGxfJp6ndFj1zhsp9DXpQEgLY8eOdSjjETzPyJd1Eg/KDpSjGDURIXBpZR0KpRiBSXs4kfgk3CJHCEz8J3yIZWy5gzt3pIzhSMdVZZiiLCd8FOZspxQSNyRu7cWMa4Kj3OdI/YCohgJ00aJF3mMX17WBk99Ibp1LGQTeDPVNIy3ekZDYF5kYMXbffXd/f3UicJGhsE4JgSt2LyRw8YwoWDmnj4EMpQW3/RgOEI1do3Yr4RY1UOo4Y+c6PjHEfec73/EELfol0kib06IJVymFun5evCMQTkhqpX5j1JctMnIIXBBaSLNOJwZLDFuEz3UhXcXq/CGHHOLIE8L2r6HHNN0n67B0HsVozn1wo5/X3iD0sykCF8+IMhxlBGFq0USHkGgUq4P63di5NnzKfU3Q0umU+/RzkBJFmi1bjW0z412KEMD4yXhLPaBcIJ6AnRbIUXhRQ2LEOf0s582Md9J+Cadov7Dddtt50h/vYCz+/e9/z2knkXAZKxhrtXEk5gVO1yUZU2JtQ0eiDaopAlfM0KrJ29rAnjtu6TSF583WxzA8iFmESf2R7YzDZ3Q+xJAjRvWYATHHA1czfZTMTcS4Kekv22dow3iZctZk9QsuuMB7JZQ00O/T/4MvInWR81Sb5l5KctpKql4LgZW4YnM06fO535UIXGXb/vTp07Pm/eRbzx/CORP3W0kcisUn/WDR+VEz40OjvJdNC3jJO5wXHR94tlFaWlkOMm8kHfX6Pe6XJXDp7VVja1LC1F6YLrvsMnfTTTdxOSst2rsL8wQIuXpeSbh6i9u7777bTZ48mcsrSdXz6NT4LJiHZBrp90MClx6vcnQUK2W0wQXmXDnrSN0vx8YbotVlL0Qs7Z2ILd7QnWgJ573ynn4mPNdp0XOfnDGKsBu11yr1MzljYpj/8LesU7hO/acdaNGkq+uvv95v0Z7btrbZZhvv6ZfwWesxD9Gy8847O+JD5ENHqftcO+ussxxlpoVyQ4/EvEPPi1pB4GpFe6t6ngs2et5QZk2di3VufKRV6l+z6+RcnSdpMAIXKJgYAoaAIbDqETACV8EyMAJXQaDsMUPAEDAEDAFDwBDokgiI8hMlLQpztjLD0NtIkUNmYh64+KoTww6KIcLEM5B4qOHLsS984QveMMX7uB8/55xzOO0kWonPjTLbIElAOQpC0ozXATGm/fKXv3TTpk3zZAS2bEJJ1r9/f4nC/eIXv3AYf5BPf/rTflsnzlEYo2BC4Y4BFCKAeBzjfkjgEmVMvXwSL7iQtttuu81/OdksgavKMNleTrZNI4+UPV9wz5gxw+cXA8qECRM8DuPe2g6A51J1gHvacwW/CRNlOB4+tOTWuZRBQMKGJIcBBYGMxjaPGMZQfmJQpk6IaAMm12LG4VR8u+22W4eXE/GwJuHWO2oPSxADMLDXE7aJe/jhhzsMO6HBQ7cZtsKDLEGdROrdi+VV5wkFMQbgp59+2odFvf3sZz/b0Za0FysdTxUGSh9hnT86PjHEaVypn+eee24HyZQvlunDIHMiKNNDj0BhdGLI4jqKfvpEtlvbeOON3cc+9jHv3UneKboFqVb+8i7EG7wlQC6CFAB2ECUQTcqK1UFt4KBt0QcyDrD9B+MARE/pE2mDkPk4asGb2kEHHaQvuZhxjAc0MSo0Hmll+B133OG972H4YNta8dhHGFUQuAgHbxVCyOU3fSxeuBDaN/26SNheuF5F2VYx3lFWbNmC0EcxrjMOMZbrfgwlD+M85DqEPFIeUr7yJb6/WedP7niX0y/ocYr+iDkDni4Ryo7ti2ULY/GmxXgDHuIZCmIMhjTwIA2Qb8kz9RhvFOASaxsagpQRkz5APF7wPPX2xhtv9P0C+IK/pIN5gfSDueOWTlN4XkV9DMPURJ4rrrjCTZkypeMR5jXUH/Knt2ySeWVVBK5m+ijpU7ShkgzExq2OjEVOcstZkzBIw09/+lPfv3KdbYxouyKawFWvTcvz4TGnraTqtfZsx3gFcRJCwFZbbeXo7/FsIqLnP0XWDq30wFW27VNHc+f9ev4Qm6+kiEM5ZQvWjeKTsaQogYswc8eHVqQlZ3wogksry0HmjaSjXr/H/bIELt7RfTrjHp46aY/0uYceemiHd8qwf8tNyzHHHOPbOHGzpqBtQIxG2HITApeMnfStoUdZ/2DtT9Xz6NT4LPkM52fS74cErmbGXY2NJvtLnmNH3b7KrCN1v1yGwIV9irzLnI50XnnllX7uw9obQo2e84Zz8FgedFo0gatV/ViMwEV+cvrpnDExhoG+JjobrjGHvPTSS72+g/kl6yXGR4T5KiRIriO6/pRpW5osTR+A7ok2ueGGG/ryJI/IxRdf7O66665Oc1n6CubMQuJCf0Ga5ONIPJxTR5BGY3eOB65m2ptPVOSP7hO70pq6DNa5a3jgkL6N82bWybltiniNwAUKJoaAIWAIrHoEjMBVsAyMwFUQKHvMEDAEDAFDwBAwBLokAloJTwJFCdpIkcOzMQIX1zFKoagWQcGFIgsloAjKJ5RSoTcQ7oeKjQcffND96le/klcLHXW+YoaVmIKQgLWCit+iKBdlKPgMGjSIW15xByEGggcGApQqokjjvn4XDCSMkMAlhkSeR5makuOOO86TPkgDRuBmCVzEU2WYfCmKMVcrh1N54TpGbhTLKdFGW5558skn/ZeHsedz6lzKICDh83UiHuKk3LhOGeFdCAnrNXUZYwIkGinTJ554whO/eD4Vnya/QJzBuFlEtIGkyPNC5EkZPHSbl/DEo0q9e7G88r7+8pzfYIcIfpxrfPjdqN3mGCgJNyU6PjHEQc6iLQvpgnKGnEC6dd1ObZcaxqXLl3uEh0i9ot4QrvyOfWXtX1B/wj5Sbul+hmss6ulnRWJ1kHxBdhGFPs/SX+u8YhyU+5QjSn+9vSR9u3jM433SEfNEyL16BC7t1Y5nQ5H8VUXgkv6PeML+l/IX70PcjxnRqijbsCxzxjvSJ+2ac0TqJ+UGCSccm6hvUsd5PrVlEvdCaWa8y+kXQoxoj9QF8iTtht/UQSHg6W14ST/3EXmec02YibUNnhFJGTFDYo88L3VVfkP2xvCmJWfc0u+H51XUxzDMcBymndD+qFcaS+2NUOpiVQSuZvooMXaFBIfUuBXmX37nljNthS009bgnYXIET7mn6yP3BEfOEWnTb/6K/y3bVlL1mtCZO2P0byRdicBFWsu2/dx5v54/xNYZKeIQacwp20bx5cyPcseHVqQFXHLGh0ZpaWU5yLxRl2ms3+N+DoELL53kT/e19BmM3XKNsYYtFvlIQ0TqV9m0gBXzBb1eD+eoxBGSeSVeOVY9j06Nz5JP0V1I/NLvhwQu7ueOu3q+yPxXtn6XOGPH3HWkkO0JMzb35HrMAxfXtcdKflM/IBHJOKPnJs0QuAhb8OccKTJGNWqvVetnyo6Jb+Yk/VcTuNJPOU961luM5rYtxmDGKD1fpzz171A/gu5Ck6x5nnKXOkC6Q69+ep3PB0GsCbTocTI23skHMSGBOLe96bj1eSvmuWEdkfh0W+FauKbmWg7WzcRX5TpZlyl5CceWlM7TCFygZWIIGAKGwKpHwAhcBcvACFwFgbLHDAFDwBAwBAwBQ6BLIoByGG84YpwXLwpsDYeSAEltX5gicPHOrrvu6t/XCiauI2ynBukp5bkGLzcY/0Uwfs2dO1d+Fjoee+yxjq2LkJiiKaUgBAc8zJAGLShxHnroIYdXLryUiHcbjMUYTRAU1rzLF4daePf888/3XpZ4L0XggjSBIiglELZQfCKzZs3qMKzF8lfPYKHDrzpM8scXvrJlhI5LziHvYWwQ72VyPXbEW4wYErTHs9izZeuc9rDDF7S33nrrSsHSDg477LBOilIeQqlFHSaf0k64jgITb1ii4NXK7FR8Whmp6xPh1RNNhKn3nNwTAhdkHb6cDg0e1Bm+zNX1F/IPC8N692J5lTj1VhJyjSNtAu929C3ydTLXG7VbMVDGlKh4noJwhsTIEv5G8EfHpw1xeJbhy3lNYtKvPvroo+68887r8FCm78XOU8pa+jWIBGyzBQESQYFaj8jJMzq8a665xr3vfe9bqY6inMdbg+47U3WQLS3pW4ScShwIaYFkSd9Hnyz9+c033+zw0qaFNBMOEhoT9HO63saMR9pjgbxHfeHLcsqYMSskcNWrgxJG7Pjud7/bHX744f4W4xL9jRa8M0l7SPU/uiz0u0XLtorxjnghLey///4dZTRz5ky/lS33aO94jov1y2BLWWpCHu80ktzxjnDL9gsYnWirlH1MIOcwptCnaMHL0cc//vFORivuU6/xEAdhRiTVNuQ+fQH9O6K9UJxyyikOL3H03WwHi6dJMazLu/WMvWXHLQkzdWy2PsbCPfDAA71XiTBfPEv/LV4+5F0ZY2LbqDJHYE6HxPppvUXXb37zmw5SQm4fJd4awvGubJ/RTDnT7vAoqkmU5B/iE9sqMmdCQgJXvTbtX4j8KdtWUvVagmZsYY4iaae/AEs8cn3oQx/ycwO9DXKRtUPMA5d+7w9/+IP3NEsatAczvFficUJE+ueQnCfvFW37ufN+PX8oOw/PKdtG8eXOj3LGh1alhbIrOz40Sku99VCz5aDnjfX6PfKVQ+DivVTfxz3W7GyTxppQSzNpoT4wH9QEEAmb+Bg7+d9IqpxHp8ZnyWfYv9cjcJHunHFXE7jYqpItK4tIzjqSj3lkvpEi9acIXKSJbdEPPvjgleYizFMgFcm6NTYHD/NUb4xotv3E+s2q9TNlx8Qw/+FvmTtAcMKTMusSLcyJ0PmE81GeyW1brBMg24juSeJjPGYexTyCdZEWvX2zvs55rE7pMTim99Nkn1i5CYErtj7PaW9hmvXvque5Orwya2pJU1msm42vqnVy7txHx8+29eKlUfCwoyFgCBgChkB7EDACV0GcjcBVECh7zBAwBAwBQ8AQMAS6NAJsi4JxHvIJRqUqBIPPFlts4YlUkHAgFEydOtV7KKoXPu9hiMBYiDcjFLHtFgzFYEJaUMJBxEFRhpAXyGGQHdgikHxp2WijjRxeK0g/W5tB9sBg3JOEurTJJpt4L2HrrLOO39YLHPgfKhmrxCW3ztVLAwrT7bff3hsz2OKQbYRkCzLewzjLV9YYEflSWhOS6oWr75FuCJExpap+rh3nGIvYEgyjLMp+LfXu6ef0OeslyEnUB0iKbCMCGQdDUFcWlP4YoNgeFLIKngwwqlDG5KGsjB492oc3cuRITwBB6a4xoE8hHvoTyDf1RCt/IRbQZ9Nn4aUAjwOQROl7yghtlnqOQYIw2M5Rlz/luOmmm/r+j7SHCltNzGLLDrafzBXpYzEggjX54avurirNlG2V4x2Gaki51CsMumEZkU7GNbZeoZ5BRGIMa2Z8yh3vcvoF2s6OO+7o+1zqK9vQMGehT5bxOVZHyC/tmHzivZK2QV/UCiFdjP/0d8wbaAeNxryqx61m6mMKExkHyRdGH7bOYjx84IEH2tY2m+2jUnnLuZ5TzhhhqYfMDWl3EF0bSaM2nXo/t62kwiMdzOVobzLHYUsrcKhH2E2F167rZdp+M/P+nPzklm1OXEXeacX4UCTe2DM540MsnCLXulo5xNJMOxtX88TDPBHCCO2QeRHHVgn1gf6KeRjzX+YUzAvLSNXz6DJxN3o2Z9wVkkuM4FIvPhk/wbJV68gwfshCEN9Yt6FLYawO9RXhOzm/291+cvvpqsdEwYr8Q1Ciz+JjFzxLN5LctsUWiHwkQFwyl623NhoyZIif87N2Yx7K3Jd+gzVeuyWnvdVLY5Xz3CrW1GWwriK+KtfJuW2qXvnYPUPAEDAEDIHWI2AEroIYM3EyMQQMAUPAEDAEDAFDwBCoDgG8Peyzzz4+wPCL++pisZAMga6DAF9KT5o0yXuRgeRoYgjUQyCm/K33fKvvQUjAexVf6se2rGl1/N05fBvvunPpWdoNgdUTAQi5ENMx+uJZIyR5YvDDAwxy9913u8mTJ6+eQFiuDAFDwBB4CwG8FI8ZM8Zpz2sGjiFgCHRvBNq9pm53fN27dCz1hoAhYAgYAikEjMCVQia4bgSuABD7aQgYAoaAIWAIGAKGQAYCfMGIByO+GMWjDF/q4UUEZamJIbA6I7Dzzjv7LWPwCMMWgLQDE0OgHgJdRflLv42Xp0MOOcRv2UKaw23A6uWjp96z8a6nlrzl2xDoHggcc8wx3nMHqb3zzjvdJZdc0pFwvICwpS6EXeTUU0/N8krZEaCdGAKGgCHQxRH4zGc+473oPvXUUw7vgyaGgCGweiDQ7jV1u+NbPUrJcmEIGAKGgCEQImAErhCRxG8jcCWAscuGgCFgCBgChoAhYAiUQABlBtt34clF5MYbb3RXXnml/LSjIbBaIsB2W3vuuac3kLJFn4kh0AiBrqL8Pe2003xSpd/GW8vXv/71ji22GuWjp9638a6nlrzl2xDoHgiMHz/ee9iSvh3PigsXLvRbh/OhhcjNN9/s/vSnP8lPOxoChoAhsFoi8MEPftBvXcf2iSaGgCGw+iDQ7jV1u+NbfUrKcmIIGAKGgCGgETACl0ajzrkRuOqAY7cMAUPAEDAEDAFDwBAoiIBWZvDKM88847dtKfi6PWYIGAKGQI9BQPeXeCx89dVXV0neIXCJgX/58uXu/PPPd9OmTVslaelOkeryI9023nWn0rO0GgI9A4Fdd93VHX744a5Pnz7RDN96663u0ksvjd6zi4aAIWAIGAKGgCFgCHR1BPSarB1r6nbH19Xxt/QZAoaAIWAI5CFgBK6CuBmBqyBQ9pghYAgYAoaAIWAIGAJ1ENhggw3cjjvu6LdNfPzxx92jjz5a52m7ZQgYAoZAz0Vg1KhRDg8pyNSpUx3bb64K2WeffdywYcPcnDlz3PTp0938+fNXRTK6XZw23nW7IrMEGwI9EgHIW3gIpc+ir3/hhRfcI4884h5++GE/X++RoFimDQFDwBAwBAwBQ2C1QKDda+p2x7daFJJlwhAwBAwBQ2AlBIzAtRIk8QtG4IrjYlcNAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQyAfASNwFcTOCFwFgbLHDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUOgMAJG4CoIlRG4CgJljxkChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGQGEEjMBVECojcBUEyh4zBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDIHCCBiBqyBURuAqCJQ9ZggYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAKGgCFgCBgChREwAldBqIzAVRAoe8wQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAojYASuglAZgasgUPaYIWAIGAKGgCFgCBgChoAhYAgYAoaAIWAIGAJtQmDs2LFu4sSJbt1113W9evVyP/zhD9sUc8+LZv/993cDBgxwTzzxhJs2bVrPA0DleN9993WjRo1yc+fOdTfccIO6Y6etRGCnnXZy48ePdy+//LK79tprWxmVhW0IVI4A49Tee+/tw/3zn//snnvuucrjWBUBvvOd73Q77LCDGzFihJszZ4674IILVkUyLM6KEWBOdcABB7i+ffu6GTNmuOnTp1ccgwVnCDg/d6+ynmHHnTRpkg/33nvvdbNnz16tYB44cKD7wAc+4PN38803d+Rv99139/PSefPmuVtuuWW1yrNlxhAwBAyBnoiAEbgKlroRuAoCZY8ZAoaAIWAIGAKGgCFgCBgChoAhYAgYAoaAIdAGBLbcckt3zDHHeCOGRPelL31JTu1YMQI/+clPfIgQuM4444yKQ+8c3OjRo93gwYPdggULuiTJ4dRTT3WDBg1yr732mvvGN77ROfH2q2UIgPXIkSN9vTjppJNaFo8F3BmBoUOHenLOihUr3MyZMzvftF+FEXjPe97jDjvsMP/8b3/7Wwe5YFVLs2V78MEHe7KE5ANj0ze/+U35acdujEC/fv3cD37wA5+Dxx57zJ111lndODeW9K6KQNX1DELpUUcd5bN7+eWXO8iyq5Nsttlm7vOf/7zP0u9+9zt3xx13+HOZly5cuNCdeOKJq1OWLS+GgCFgCPRIBIzAVbDYjcBVECh7zBAwBAwBQ8AQMAQMAUPAEDAEDAFDwBAwBAyBNiDwmc98xr3jHe/wMS1btsyTOr71rW+1IeaeGUU7CVwYjTHqvfrqq+6UU07pcoCLocwIXO0tGiNwtRdvie3LX/6y23DDDR0EruOPP14u27EkAl2RwNVs2X7ve99zYjdZtGiRe/rpp92ZZ55ZEhl7vCsiUDWxpivm0dK06hGoup4ZgcsIXKu+VlsKDAFDwBBoHgEjcBXEUBYiBR+3xwwBQ8AQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMARaiADEnuHDh3tSwde+9jW3dOnSFsZmQRuB6+06YASut7Fo55kRuNqJ9ttxNUvyeTuknn22OhK4TjvtNO8Fc/78+c4I1KtX/WYLxSOPPNL16dPHPfroo+4vf/nL6pVBy02XQKDqetZTCVxs7c02vS+99JK75pprukTZWiIMAUPAEDAE8hEwAldB7IzAVRAoe8wQMAQMAUPAEDAEDAFDwBAwBAwBQ8AQMAQMgTYg8N3vftcNGTLEE7e++tWvtiHGnh2FEbjeLn8jcL2NRTvPjMDVTrTfjssIXG9j0czZ6kjgknHhqaeecj/+8Y+bgcfeNQQMAUOgaQR6KoGraeAsAEPAEDAEDIEuhYARuAoWhxG4CgJljxkChoAhYAgYAoaAIWAIGAKGgCFgCBgChkCXQmDixIlu44039mm6+OKL3euvv96RPr7WPuCAA/zv2bNnu2uvvbbjHicYQrbffnu3fPly99vf/tYtWbKk4z7X99xzT+8Fiy1QFi5c6F555RU3ZcoU97e//a3jua222srtuuuu/vf06dPd1KlTO+7pEzw9DB482LEt3SWXXKJvdTrfYYcd3GabbeY4Ei/bet1zzz1+C8XLL7+807OTJk3y2yyus846rm/fvg4vIRiaf/e73zm2e9IyevRo9/73v99fIhyeJ02jRo3y4Ydh63dHjBjh+Podufnmmz2pbPfdd3cTJkxwa6yxhnv++ed9vu+66y79WqfzoUOHur333tuNGzfODRs2zKcVrxd33323e/HFFzs9q39svfXWbo899nDrrbeeGzRokMfhueeec1dccYV79tln9aOumXSKof6JJ55wZ5xxRqdw+UG6d9ttN7/VGuVInh966CGPXYj1Si+/dWGvvfZym2yyiS8zvDJQ7wiD/Fx99dWdXitTtp1eTPwYP368bwtgNGDAAIfSlHhvvPFGN2vWrE5vaQLXd77zHbfTTju57bbbzo0dO9bjTx2jHdCmYrLBBhv4uKiXlBntivjuu+8+d8stt/h86/f2339/X76U53XXXecOPPBA3y5J58knn6wfraQcdIB4X6Fukz8Ik5TLggUL3Jw5c9xVV13lMdLPkzeef+ONN9zkyZN9XZY+iHoxd+5c783l+uuv1691nPfu3du9+93vdltuuaVve/PmzXMPPvigx+aEE05wI0eO9PGfdNJJHe8UOdlvv/18e1x77bW9Rxn6mUceecT96U9/cmzBGgp9y6GHHurx5B0Ezxa8Q75D2fn/s3cmcP9OZf6/ZaZSjUojQiFiRIlJREKElCVrWUpJlK1lmmKmUqF9poVI1pCUrSIhS5YQIVtlCZGmlKlm2mf+/X/vk88z1+849/297/P9Ps/v+fG5Xq/n+d7bWe7POec6y/W5r7Pmms3KK6/cPPDAAylO9N5aa63VgAcY3nvvvcmDzXXXXTdX0NpwMZLnP//5U/WP62xjd8MNNzTXXntt0o/xWR0/9rGPbbbeeuv0fugeyos6eMkllzSXX355eowyoE2vsMIKDc8jxIu3Q/oDwvSRvjpqtdVWa/hD8PZDWiXZdtttU12kHlLHoiwzQA+No/OVJrp6u+22S+0TXY+u+/nPf57yT78UJSdw0Z7pG1dcccXU96Ezb7nllqRzYrh4PETvdb3fbbfd1tDWasuW9sS7U8cR3vv6669P/eu3v/3tqSwPbUeqA9RF+mnON9lkk7RN4wknnNDceuutza677priJx10M3mgDWC/oc+89NJLU3vjIeov4xj6R+oLz+OdhrHKUOnbRy+11FLNxhtvnKKnDzv55JObP/7xj3Mlx/iFLaDRPfQXtDvGU+ST9su7kW/6FrYvZXx19913J90CNm0yRBf0wXr77bdPSVG2YJvLkPTG1XV9dFaeP+r4RhttlMaMjOXQXdR99AtjmZIw9nvlK1+ZwlCnKEPGYLfffntz1llnterUUly6NqTdjtuHKs3S75BxjsIPbcMK17e8KKOuejZ0DFJD4Npyyy3T+PjOO+9MY3jiQK8sscQSSZfQHpm/IMw/GKMsvfTSqW7Qt9N+qR8lqamDxIMeoc0wJiYO6ut3vvOd5tGPfnSz1157paTQkeqvGXcxfmXcSX5y6Ttny8P53AgYASNgBOYNAhrvMWelL2QOTn/AH8c675u7BeZ0bH/p+/D89JwJXPNTaTmvRsAIGAEjYASMgBEwAkbACBgBI2AEjIAQ2G233ZKhkPMzzjgjEUR0b5tttmkwKiMY49/+9rfrVvrdf//905YckKS4hyELweMVZJU2waDxsY99LN3G+LDvvvumYxag8KKTC4YKyBnIr3/964btEdtkv/32myKk5c+85S1vSZcgiuyzzz7JGJo/wznvCpkNY4gkGn0gsmE0ZmEMwah+5JFH6tGH/GKIBWcEIw/ELYWND2OA+cxnPvMQwghp77LLLmlxLj7PMdifeuqpU0aaeH+nnXZq1lhjjXhp6phwEMYiGW6cfHYRuDB2YnwuCeQkCF/UiVHyzne+s1hmxME2mUhN2Y5K97Wvfe0UcSR/FhwhTUVyowhcEAIwwENsyoU6BmYY46NgKITQ1CYY+g866KBEptEz8jYHmQkSIoZDieo855MqB8WNAZa0MRi2yYUXXthEcmMkqYAZRAYWmnMpEQFJj3IWYSqGAReM7094whMGEbh4/q1vfWsybMb4dEyc4I3ekWBAx0BKfkqCHjviiCPmqtNvfvObExmGcr/sssta2wPkr/PPP38q2tpwREC5EH6ZOaSlkkA6QA/nBEp08p577tlarpDsDj744GaLLbZoIB6U5N3vfvdcdbT0DNeG6ChIr5ChEIhw1L1cIPCpD8m37Rta/8fR+eQLchD5Lel67kM2op6o34xtAzIMBLmSQBL87Gc/O9etGr3X9X6QETH2l6RP2X74wx9ORNc8PMYmwiM17QhyFvlG70LypB+WQISClPjRj340XYJswdgBgkkuEDMhxkGkzAVdQv2m/vSVIX00BCDih2CLUA/o9yUQwRjjSC/yXpD9DjzwwESKo1+BaA/hKBf0y/HHH/8QcmONLhiCdf4ONemNo+v66qyIF/gxXqQPKAlkWrCMAumRsW6pTvEc/QTbhvYlANa026gnhvah8V3y46HjHMLXtGHCDSkvsFabzutZzRgk6j3GJoxRRonGdOgv9AP9TC4Q/5gLMIbLBX3FHAHdHaWmDhIeghjzs1LfwtgJfJFI4NI7QPZk7hZlyJwthvOxETACRsAIzDsETODqib0JXD2B8mNGwAgYASNgBIyAETACRsAIGAEjYASMwKxCAK8wu+++e8pTbhwRQUsZ/vjHPz4X4YQtkTAy4lHkkEMOSY/h/QePBgjGRL46h2ADoevJT37ylMEB46u8Jn3oQx+aIkN84AMfSJ5sUgQP/oO8hEcKBHJDycPNg48m70MQkTCMkDcMJ+QPIzlGZUSGUI4haECiwSiDsVdrPIT7yEc+kjwJ8Vw0+nAu4Tm8T+SGPt3nNxKjdB1iD1/C46UkEnzwJnTsscfqseQ553Wve93UOYZByAsYkGI4DHmRRBQxI4+khVGecoAAI8MPnrjwBoWMk882AlckaKgsMCDh3Q3vUgjXP/3pT7d6vEgPzfmHwQqPMHimIv+UKWQSiEsQIZCask0BW/7h4WTHHXdMd6nPeJaiPpEH6ovksMMOS547OJehTPd4P/IJUQYPKjL+cv6ud71LjyWjG8RC3o0weFTh3fDyQHnzdS0SCZCci8DFcRTwYXs7ZJLloDSi0Q9CAYZDSAnkNxrFad94K0Oi8Vnx0AbxLkTbg7wgwZsJHs4k6IbYPmkH4ATRRPWZZyE49PXAFbFT+VI/IT2pnGg3pI3gRYljkSsga6E/MCTTtkRmQ5+gP0XOETkhRfLgP8LSninbSAaLOrA2HEm85z3vmSK78W7oAOqQPA7yDHWQ59DRCOVG+sKT63gAgnQCMVDXIZRgsJZXD+4jlCMCiSEnhqUb4d9QHQXmEM6EPWVMWUd59atfnbyScC32FTX1fxydTz2lbqktU4eoJ+C73HLLTb0D3lEwsiOltkE9ou0QLhKqcrJ1jd7rej/KnDpJmjVlCwGQ/h49j1CP0GXoz+OOO666HYlURJy0fdVHjk888cREXBLZg2cQ+lp0Nn0thJlcMIChg+L4hHHLoYcemj9aPMf74NA+Gi898pJDpJA7IH8gkagc8xHLOD045x/6Az1Cv6L+ASwgyXFdUqMLhmCdjx1r0qvVdUN0FmQ4JNfj6CrqJu029ieQAD/5yU8KxkTmJT1EHlvBHcKMCHkiuE4F6jiIZdp3LFrSE3370Las1Ixzcgz79oVDy6uLwFUzBol6byiBS/jRxtAp6Maol3UffQeZTx49uY4up21Lcvz61kG8Mu6www6KZsq7MforjiN4oA+Bq2bONpW4D4yAETACRmCeIWACV0/otXjQ83E/ZgSMgBEwAkbACBgBI2AEjIARMAJGwAgYgVmBAAZQiFkYxXPiA9dlFCSzbLtx+umnp3yz1cree++djiFZQLZAID1BYoAwgBEvbsnIVj8ilGAghgCGQJDBgIREg3a6MOefCDEYTSBFjCIGEE7EjPyd2HIEIz/CwheeMGJ8kVQQvbxEow9hMd5hdO3jOSonRuVGQbYQe+Mb30i0ySiNxx8M/gjEOBmdozGGe7wL2zhShhiM5IUKEgyGQa5TDhggwVuCVy4IDQgEE4y9lNM4+SwRuDBs4Q2HfGBEJx/R49RrXvOatNUf+Yj1gfMuwUCPUS/3xlZbtl1pUV+pt0gkaXEe60Q0YKu+8gzlQj0XgQkDG9sqythL/cPgi+Ata5kHvSXhkYU4JYSjTtO2YllzX3WdY9rIRRddlMh8PIdMRzlAHiRdBE9OvAdpS6J3pkguzI3PP/zhD5ujjjoqESgIG73+sYWcyBjR+xJtGt2kNoJXFLxoCdO8zStP+W+ME6zQV9IF4Ew7FCFLXodop7RXBC9JIg5yjq5ky0p5CGM72JNOOolbyRMW5EMEnCAXicDKtUiWjfo0khqGhMNDB1sJIugx8JIupu2QnvIZywfijTw/RfyJB28r8sIYjdFqI+SPcugjtToq4hHzrTSlL8mL+ora+h/bN/EP0flsIcgfwvZa0dNhJDDEupq3DbbHjVtAxjgjibNW7/V5v5qyTS/94D/1C7Tzww8/fOpWbTuKpCIig0QIeVpkpUj24D51mDxAcEHoFyFyScAXnBHsO+hmxkLEJ09uerbtV3WO+337aJ6NYw3aE7oDnYSnOASdRB6kx2Peqd+06Tj+ILy8coEL/RVSqwuGYB37v9r0YtseoutqdFZMKyfN05/g8VXERfpziKkQbVUncpIWdQavaSIgQ9ShTLuktt3meqJvH9qVF7Vznuk7zqltw0PLK7bpWM9qxyBR79UQuChX5jkaf8R2DH542T366KM5TBLbLfWKsTBSUwcJF/XNxRdfnLZN5TrCVpN4fpREfaRxae6Bq2bOpvj9awSMgBEwAvMOARO4emJvAldPoPyYETACRsAIGAEjYASMgBEwAkbACBgBIzDrEIgeH/AIBIkhGusxpkHCwYiFEQCJnk4weuJdA8HLFUSGm266KXnaSBfDP5HCIH1AkEAwsGtbJYzkECkkePMRMQkSDN58+ohILdFATji8y2gdJ3oGinFCRuHreASjKMSHaPTBAAM5QEbhGLZ0HIlRhCXO3LgXjUAiyrHtoIy5uVFI6URjGJ5N8AYWt8WMns4Uht899thjavsoGbFq80l8MtTHre/wMIKnEQTvKNdcc006jv9kjKKOUfdyXOKzOm4jcNWWreIt/VK3ZZQt1RfeEUIGnlJOO+20FIUMZZyoTGLcb3rTm9L2XVw75phjpra7wohKfYeYxrvkonYKVpEoo7rO87lBj2vTUQ4QOKl7CFt4igSRLsz5F7c9jQSWaHxm4VntXuEgTGFQRCJBL5Yt3p3wThYlesHJ23x8Lh6jf+RNhW08qbtR4pZ7X/nKV9LWh+QNXUjbx/tHLhDtpKOoy/K4EY21pbYcvWrErbtqw5EH8kJdwYAMllEghdL2EOnciD3ETzxc5e2RNgrBAYHMANFFxv+8XqaHWv7V6qhIHI59CMnEviISnGrr/zg6P5JTS0Qz6haEPvoWeXqKbUNlEuGDoCgvkm1to6SjiKO2T6sp25hn9QuRwAUho7YdRVIRHpOEh9KMZA/qI3VUxEWeif1siTQsElTfulzbR5MX9Ag6SGMNSFfROxvkUEiikkgE+da3vpW2vNY9ftFl9AWQiWi/IlvW6ALiG4J1JNbUplej62p0FuNT+nWk1AdxPZJ7GVPRj8fxUanurL766skjIeEhJYvgw3lJYp82pN1GPVHKf8Qk6olSHnRt6Dintg3HvPXtY+iDROSO9ax2DBL1usa+wqHtN47p4piN52O9gGxJn4/+kLCNLvUJUZtmTFlTB/FEjA5DSvqP67Es+xC4auZspGMxAkbACBiBeYuACVw98dfCX8/H/ZgRMAJGwAgYASNgBIyAETACRsAIGAEjYARmDQLRsweeQiBcbL311s2LX/zi5KHpjjvuSEScaBTUFjl9yBIQtCCErb/++s0yD3oYyo3v0ejAsQhh0eAKUeSyyy7rhZtILXn+IH9gOO0ybOFZDOMQwpf0EC6i0ScSLPpkJhp4IJ6Qh1zilkoyUkWSA4bwu+66Kw+WPJdpi7/vfve7zQknnJAIYnjiwIgE+a1ENKMsttpqqxTftdde23z+85+fyxA1JJ9EIkN9JHDJQBm38stfgO072cYTEdb5M/l5G4Grtmzz+ON5JLrxHt///vcbjOeUUZvI2NdGAohb1kQDWyk+jPK0HYzDz3/+81PdzeNVXec6hAt+o0xXOcQ0OIY4wFZeeCzbbLPNpoiSbQSuEgmBeOQRIrbdPmVbCkd8baI42+onxmZt3XrzzTcnDyz77rtviq5LB2AQhWwTyymSEyCxRk90RIg3LBFXo2evmnCUg7wbss2TiFo5DiKwKZ9sA6b3yz3MKCxeaNBVYPad73wnvWMNyScSZYboKPIR8ZWXLa5H0lQkTtbW/3F0fiTkkTfKG2IuZJCSPuaZSMwA2y984QtcnkvYQhLPQEPbRm2f1la2a6211hQJNWaQfKHTJOoXIoEr1rOh7SiSithqGO9JUSKBCwINZR+F7Z3Rv8iFF17YQOCIgoccbY2LR8RRUttHK150OxgzJomivjxeiwSuOEaKz0CIZbyF0K4gjNfoAsIPwVpjllrdQ3o1ui7Wpb46C52+8847k2QaTzKuzCWSJdmmEuwjOZfnGaPSt0FelhE1j6frXP3P0LFo1BND+tCuvAwd50Tch7ThGK5veaHvSgSu0vv0GYNEvT6UwKW+MqaNXVh6pkSqevnLXz5F7DvllFOaK664Io3laupg7OPaxstsa0yaSBxfalyae+CK76LjPnM2PetfI2AEjIARmDcIaOzBxzSQ0xlL0g/yx7HO++ZugTmD+7lXEPqGnOXPmcA1ywvI2TMCRsAIGAEjYASMgBEwAkbACBgBI2AEWhGIBggIKp/97GeTNyQ8reB1C8MzX5EjkA/YlohfFoZK3mQw0rAVGiQiSBAlyQlckdQiD1SEk4cmCAMYV/ntIyK1REM3ni7wRILELYby+KLBRduZRaPP2WefnbZAy8O1nUcCV5thHiMVxnlE2EBOWGyxxdK1nJSTLj74j3JA9E4iOOBJDY85JaFsIFEg2iqtNp/EIUN9JHDJ2xr32/KvvPPMGWeckchRHHdJicA1Ttl2pQVOGMIhBkThfWgbkN+or9p6j2dkKIsemGLYSCKIBjaewTMD3u2WmUN0xIgc8VEcuRFRdR3vD+Q1l+kqB9LBs8SGG26Y8s3icUnaCFxnnnlmc/GcLYBykQcXtd3ooS/WrzzcEFJpyQNVHl9+vsEGGzRbbrllutxVVyMJRJ5VIjlBXg5j/JEcEMkuNeHiVrWk0aft4TUE0pFInfm2YjGv+XEbySd/Lp7X6ijiiN4foy5WnLl3tNr6P47Opy3gwUxkmvjuEDbwUAmBKHrpicSMNmJBTlIcR+/1eb+2ssX73JJLLhlfKx1HkjcX1C/EOj1OO4qkIto7nsqiRAJXSVe85CUvadjeFRFZPYaPbbcPgau2j45pxm1juY7HMDyH5e0WEhHbP+b6P8YVt7XDsxt9AmUoyePU9djPoAvou4ZgLQJXre4hvRpdt34govfVWfo4Qe8+CpM4joJ0A5E6F0gxjL8YC+CNc5SM026jnujbh47Kz9BxTm0bZnveoX1MbNOqZ/F9ho5Bot5r07Mxfo67xnSxLCP5WnHE+YQIXLV1EK96kD4Rxu/Ru6DSi1uyx/Gl3qFE4KqZsyk9/xoBI2AEjMC8QcAErp64m8DVEyg/ZgSMgBEwAkbACBgBI2AEjIARMAJGwAjMSgS0xRIGAQwDMnpjkPr6178+tS2YDFSvf/3r03t87nOfa/BOI4kelXSNX4xkeCzA0wwiklI6mfMPQgV5wJCo7aPw5qMtyErGWIUt/YrUIhIIz8Ttt+R1qhQ2GsguuOCC5mtf+9pcHrj6Gn0UdyRGdYWVR4YHHnggbYWkMlE8o34xHGK0VTxdnh2iUaxE4BqST/IlQ73KiXIkH0OEbeouuuiikUFKBK5xynZUgk9+8pObHXbYIXkeKpGUIC188pOfbNhWCZGhjC9iMcTn0kbgwigKuSAa0wlL26EeQ/Ihfc5LWyiWCGPTVQ4QM9ET2l4yviNeX3h3yAZIG4ELD0MQGnPJCVx4aEOvILfccktz5JFH5kHSucgUsc0XH5xzMXoCufPOO1P5tT2r69tvv32z9tprp9M27xfcjESEEoGrRAzpQ+DqG+4FL3hBI698yvuoX8oSQ7Pe76yzzmogr/aRNpJPV9haHUWctMf3vve9KXp5GFt66aWn2kT0CDNO/a8x9Md3hpTLVokQPiBjloS+hT4Gif1OW9vICVzj6L0+79dWtuMQuMZpR5FUVCIwxH6NcQnjkyiRwPXFL37xIVu/DiVw1fbRMU+0OTCRtG0VLQJXSc8rbCQ3oifx4FijCxgHDsFaxJpa3UN6UW/21XWxLvXVWW1jVGGY/+Z4v+hFL2o23njjZuGFF84fTefConjzwYvjtNs+eiLvQ7vyontDxjkR9yF9IV59h/YxsU1HbGvHIH30njDRb9eYLhK49AGMwvFbInDV1kHpAOIttRGuQ/DS9ql9CFxteRk1ZyMtixEwAkbACMw7BEzg6om9CVw9gfJjRsAIGAEjYASMgBEwAkbACBgBI2AEjMCsRCAa6yCj7LfffimfeIW69957E7kKQwVeh9gmBCNIvvVYNN6x+I/XjWuuuabhq3QIHYiIYTmBi3siYHCMd43NN9+8WWONNThNhthIFEsXO/6VCFzR05i8VZWiwOi+3nrrpVtsLQjZq8boo7gjgYstIEtb9mCMwjiPaBuWiAcEKYgxXQLxC/KKjPzRc0QeLhp5brjhhuaYY46ZawvFIfkk7pzAxTW2bsKNP15APvWpT3GpU+67775eHtZKBK5xyrYzU9lNto9jC6YVVlhhiozII5E01GXs49kSgQtDO9vwiLxFm6Pe4VVEXnrwrrXEEksMInCR3nSUQzT64fGIfNLWIUNh8I7ky3EJXGzbJyIc24iqrvFuUbRVXiyLeD8eQzxjGzJEBMZ4v3Qcy63LAxcetvBeiEC0QxfWkBMIXxMukpl4t5NPPpmoWgW9Amkkbr0kz4OtgcKNNpJPeOQhh7U6ShHJ2xrYQvJlq130LEL7o5+S1Nb/cXS+0tYvnhTpy/COQttQO+c+295hhOlDzBBuquPj6L0+79dWtpA9GA/korqk62qr0QPXOO0ojlNmA4Grto8WPpQfpIycGFwiM4u8kY+7FBe/UV/QT0IiFNl3iC4griFYi1hTq3vyvJfIKSWSa43Oih7P+CChRCImPxLGrmyjmAv1H3Imeof3hmgkGbXd9zjtto+eqCFwKe/8jhrn1LZhwkFoQvr2MW0ErtoxSB+9lzIY/nWN6WoIXLV1MBJMITHzkUYu1EltzziKwDXunC1P2+dGwAgYASMwcwiYwNUTawZdFiNgBIyAETACRsAIGAEjYASMgBEwAkbACMyvCLC1yZ577pmyD3lkqaWWauJ2SLvttlsyVHENbwl4H4BwI9IRAffee+/k5YrjaDjgHIlGqxKBKxqm8MS05pprJs9c+ZZYf42t+3+JwEUIeX5p847EM9Eoe9BBByVvYTVGH+JCIoGrjYACKQgSAnL55Zcn/KKBim0t+bo/F8gteINArr766rSND4QAtu6C3ICRG2N/LtGAdN555yUva7X5JG4Z6uWBi2tgBzGJfOARoLT9JWnyh+D1S0S/dKHlX4nAxaO1ZduSTDJ+yzMKZCqIiFHwMAWRBJIaAsEKslqXsY/novFT7SRuBRXJDjwv0RZx4CmjPPdU13NPIQo3HeUQ80L6eNeLErdYGpfARbwq29L2P9zHyIsughgjcgvXu0RxthEd2ZpJW8d+4xvfaO64445mr732SlGyBd5RRx31kOhJH9IrhIyotyKxoi85gchrwpE2RFlE3vzSSfaPrS8hFoEphDRIiaSHtBFc99hjj2allVZKz0g3tpF80kMt/2p1lKKLxA30xqabbpq26y15Hayt/+Po/Je97GXJUxikmW9961vKdvqlfNAb8l6nrfxi/9fXAxcRqh5PR59WU7bxZdUvRJ0GOaS2HQ0hFc2EB67aPloYRbIndVd1Ah2f61URuAjbRt4QiVUkr1pdQBpDsBaBa5z0anRdjc7C+6K8krVtaY0eR/dDgMOzJoR2xmgrrrhi6uNLJPzXve51zaqrrgp0aaxwxBFHpOO2f7Xtto+eGELg4h2HjnMgrNW04Zryol0w5kNUzziuHYPU6PWuMV0NgSsSp4bUQeYI1EPk9NNPT1t2ppPwj7pN/IjGlxzrHeIYatw5G/FajIARMAJGYN4gYAJXT9xN4OoJlB8zAkbACBgBI2AEjIARMAJGwAgYASNgBGYtAvKOpQzKExTnq622WvPa175Wt9IvWytC/pHgCQXvIgiGW8hKUTCIQYpASgQujH+QHzCeQUbRtlN49zn22GNjVCOPRWrJyRy6TgSf+cxnkkEoRgbxCW8/yoO2cKwx+ijeSIyCAAehICe8YJDFqwmCYe/uu+9uIqkHUh3Y5BK/yD/ssMMS8SIaEq+88sqGraKiQDgiD2xbiWD0xctTbT6JQ4b6SOCKxqHzzz+/Ofvss3l0SihvDI2QbzA48y4lktdUgAcP2ghctWWbx69zcKJNUBe0VZvu6Te+ozzCyFDWRqgoEbi23HLLBtITctVVVz3Ea1IkPQwlcMU8TqocpCtK5DzwOuCAA5pFF100vc8kCFwi4RAh3uLwGhcles3L23x8Lh5HQkS+FSzPxe3RaCOUJ/WV96OeohsgaUVhi6itt946XdKWppzUkBPGCSfjNnHQXiASRYke+KSL8QJION6PcoVcgh6WsP5NG+M+REUIi4hIPhyXyGlcz6VWRymeSNiDgId3HgSiHX9Rauv/ODof726QcagnkFfBM0okOMqbWx9iRu6Bizhr9V6f96sp2/ie6hcigYt+vbYdDSEVzQSBq7aPBqNIQqQ9oTP32Wef5M2J+/TJ6B1J1Fdxm1Ddj1vN4oGOfgip0QWEG4J1iVhDHH11D8/W6MganRW3YEV/o8fy9hm3vdMYivEv42CkNL5lPHXIIYek+20E2HTzwX+17baPnhhC4KoZ5+Bpr6YN15RXmweu2jFIH70Xy4njrjFdDYGrtg5G4leJrMyYmj6CX2QUgWvcOVtKxP+MgBEwAkZgniBgAldP2E3g6gmUHzMCRsAIGAEjYASMgBEwAkbACBgBI2AEZi0CkQxEJi+88MLkFYnjSK7iHIHoFD0mRQMXBkS2zcNbF6QoCCrPfe5z/xpwzv+S8YGbbN247LLLTj3HAYYitvgaIjKO5WSOaLyBTIVnK4yPCPnEmCfiGJ5dwACJ4eL1dHPEv0iM4lEW3PDeA0mLtN70pjc1yyyzTIolfh3PhUgiYYu6k046KRkbKQ9IEKusskoKB84QiBAMShBeIFog55xzTnPuueemY4gOEBrwsIZEwtU4+ZShPsa3yCKLpO3BRAjBmASZB8F7FcZqMEfwLkZZ9BEZ0zC6YiCmbkCUiGU0pGy70pRHE5752te+1lxwwQVTj4MXZYABNBJalL8hBC48dxAXQlxs+8Z7YRTGYIuHIZUn7w2xQkZn1fU2D1zTUQ4iqJBfDNyULZjTdnfaaacGz3ASvJexFSlSa3yOHuooawidN954Y4oTEgX6RfjkbT49VPjHdnZvfOMb0x3yTpukHhIP78BWREhsW1HH0Y6pf7/5zW/Sc+g3ylD5oA1CjkJqyAnjhFt77bWnvKpQLw499NDmnnvuSXnBC8ob3vCG5LGKC/IAxfGrX/3q5PmQY96Pdg2RBJ0COUukvNhe47s7NFy2AABAAElEQVSdeeaZyVvNqO1ea3UU+ZLghQ5PMBLaAwZp2k+U2vof9clQnS+PleQDrCAIUm8RvJ6h+/BOiEDMYYu2Pm2jROCK+Ryi92K4tverKdv0Ug/+U78QCVzcqm1HQ0hFM0Hg4l1q+uhIFCcO9BlEddoF+lwEjLjNXCRwEYZ7/EFgpP8AU/oiJHrnqdUFQ7COBK7a9GJdKxFBS1so8q41Oit6EoScTX8Ljoi2+UOPo1MoDwj3cWs62itEe8axCOUGXhCtEcjqkKW7JLa/Ie22j54YQuAijzXjnNo2PLS82ghctWOQiHub3svLrWtMR9mjA5DYLyqOSAY85ZRTmiuuuCLdqqmDBNR4j2M+kjn88MMT0Zp8MIfThyDcH0XgimU4dM4W9UOcK5KuxQgYASNgBKYfARO4emJsAldPoPyYETACRsAIGAEjYASMgBEwAkbACBgBIzBrEYiegchkTpyKBsQSOWXJJZdMBgQRGIgDw5SMkRjDMOxjkEHwfoBhhG2+JDmJCHIE200NFRk5SmSOnKiGYZ28KZ+klXu/qDH6KM/5O+k6aUasOMfLVvSWw9aWGHr0HM+AG54MJFzDGwFeuiRbbLFF85KXvESn6f14z/iOxEMZ867IOPmUoT4SuIgzekbinLzGOsA1DKdsqUZ++kg09PJ8JL3VlG1XmpFYxXPUZ/IJ8U5lwnV50eG4y9jH/djOZGCjXPDeIfIgz4ET2xtJolc68oHxke3ZVNfbCFyEn3Q5sP3e5ptvrqylcuVEmIARedc5Bm/0xzjGZ8iVSyyxRGua1C3SK7X5qUDZAdtAyejOLeJAlG/OqduQLRHKB2N3bH+UBc+LPMFz+bZIsc4OISfUhiMP0bsG5+QTiTogb6/oZgzj8s7H8+iN+G7UM3Qyv0gs03Rhzj95o9N56bdGR8V48rZJGUGqKElN/R9H50POol0KN+oRxDKwj206euqJOA7ZQpH3rdF7fd4v5km49ilbPat+ISdw1bajSBoo5SOSPWaKwFXTR9N+IBYieRuMBCjqDR60IPOiPyE+58Iz0lfci57/9GyNLhiCdSRwkWZNerW6rkZnQQZDj2ssSp7zPotrOWk7lgG4owNp0zEe+h/GM9K3xNMmNe02tsk2PTGUwJXrUvI+apxT24aHlhfPQ5RGYj2rHYP00Xt5eXWN6WoJXLV1MNc35DWOFeOxxpc8o3eI4+Vx5myQ3yHBI4xDGQNbjIARMAJGYOYQMIGrJ9YmcPUEyo8ZASNgBIyAETACRsAIGAEjYASMgBEwArMWgWiIwAiAcSnKjjvu2LCFBxK96sRn2CJxm222mTJc6x5GA7yQsMUiWylKICxF4hHXMdbIIBa9RylMn1+RWtrIHPFd8vhKWzbiXef1r399ejR6t8jDls4jMQpiB8aySNQhDMZ9jHHXX3/9Q6JguzO8dEVShR6C4Ea4H/zgB7o09fvCF74wYS0SwdSNOQd4BoLsgHchyTj5JC6MyJGQoHi78gFZjXoBwaevQLjBg5AwBDtt50YcQ8t2VLqveMUrGoyF0UiuMJBb8D5w1lln6VIiYlFWbeTDEoGLwJQzRuyFFlpoKi4OMKYS/0033dTsv//+U23r4osvbvB4JE8U0TA3VwQPnky6HHbZZZcGz1i53H///Yn0RDmwrRfCO7CVHPpB7f/4449v2AoslzbjM/jTBqmnuVx22WXJm9tKK600iMBFPOgrjOK50DZoWxBBolC2eFDSdrHxHkZ9ygSDZpQ999yzwfCKDCFw1YZT2tHbia7xSz7ZhpIyoA5HQffyfs94xjPi5XR83333pS0s5VmMixiiaX8ipHANAoMMC5y3SVedLOmoPJ7YVxx99NFTXtny5zjvSqukh8bR+aRHPcXDSSRscV2CzsbrG30t0qdtlDxwKb6heq/P+41TtuSrjcDFvZp2FD3GsMWnvCYRHwLW2mq41I9DaoY4iJx44okNXi2jREJNqZ3GZ+PxkD564403bjbbbLMUHL1IW2GcEiXmQ2RykYfo72655ZY0jsj7JPpfPFmqTsU4h+qCIVhHYo3SHJreOLpuqM4ij9Q/+lt5I1W++QVjPGjlXrTwnLbvvvum7VHj8zpGZ4E/fWBfGdpu++iJtj60K09DxznEVdOGCTekvGKbzutZzRikj94jj1Eg1/OupTFdnDfhFZR+KEr0wJXrnJo6SNzPfOYz05xA8ySlB6EQb3KME5FI4NI75OPE2jkbY3B5AI6eApUX/xoBI2AEjMD0IqB5Fh9VQiZnTMiaD38c67xvLhZYa6215t7wvW/IWf6cCVyzvICcPSNgBIyAETACRsAIGAEjYASMgBEwAkZgxhDAoLH66qsnwxgGf0gaGD4kEB/48htjA+SInEAg8hUkA4gBeAKYDll44YUbCB/LLbdcSgPPFXgKyY2p46YdiVHaWodttCDDYYC588470/ZJOQ4xXRbh2J6OL94hSuC1DCMu3ju6BJITW6bxBzGItNjmJXo9U/hJ5FNx5b94LFpxxRUTiYV3xih99dVXF/ORh207p+xYpIQAqK2M9Oyky1Z1GsMZmLKlEnUbwl0kwSn92l/eh7ZDWVMPIUr+9Kc/nYqONUjqKxhiLMwJDFMPthxMuhwWX3zxRFRhy0SMq+QJ47cE0hLeiChv6t4kBCP6aqutlrYKYltAiEjjtlkWvqmfYIuuIq8//vGPO7PLu1MHITrxfrz/HXfc0cvzSmfEE75JnYFIR92F2MFWSWzjFMuplCTGZcKhr/EAxLvxnm0CiQUPQZAYYp1te17Xh+gohan9nXT9H5UP6hV6Fb1NO8DIApb0e5TDpGXSek/5qy1bhe/6nV/aUdc7cK+2jx4Vr+5HAhfjIogtkMHp26lLfepUrS5QHob+znR6Q3UW70P9o31C5KJ93jVnWzr63S5Bl6oPQqdKP0Kgq5HpardD81I7zqltwzXllb8TaaNjZ3IMkudh3POaOkjfQr1lzAIRlHGQPIUOzY/KnTZQM2cbmp6fNwJGwAgYgfERMIGrJ4YMRi1GwAgYASNgBIyAETACRsAIGAEjYASMgBEwAuMhwBZp8qaEh5IjjjhivAhnQegSMWoWZOshWZhf8vmQjPuCETACRsAIGIGHMQI5geth/Kp+NSNgBIyAETACRsAIGIEOBEzg6gAn3jKBK6LhYyNgBIyAETACRsAIGAEjYASMgBEwAkbACPRHAE8SeK/AkwHkLW1NxjYweDaY32V+IUbNL/mc3+uD828EjIARMAJGYAgCJnANQcvPGgEjYASMgBEwAkbg4YuACVw9y9YErp5A+TEjYASMgBEwAkbACBgBI2AEjIARMAJGwAhkCIg4xJaJELkQtsX72Mc+lj05f57q/ci9tlCcjW8yv+RzNmLnPBkBI2AEjIARmC4ETOCaLmQdrxEwAkbACBgBI2AE5i8ETODqWV4mcPUEyo8ZASNgBIyAETACRsAIGAEjYASMgBEwAkYgQyASh7j1+9//vvnoRz/aPPDAA9mT8+dpfD8TuObPMnSujYARMAJGwAjMKwRM4JpXyDtdI2AEjIARMAJGwAjMLgRM4OpZHiZw9QTKjxkBI2AEjIARMAJGwAgYASNgBIyAETACRiBD4PGPf3yz4YYbNgsuuGBz9913NzfeeGPz5z//OXtq/j19zGMe06y++urpBW699dbml7/85ax8mfkln7MSPGfKCBgBI2AEjMA0IbD88ss3iy66aPO73/2u+d73vjdNqThaI2AEjIARMAJGwAgYgdmOgAlcPUvIBK6eQPkxI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBHojYAJXT6hM4OoJlB8zAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGoDcCJnD1hMoErp5A+TEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEeiNgAldPqEzg6gmUHzMCRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkagNwImcPWEygSunkD5MSNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYAR6I2ACV0+oTODqCZQfMwJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRqA3AiZw9YTKBK6eQPkxI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBHojYAJXT6hM4OoJlB8zAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGoDcCJnD1hMoErp5A+TEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEeiNgAldPqEzg6gmUHzMCRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkagNwImcPWEygSunkD5MSNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYAR6I2ACV0+oTODqCZQfMwJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRqA3AiZw9YTKBK6eQPkxI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBHojYAJXT6hM4OoJlB8zAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGoDcCJnD1hsoPGgEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEpgeB//7v/24WXHDBZoEFFmge9ahHpT+Odd431QXWWmutv/R92M8ZASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMQNOYwOVaYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARmEcImMA1j4B3skbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYARO4XAeMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMALzCAETuOYR8E7WCBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgApfrgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGYB4hYALXPALeyRoBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBEzgch0wAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAjMIwRM4JpHwDtZI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyACVyuA0bACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYgXmEgAlc8wh4J2sEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETCBy3XACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASMwjxAwgWseAe9kjYARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACJnC5DhgBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBOYRAiZwzSPgnawRMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAELtcBI2AEjIARMAJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIzAPELABK55BLyTNQJGwAgYASNgBIyAETACRsAIGAEjYASMgBEwAkbACBgBI2AEjIARMAJGwAgYASNgBIyAETACRsAImMDlOmAEjIARMAJGYD5H4OUvf3mz+OKLNz/72c+as846a9a+zQYbbNA89alPbR544IHm/PPPn7X5dMb+isCaa67ZLLPMMunkq1/9avP73//e0DzMEfjHf/zH5nnPe17zl7/8pTnmmGNm5ds+5jGPabbaaquUtyuuuKL58Y9/3GyxxRbNQgst1Nx2223Ntddem+6tsMIKzWqrrZaOTzvttOZ//ud/mgUWWKDZeuutm7/5m79pbr311ua6666btndceOGFm0033TSlefnllzf33ntvSmudddZpllpqqeZXv/pVc+65505b+ptttlnzd3/3d82dd97ZfOc732le+MIXNs94xjN6p/v3f//3zYYbbpjyd+mllzb33XfftOXVEY+HQG3fSl38h3/4h+b+++9v0PHjCu2S9hnb4bhxOvz4CNTWj/FT7hcDuvtVr3pV86hHPSrpROnKfqH9lBEwAkLgmc98ZrPGGmuk06uuuqq56667dKvq1+OAKthSoJLeXXnllZu11lor3T/66KN7R942nuwdwYQenB/mCLwqOK+yyirprb/85S83/+///b9qBGZy3lCdyRkMWKrXM5j8jCZVmq+1tcVHEi4zWgiPkMRG6SzWeldcccXmSU96UnPZZZc15513XvP85z+/WW655Zrf/OY3zTnnnDNfIsW6xMYbb9wsscQSab3kfe9733z5Hs60ETACRsAIGIHpRMAErulE13EbASNgBIyAEZgBBD796U83j3vc49IE/q1vfeu0p4iBdumll07p3H333c0f//jHXmkqn7/73e+affbZp1cYPzTvEPjwhz/cYLxBDjjggEQQVG5YMFpwwQWT4f8///M/ddm/8wkCbeX3T//0T81KK62U3mK33Xabkbdpy0tb4ksuuWTz/ve/P91mwfLUU09tPve5zyXj/49+9KPm4IMPTvfe9KY3pcVNTvbbb7+GSc/f/u3fNkcccUS6//3vf7/52Mc+lo75x+LhE57whPTcJMhKLKySB+RLX/rSFFnr3/7t35onPvGJzR/+8Idmr732Sven499nP/vZRFS75557mgMPPLA55JBDmsUWW6z505/+NJWvrnRf8pKXNDvttFN65KSTTmouvPDCrsfnm3tPfvKTm0UXXTQZ9G6//fb5Jt9dGa3tWz/4wQ8mUvWk+mQZpCFH0n9YZgcCtfVjpnIPifAd73hHSu7EE09sLrroopFJ145DR0bsB4zAfIzA61//+gbSAcIHPWecccbU29SMcR6u44ApUKbxoKR33/jGNzZ8HIMMGWO3jSenK/tt4/L5YY4AJjGfe+65Z/PnP/85QVUz/uuaN0wX/rM53lK9ns35HSdvpflaW1t8JOEyDqYOW0agTWfx9Hvf+970AZZC8vHZoYce2nz0ox9tFllkkd7zeoWfLb+Q1li3hiQrGdIvKox/RyNQ0/eNjtVPGAEjYASMwEwhYALXTCHtdIyAETACRsAITBMCWjTiC6yZIHBtueWWyeMNr/OZz3ym+e53v9vrzZTPSRmLeyXqh6oRaCNwQeqScf7qq6+eIsRUJ+SAM4pAV/nFBcSZWETryksbKHiV+sQnPpFuQ96CxCWyUiRlDSVwQezCUAMhERzGlbZF/pJBYNy0SuHRzZAcIClB1NECcF/9+3A13P7rv/5rs+yyyyYvc294wxtK0M1312r7VhO45ruirspwbf2oSqwiUA2Bq3YcWpE9BzEC8w0CXQSumjHOw3UcMBMFWtK78wOBq2tcPj/MESjbmM9I4KoZ/5nANXdrKdXruZ94+JyV5mttc7tHEi4PnxKePW/SprPwGM4ahwQD7iWXXNLgWXx+J3DtvffeU57S8ZL+29/+tnnb296mV/XvBBGo6fsmmLyjMgJGwAgYgTERMIFrTAAd3AgYASNgBIzAvEZAi0bzgsB12GGHTW1ZNgqHV7ziFWmrx1/84hfNmWeeOepx35/HCPQhcLE1W1xYmsdZdvI9EIjGmbz84gLiTBO48ry0vQpfah511FHp9uc///nmW9/6VoMeeuxjH5t0EcdIicBF2F133TV5j7v55psbtmCU1Bg3Fbb027bIXzIIlMKPe+2Tn/xk8ih20003Nf/+7//evPOd72zYVvLXv/51rwXSpz3taQ1bNiBs1cBWlQ8HeTguYtb2rSZwPRxq9Oh3qK0fo2OezBPjEriGjEMnk2PHYgRmJwKTJnA9XMcBM1F6Jb07vxG48nH5/DBHoGzZ6lFbqLMdvLZQrBn/dc0bZqIezbY0SvV6tuVxUvkpzdfa5naPJFwmha/j+T8E2nTWc5/73ORFnCevueaa5vDDD58KtOmmmzZLLbVU88tf/nIub5tTD8zyAxHQ/vKXv6Q1G3lKnOXZni+zV9P3zZcv6kwbASNgBB6mCJjA9TAtWL+WETACRsAIPHIQmF8IXI+cEnl4vKkJXA+PcszfYn4ncPE+ELgwqsgDoMhKl112WXPsscemVy4RuHIs4vnDjcDF9pC4zJeXvH333bdZddVVGwi0kLkeqeJFzP8reRO4/g8LH807BEzgmnfYO+WHFwKTJnA9vNCZ929jAtewMuiarwyL6a9Pe/xXg9ojN8wQAtcjFyW/+XQi8IIXvKDZY489UhJf/epXm6985SvTmdyMxq32BXELT4mW6UPAfd/0YeuYjYARMAIzgYAJXDOBstMwAkbACBgBIxAQWGWVVZr11lsvXbnooouaW265Jdxt0kQdl9lMaI888si57i222GLNtttum659/etfb+68884mErj++Z//uVlnnXUavhB8+tOfntxR8wzbjN17771zxaWTpZdeutlmm22Sd6yFFlqo+dOf/tTgzevKK69szj///KkvV9m67DWveU2z5JJLNuQDIc6f//zn6cuv++67T1EWf/k68SlPeUrzk5/8pPnmN7851zMQDXbZZZcU9xOe8ITm97//ffMf//EfKQ+QMiwzj0CJwPXa1762YUH92c9+dsoQA8lbb701fRV41VVX9crkwgsv3Gy33XbNcsst11Cn5Db9Bz/4QfLMRpxRXvrSlzZLLLFE8gB08cUXN2ussUaz9tprp23Q2A6O+v3FL34x1dlJhItxvPCFL0xfctOWIAzdddddactQvkrni8GSDH0/Fud4J9rcCSec0HC++eabN8Tzuc99rsGDEjKJNjKq/PKv6yH8gAGem/iC/Z577mloj23bpg5591F5KWGra+DyqEc9Km0fQL3RV5x4ijrllFPSYyUCF2HQMwhfsuKFa+ONN26e9axnpa/1KWPe8/rrr2/QZ2eccUZ6Vv+os+uvv36zzDLLNI9//OPTM9/73vcSJuisKG1faWvB8g9/+EOz1157pSDEh7cr6tRJJ52UtnJk66LVV1895Qe80Zl4z6JvYPuwlVZaKelTvrwFg9NPP31KVxPpIYcckvQ0Wy0cf/zxze67796stdZaSf++5z3viVktHvNV74YbbpjusWjM1pJRyN+LX/ziVC/B9YEHHkj1g7aYt+EYLh4TjnJC8KSGLtliiy2aFVdcsXnqU5+a4rzjjjuKekHx8HUy+aRfetzjHpf6PMqO7SWi17CVV145lR240c8h1157bepnjz766OZ///d/FWXn75prrtlstNFGCXu21qHcKYNvfOMbDXWhj0x6DNDVt4IxZcX702+TV96b/p16AM5t22oOqe+8NzgilCP9B97uaCe0KQiSJZ05CosUYfiHIYM2QP7RjWyDST4Zh3zoQx8KTzZJd/FV+xD9Td149atfneJE51IvaHcXXHBBw3itJH3rYB6WcIzXaGukQb/y7W9/O53jPeS//uu/kvc7wo3TVkr1g2uM/UYJ+TrxxBPneoy8bLbZZg3EK+oUY1V0EGVC2bfJM57xjGbddddN+hZde9ttt6X3ffSjH9284x3vSMFIqw1nHhgyDh2az1F9MXpt0nq6Dav8+iabbJL6g8UXXzzVf8qFsQ945X0P46VXvvKVKYovf/nL6XnaImVF/821PtJ3bpDHRTlvtdVWqd3R/v/4xz8m4jDpxvoxJJ/kfeutt05xPulJT0px0uYvvPDCpjT23H777ZtFF100bSF87rnnJh1Im6KfoO8FO8YLd999d8r+E5/4xGbnnXdOx8w96D9KQjksv/zySZfhAZTx8LyqE6X8lQhcQ8Y4eZwzMQ4QhqTNnJM56AYbbNA873nPS15Ef/rTnyY9wVimTWp0cM2YmrEJ9ZC6xRbVjNsZbzAfZ6wSpaR3cwIXfTNjCtoC701cjCUYm0ZpG0/GZ2rmKzE8x6PG5fPLHAFdzlgPTE8++eQ0/ll/zti9ZvxHP5LPGyJuQ+pEDNd2XKt3S/FFPYi+Y65JH0y7Rvcx7r7uuutSULABo2c+85lJv3GfeccPf/jDuaIu1Ws9UNOm+s4lJqEnho4JSvO1trZYwoX0JjHH6TtOVDmM+h2qL4eMu9vSHrI+0BZHfn0m6w51+2Uve1lqH4ssskjzq1/9KunpSy+9NM0/8rzpHF1Eu2LswZyR+RZzU9Z74rw611k8u+OOO6ZxGzoGoS9k3MxYm3bJXJR46Yfy9QqeZ62AdRz6K+oiY9gbb7wxrZmx9tZHRo2LtUZFXH3njPR5zB/os3hP5obgCDb5+LRvnKQ/ZEw5VBdEXdpnTEl+ovQdF8cwQ949hovHQ9Y+huqFmI6PjYARMAJGYHoRoA9fcMEFk02KPow/1tP447ivLDDHIFC2YvWNwc8ZASNgBIyAEXiEIMCEFaMjIvKBXp2Fs3/5l3/RafOBD3wgkUZ0YaeddkqGCM7xOnPDDTdMEbiY+GJIwxCRC0ZIDPoQUKIwIcUg0SYYOvbff/+0OMCC3/ve977ioxg7WMDuEhHNcmMxZDYWRxl8lASCG1uAaQuE0jO+NnkESgQutkvEeJ7L7bff3uDNZZSwQPz2t7+9taxZxPnEJz4xRVoiPpFzqDcYtDHk5cJCFHmDmCCpDUd4DMoYKlg8KQmGw/e///0PMZzWvJ+IRrz7WWedlchbShNvUhhcJ9VGRpVfNM5ARmHhrySQdDC0RRn67qPyEuPOjxUWAgqE0IMOOqhhqx8WMMEQEa4c77fffmnhkoVCiCTI97///QYvVeg0dFsuGJ3f/OY3T12G1MFiaUkwElH/I2GobZG/ZBBgERrdjrAwy+Jmrg+pHx/5yEfSV6olHQ+5jnfhOeS9731vw4IhxkAWRNV3/OhHP2oOPvjg9EzXv5gnSGUYyhEmjvQlGFNKQltE18dF3dJzXIvlgacwFhohYeXChJV+CB0QhX6U/rQk4IDBFyM70tXXvfWtb019XCmeeO3AAw9MBIJ4LR5T/m19ZHxuOscA++yzz1RSkJHID8TpXKjf/GFUyftknh1a3wmTE7hEIuQeXvPilqVcQ2L7k0e9v94p/1cakBqpL2ydilBHaOdIrf6GqEpdIHxJfvaznzUHHHDAXLeG1MEYkK/NMabmQr2FCCCSiMaK47SV0tgLXVKqF3l+0G0yQHKP/vdd73pXqjf5s5znW27pGfQJxqhcr3EfUg/YI6MIXH3HoTX5VJ9BGZT6YurFpPV0eumOf+hD8MZIVxL0LTru8ssvn7od+x7ItxDChTtzBuYOo6RLXxI2zg1iXMwnIOgrvXiPY4x0xx13XLrcN5+jxj/0afSrkcimOg8Zi+vLLrtsSjP+Y05BO8AgysIn4wp+KX/0aIxP4TT24BnGBy960YtmvE4oL6XfEoEr6tgYJh/jxHs6nolxAMS6vffeOyUJmQTiVqn+UE6M2XLDc40OHlWnSvPONp1NxqkPtDX+JKqDsX+NBC7m/xixS8K4X95kuR/bype+9KUGA7Kktr9T+Pir+h2vcaw53vwyR4j5pNwgtLatdYwa/8W+V/MG4TO0Tihc22+t3m2LT3UQggc6G8J+LtRD+gXSzoV6nc/JFWes14Qb2qaGziXG1RM1Y4LSfK2tLZZwiXWnZo7TVr8ol9I4MS+/0nmNvuwz7i6lpWtD1wcUru13pusOZY7uJt1cKAvGrRfP+cAwCn0YH2lRb0tCOLZD1Adxuc5ibkPbKwn95KGHHjq1PpaP0/nQgbVkiFslYa2YOXUkkJWe49qocTF9FTJkzkjazHFKsttuu01dHhIngWLb7Br71ugCte++Y8qpl5hzMGRcrHBD313h8t+uPiX2fTV6IU/L50bACBgBIzB9CJjANX3YOmYjYASMgBEwAkUEWABgkZTJfb4AhoFLXk8IzNePfD0qkUEUwwMeVRBNKvUMiwIYGjE+YOxiAQnhXIvknON1hq20yAdhMILgnQNvFSzyaaGCrzAhq/DF2dve9rbkAQEvWQhfoBEvBC59xZluFP4pn/GdMR6zQKY83H///YlkxnUMemKTszDC12qWmUOgROCivuD9QIvAGIDwvEPZt3ksUI75Wp3FKBnHWQSB8AARBOMk9xEWlli0lIiIpXN+SZNBLF/biVBGHWYxApIXUhuOsNG4DfkRT3O0B3m/4BnqPe2BhTOk9v20OEYcvANtQcd4msJgMKk2Mqr84gJiysScfyzwgTcLXmr33ONZLf7VvPuovCj90u9hhx2WiBtafILIxVfrkWgUce0icGGQZ3GZsgV7dCv6k3cGdyQubFFGWsAjjAhHXKfNYGRE4kJiNLiVDALRSJoCz/n329/+NuEb65zukRa6mraErpTEdPDGyFe7ItfiEQVDPobJj3/84wrS+hvzFHHFsI2BFaH948mEvC6zzDJThBDaDH2N2kZbItG4oWd4N0h53EPPqD3IgKjn5FGMc8JAYGN7SEhrEFMU7tRTT00eKCFN4SmFBW3pDL5mRiC00Z67BC+VeB1CMCBDOuH9SY++UelB/Ch9CR3jnq4xQOxbSQ/is+qH6gz1GwyUX57Lw9XUd+KRkQdsaAt4T3zVq17FrTS+yImDlAOESvICptomJAVo+ac08ttxa9Aa/Y3BA7yEC2V71113pXoI+UPXo2F9aB1UnqNu4hr405/RF2rMw3XKjLJAxmkrpbEX+rfNA5f6YtKNdQP9D0bKI22Gtkodi3Uqb6t46sOzi4Q40a+0U3nD071RBK4+49DafMZyAXuVOcf0xXiTEoFL+R1XTyuett84jmFshL5Fr6J3YtuGsEtZILHvifHyHnieFIk53ovHQ+cGCounhtiGIQ2glykzCNaST33qU8lbYZ984oVSnioJD3me8Rh9A+M/1cW8zqnOK02FBbvYp9LORdTGC5wINZEMrjgigUBk3dhP6rnprhNKp/RbInD1GeOU4uJafL/pGgdEXJUP+gPVnUhYp/5i9JbU6GDazdAxNV5/ZVhmfENbYxzIuDjqUdorhBhEdTDq0Ejg0jswj6ZeE1d8V+a8IgXEthLHecRR098p7fx31Lh8fpkjxHwynwS/2vFf7HsjgaumTuR4x/NavRvjyI9VB3WdPoB6yzuVPsJAH1Ifuac1IIhfUQcrzliva9rU0LnEOHqidkxQmq+1tcUSLrHuxDLoM8eJ4xHCgveocaLSaPut0ZfE1Wfc3ZZmzfpAW1y6PpN1By++GiOQPmNX5hu5vmZrw0jgjeVHu2POTtuLYzf6EtaRKNdcZzE/goTFGoP6BcZUjC8g7LOzgsaHjGtIT6J6y7n6K+oPHyVSJ5E4Z0oXWv7l75GPi/GAOnTOyHya9gyGjOGkl5ifykP40DjJfmyb8XXi2LdWF6h9x3jpt7vGlDw7dFxMmJp3J1xJ+qx91OqFUnq+ZgSMgBEwAtODAGMFxub0w/Sd/HGs876p2gNXX6T8nBEwAkbACBiBOQiIiAUYLEQwsUbkSSadzPmHgZl96yV4saCTlvGA63FSyeIbcbANA4JxjC+W5aUC7xGQExCO5WEIg37cypFwLABAEMgXBnDJzTZXCESK6PUoXWz5p3zGRb8YF6785S2FKKJBNYZpid6XJ4xAicBFEizscg9p87aRbmb/WDxn6xEkemHQY9FLAEYEFpcQLVBxzCIMZELqvyS2pbi4XhsuGqxYcCM9BswIC1+0Ly18xwW72veLi2OkgfEHA6uIaJNuI13lFxcQwZp8YDCToIvkySISVWrfvSsvSrP0i8EMHcUWAuQTYy5GYsimIuJEXLsIXIqfd6V8IaWBg4Q8sjUbehejIp62IHZIMFizQIdgXKd+IHEhMRrctLAajSKxzhE2bgWJHiSMDNXkASKM2kA0auA1DeM4Qr75w9hN/WXREkIv7UptKz3Y8i/mKRpuhROLwhh1IBVIIuYxjO7nv7lxA+zRLRB5EbbXg6SHRIIPW1nQvikT8kGZgL0kerhiQZg41IZVh6k3IscoXNcv3qEwRJAPvAgqPsLQJtRPxzrQFV/UW5MaA8R+cv05W3Zo2x+uo8eEK8QH8iuiTgxXW995Vxl5ROCKBC3KAeM1uEvwaMdXxgjbljJGGCVKg+coi2OOOSb1Q4o31tsh+hsjCh69kDi+4hx9I6OC2m1tHYz4kmfqAeR5hDb67ne/e6p/iXW0tq0Qb2nsxfWSxDIhfXSPxobUe22fjNcA2oSEOgWJSORI6pcIkugkyE9I1G2csy03njskowhcei72i/k4tDafUX+RTt4Xx7rF/fgutXqaeNoE706ve93r0m0Md4zZ1b9xMRpdqOuQdpHY93DOdouQ59VncK1LaucGsZzxyBtJ/ZEAS78N6aRPPvEWRrtAYj/KOf0+XopV5/BIoQ9JVOd5Dj1BXdUWt5QV/QdtCqHNMWeK/U3J214cH8mj4EzXiZThjn8lApceV9+dj3F0v/Qb3y/26YprEuOAOIYhDzkZj2195F0RnYT3DvqyWh0cdUffeafGDeSPuiOSFuexHkeCvOpg7F9zAhdzXvIgid4ymM8zxuKdYxqxHcTyGdLfKb3SL31U2xwvtgHyRT2YjXOEmE8IXBqnqhzJe9/xX+x74xxTcYFh3zpRwlvXavWuwpd+VQe5x9iFPlrj/7wuyquP4uGdmFchzHPQo4jijPW6pk0N1SG1eoI8144JSvO1trZYwiXWHfLRd45TO04kjTap1ZfEN2rc3ZYm12vXB7rinMm6o3IlP5FUyzljtF133TXNReNaKe2GMQ5zVHQNbSluRSoP2cQhfd6msyIBKB8fq43GtOPcjz6Euq9xI2u6jKn0IaU+giMfbTJqXBzrKjqi7xoJ6al9RV3C9do4Y9skntLYt1YXxHrQd0xJHoaOi2vfnbS6RP1V3veNoxe60vM9I2AEjIARmCwCrD2bwDVZTB2bETACRsAIGIGRCMRF2rggcOSRR6aOmQmWjNMssiEYzph4Iqeffnpz9tlnp+M4qeTL5LiYygNxshoNXUzm8LbF15YQZnLB4ItnpHyyFxfqYnx5+Pxc+YwT9Uh+4Ms1CDFRMO7yRTxfnLEYYpk5BCZN4Nphhx3S1/zUJzwdiKCkN+L+xhtvnE4pay12aYGKG7lHOq5FwynGJLWX2nAYASEHKZ8suEbha0jqMsLiEIZ/pPb94uIYX+WKKJAinfNv0m2ExaE+xpnc0wH52WCDDZqdd945ZS2S92rfvSsvKZEx/kVcxyFwRa8cMtrm2dICHXUGgxuLpnEhUQu0hNOCpYggXItGuJJxNdblSBokLIJ3GAheGGZKuvyvTw37H/MUDbciEfOO4CrDGLGzEPiWt7wlJcTWjaUt82IuonED7GhLEBWiRAO+PGDg3QtjDhKJhDEc+XjOc56TLrGFJPlB2hYx082Of/R1EAUgGkUPIAqivpsvcjEwj5LpGAPEvjV634LwJ5KQ8hW/Ko/haus78crIIwIX1/iCnK2hETDia21JJO3m20XrmfxXaXAdYqU83um5Gv2NMUNlSh8CoY76HYX8QVJCqFt4lKqpg7HultpyXEynTcjIXNtWyG9p7MX1XPDYR/kz9kSi3iJffAyA0EZFrEwXHvwXDUfqP9gOVn1yqX8jKPESP5IbqNLFwr+2cWhtPkki9hmlvEadOBN6OrZhkYxyKKJ+VD8X+x4MTdS5qKfzOPLzmrlBJM2WyE/0T7QxdKj6vlH5jHVHnoDzvK677rrJgMp1dAE6AVGd57jUb1N/IWwhcetWEXVjX54emvNP28uhIxiX8cxM1wnlpe13pghckxwHRGIG9XXfffd9iP6NZBPNA6IuHTIOqBlTRx1VaosY35mL0PcxXkJUB2P/Gt8jEu5jefJBFZ6OEQi+bJUW20rUyzX9XUyrdNw1Lo8kA+n4GMdsmSPEfE4XgaumTkSs8uMavZvHkZ+rDnI9X6+J7Q4CCF6G0GkSPgCgT0eYt9x8883pWHHGel3TpobqkJjfIXpinDFBab7W1hZLuNSO26JuGzJOTAXU8i/GOURfEt2ocXdLkuly7fpAV5wzVXfiBw05wVH5ix9/aA02egjDi2K+gwBzIuZGiAjtbTprKIErjhtLc6S4NV/sS/Q++e+ocfE4c0a1r6hLSL82ztg2S2PfcXSB2jf56zumrBkX1747+eqStrWPcfRCV3q+ZwSMgBEwApNFwASuyeLp2IyAETACRsAI9EKA7WZkaLjpppvStjTLzNmCioVhBA9FGCYQGTYxXuOyH5GhhmNNKqOxj+uS+OV7JIvpfvxlARpvE2uttVb609djMiLybJvhLMZTOlY+40Q931oHI83555+fSGhDDE6l9HxtPAQmTeAq5QYPMLiTZws7vtKUJ4U2AhcLG7ivz4W2RJtCWGxiMTqSXvqGY8EHMgyCq3stsKUL4Z8WyNranB7t835xcSwaEhXHpNtIX+MM74huigKBAX2EsJjP4lub9Hn3rry0xdv3esRV+jIupscv6YlTX/TmhnmVNR6E8HhSEgyOq666arolLyBxITEukmrBUkZsAkUjcNyiTWlBCFp++eXTKV/u4lUrigzLTOx410lIzFMkcEXCACQXSFoXXHDBlNfHIWnH8sADivq/GIcWHbkG/pSDvFfR/ijnUl8BGRTDARLJhopvVNtNAUf8Q+fQX26yySZTJKW+BK7pGAPEvlUGjrw+x1cSWSGGq63vxCsjTyRwRaJYJFig62lzjDFi+jF/pWOlAfFchHY9B0mkRn+zXbOIjyXyCfHj7QcyOfWP7UsgxbH9x9A6qLpLnOqrOI4iYluso7VthXhLY6+YHscYNuhH1QdfffXVqXz0XDREXHTRRYlopXv6RedTpxD6afrdaNiVbtTz+o0eGsYlcNXmk7zEPqPUF0edOBN6uk8b1la55F/4xr4nL0eeq5FRc4PoSa1ksCRN2hnbF9J2b7jhhrlIKaV8xjhz8oHeAe9c9ElI1HWq85HUrzD84kWYuQwS50WQPdZYY410PRJ/6d/p55HoZWmm60TKQMe/mSJwTXIcEIkZEI3zrXZ5XeYIGLgR4S9dOlQH14ypI+GPPuDGG29MH5TIO2HKWPZPdTD2b5HAxccaEEVz0XbXXNfYK7ZpjSdr+7s8vfy8a1weSQazeY4Q8zldBK6aOpFjPep8lN4dFV51MI4lFIZtDylDhDkFc4sor3zlK5tXvOIV6dLxxx/fXHLJJelYccZ6XdOmhuqQWj0xzpigNF8rtUWAKeFSO26TbiPeIeNEnm8TxTlUXxJf17i7Lb2u633WB7rCz1TdieQW8INMm0vcSpUPVPhQRVjzbIlExXWIWeyOwLonf206ayiBS+PGtjULPlhhjReBRMyHiF0yalw8zpxR7SvqEvJSG2dsm6Ux5Ti6QO17yJgyjmH7jotr372rDLnXtvahulqjF0al6ftGwAgYASMwOQRM4Joclo7JCBgBI2AEjMAgBDBIMHmX0V1fRTE5xMMDk0VEXxuLkJJ7P9CkEmM6X1DmstlmmzWQuJDcOIbRji1a+BqMvMjzQowjX/ibJIELN6BMHlkwzgVjDIsLeE7R1k/5Mz6fPgSmg8CFcZh6vvrqq6dtcTBAlKRE4MrrYQwHaYVtVhCFVXsZEg7iF4scEsKWJLYT2py8tdS8X1wc40vOfDFt0m2kr3FGpKf4/nHBX0Y03a959668KN7a34ir3iUupvclcMmzEvnoUx9OPvnkpLPjQqIMbsShBcs2Ald8lucRyCUYvxHItHk+ZpLAxQIoZOLYBsgX5Ee2D4R8DMGlj8Ty0FfIebj4NSrGT/pH9Z1sC8ECe0kg19C3IHFLvLZFzFIc+bVnPetZzU477ZSIO9qCIn+mL4GLcHqPSY0BtAgeyWGRNJXnFV35lKc8ZS4CVW19J24ZeSKBi+uqn3HhOxIfMAxiIOwjSgNDispX4eJWllzL24mei3UX/c0WfiL75VsDKkz+q7IbWgflMYX2ArmpJCKEkn+R52vbCvFrjKj6kadJH4M3EfQ7km/dzbUdd9yx2XDDDTlMMgpb4QJJAZIjIj2cTsK/uEVaPkYNj8112DYOrc0nkcc+o9QXxzo73Xo6EpPYro3xTEmikV3bFsa+J3rrLYVvuzZ0bhCJZLR3SLOjZFQ+Y5zoeW0DlMcr/RLblOp87GdjuJe+9KXNq171qnQp1jm8DuN9GIntIHoypiy0hd5M1omUqRH/ZorANclxQCRmfPvb357qR+KrMr6knBH1sbU6uGZMzXgCYi16OAp6kHqC4R4ie6yjqoNR70YCV2ksR9wRD23xGNuKdE9tf6f5SnyPeNw1Lo8kg5I+ny1zhJjP6SJw1dSJiHPpeKjeLcURr6kOlvRg7GP0IWEMG/uWUQSumjY1VIfEdjFET4wzJijN10ptEdyEdWzvteO22nFiLL/8uFZfEk/XuDtPp3Resz5QikfXZqruHHTQQc3Tnva0lGzbmJebmlNorCasuacPjzjukjadNYTA1eYhvivdUfdGjYvHmTOqfcU2Q35q44xtszT2HUcXqH2XdCl5Lo0p4xi277i49t3JQ5e0rX2ormrOVoqjbT2l9KyvGQEjYASMwPQgYALX9ODqWI2AETACRsAIjEQgbvHEBJnJFcYDGQ30hRnnfBnJ5I9FgiuvvHLKywSJaFKZE7uUgTYCF15Dtttuu6mFBz3PIgVbFrKNHItynMuIyDNthjOFb/tVPvOJuhZ2WJDhq7ySnHrqqc0555xTuuVr04TApAlceJjDkxDlnQsGBeoZJEJEJCyORcRqIyjyDCTEF73oRRwmbwx4d6gJx9fOGMCGiIwYte8XF8cUV57+JNtIX+OMtquLeWkzztS+e1deYro1xyVc42J6HwIX+pavWYeIjGtxIVHXiEcLlnERMBqBWSjHOBElErhK5SIDtohAMWztccyTvEAoLjwRsRCKRzYtXOsev/RF73rXu6aIjfFePI7l0badUInApS+Mo8eVGC/HMe577rmnOfDAA9MjbYuY6WbHP5Fq8kfQW7/85S+nSMgyLufPlc4nPQZQ3xq9xaALGUuURMYJhRunvhO/jDw5gStipy1GGNPgfRFp85CYbmb/2tLgsXXWWadKf2+99daJxEUcp512WgMRZpTU1kEtzreN10hX9SKOvWJ9HtJWiK9t7MU9JG5zSV3AkJQb+WMZ/jVU93/pN/XDPF3SXVyn/5D3vUim4V6btI1Da/NJOqU+I6YfdeJ062m2s8T4g0QPgjE/HMc8MUZmrNzW9+Rh285r5gb6ip84qb/5FtmltEbl84Mf/GDyfBfbQSke1e8SgautnZWMbYpbHhBIlzqBh0f1sfk4NOI/3XVC+ev6nSkCF3mY1DggEjOi17P8PaVz6W9pGzqvGQfUjKkXWWSRtF0n783cOBcIytRZiOyI6qX6V671IXA9+9nPnvIu2UXgqu3vGCd2Sde4PJIMSvp8tswRYj6ni8AFhkPrRBfuNXq3Kz7uqQ6W9GAkcOFR7hOf+MRc0Q0hcBGwpk0N0SG1emKcMUFpvtbWbwnr2N5rx22148S5CjA7GUdfdo27s2Qeclq7PvCQiLILM1F3tA6bJd16qg+RhHU+XmgNOOdGm84aQuCK3oRvv/321B91pdnnXte4eNw5o9pXbDPjxNnWNvWe4+gCte+SLiX+0phy6Lh4nHfXO7b9tq19qK7WjKPa0vJ1I2AEjIARmDwCJnBNHlPHaASMgBEwAkagFwJxosniyK677poWhc8777zmlFNOSdt1YIhlUZjFHCbRSO6Oe9SkskTggqiCkYLJIoKHEohh11577ZS3q9I2PjzbZjjjXpcon3Ginj/Pl264I8cjA2Q25Y/n2K6AibNlZhCYNIFLRjFyj5cp6htfrUOuwNU7ZMJNN900vVyJwNXmDp4AceGLekt9luF4SDgWoLVtIvk65phjUn7a/mEwZOs3pPb9uhbHSumO20amwzhT++5deSm9+5BrJVzjYnofAhfpsSUbnuLwzoZxbpSwFQl1Lur3hxuBSxjQXtgijy2n8E6FQUiSe2jT9fgby2MIKUVb/3V9MYrXH7z/IPRrfGWKtC1ippst/6KhFKM+24eiv/BaIKKCDPxDCFyxjkxiDKC+le3+VFfbtqTiVdVuFY5rtfWdsG1GHjx8Sq9i4Pj4xz8+RUjXVnuE7yNtaRA2pjNEf7NNEMZKRF6M0knHv9o6KM8KIjiVkigRV2rbCvF3jb122WWXZv3110/ZQG/RPthGMhc8z0FWQfAKO8rLHu0Cg0D0wEU/zbVcIO/rI4FxCVy1+SRPpT4j5nUmyTqRCCGvDjEvOpbnXs6ZJzCminol9j0K0/VbOzeIJEDIeBoXdaU1Kp9x6+AuD1wydjOuxHMaojo/xNimvEYvDczF2E4bcj2Sf0Azk3VC+ev6nUkCl/Ix7jggEjPatmfF4yXkX0TbvdXqYOVbvzVjasgDa665ZrPyyisnL5aKK/alqoPxWiRwaa6isPqNdarLo2ttf6d02n67xuVxrjWEwKWxBmkOmf915aUt/1yP+ZxOAlfMQ586EZ+Px7V6N8ZROlYdLOnBSRO4YvpD21QfHVKrJ8YZE4hgEsdrbf2WsI7tvXbcVjtOjGWQH4+jL7vG3Xk6+Xlt28/jaTufzrqjj1xIGzIOZO4u+cUvfpE8GgvrSCrvCse9Np01hMCFBz+86SLR83S6UPlv1Lh4nDmj2ldsM2SzNs62tqlXH0cXqH2XdCnxlwhcNePi2nfXO7b9tq19qK7WrKe0peXrRsAIGAEjMHkETOCaPKaO0QgYASNgBIxALwQgBWB4gKTERFveKLSoy+Iwi72I7pcWA0ZNKksEro033nhqy6A2Q7vcKudfv0+SwLXVVlulxe+77747GQUjcHzhDImIBQnk85//fMPXyJaZQWCSBC4Wp/V1L97d8NCAsThKdDVeInDxbJsBWAuEkawlAteQcNQ52iSir/zTSfbvZS97WfI+dT5r6gAAQABJREFUxKITBpZx3m/U4tik20iXQSQuIPY1zozz7l15ySAffFrCNS6m9yVwqW6hB9HHeb0lYxgW2BYUwWAOeaFtIVELltEgEA12LJTPVg9c9FEsUiJsT5CTMWKfRTvna9cuieUxhMAFiZltAikT0qAd5hIXas8666zmjDPOSI+0LWLm4eN51E0lgkkkWwwhcE16DBAXwfVVbbwW3wnsMYgz/ojP1NZ34u4y8micwhjmi1/8YrPzzjun7Hz1q19tvvKVr8SsdR53pVGrv6PHkzayjLxikTmIJRBFauqgvsim7rJ9I8TQXEQGjGOv2rZC3MI+ljPX8VqJ90rJpz71qbRttc7jL8R69Qlt2xdRlyCEkVe80Fx44YVzeZwRGSHGy3EknZTaV/48523j0Np8Emepz+C6ZKb1tNpwm8GIfEUDI/US/dPW9+g9un5r5wbRC2pbOUu3yCvFqHzyUcu6666bsgsWV1xxxUOyHkkszCO0/aHqfBt2JWObIo/6HLIQcaAjEPoPvCJLZrpOKN2239iWYr/H80cccURqm12eFvJ44/vJE+ekxwGRmNFGOI5jC5G8ascBQ8fUkFVf85rXpL4SgiTE7SjMUZmzaEt49Ah6XXUw6t1I4BKeMS6OY18DmRdPKqW2Utvf5enl513j8vlljhDzOR0ELkgjNXUix1rntXpX4dt+VQdLenCSBK6hbYrxwdC5RK2eGGdMUJqvldoi+Avr2N5rx22148S2esD1Wn1J2K5xN/fbZJz1gbY4a/qf2roTPTYxfsl1P3nkg5nNN988ZZe+gjlEXD8rbcdNn8EaFWNm9XltOmsIgYtMaNzYRsjhgw3G6QhzL+ZgXTJqXKxx3dA1EtJU+4pthuu1cba1TeJExtEFat8lXUrcpTFlzbi49t3JQ5e0rX2Moxe60vM9I2AEjIARmCwCJnBNFs/W2BicLbvsslP3WQwqLdhOPeADI2AEjMA0ILDRRhs1eKW4//77R07YpiF5R1lAIBpfuB0JWiwQsuBOHyLRRF/n/I6aVJYIXNtvv32Du37ksssua4499th0rH98Scq2Rkg0InIeDWd8ufPd736XyyNF+YwTdTxxPOlJT0qkCBa2SSsKeSSvCEbf888/P9728TQiEBegDjjggCmvHHFx/+qrr051dFQ2WHBjyyyELxTZEi4Ki3x8NUidR9oIXKWthOJ2Ydp+lDgigWtIOBEXiUNkSo4l0bOPyBrjvN+oxbFJt5Gu8osLiDLW6735jYZNET/HefeuvMR0a45LuMbF9DYC169+9aup7WtIN5J3coMo9zGiUWeIG3LXHnvskX7bFhK1YDk/ErjwtgXpBJERNZ2Ef2o/bYvH4dG5tjkcQuCKZXvppZc2xx13XIw2GVLZ+uJxj3tcus52jox7EC1iclyq41zPBT2w1FJLpcsYVu644465HolejKQT5nqg42SSY4DYt2oRmKQpE7yQRYmee2K42vpO3F1GnkjGoO6zXS79/V577fWQ7fpiPvPjrjR4VvWP4776G+8ujGUYa5XyhN6j3XKf+Tv1r7YO4kkIYxJSakMRpzj2irprSFshndLYi3UJvg7X+HKUIYetouhTEbwQ8P75eC1uuXTJJZc0xx9//FwGkxJxBP0J9ur7awhccRxam0/eK5ZpaTvjSGahHk430VZ9BXljHECfG4X+E+MLZRj7k7a+J4ZtO66dG8RxWMmrXiQLYrxH743KZzRcor/R47nEMUv0NqY6P8TYFuOORi28IFM/o4cvPTvTdULptv32IXDlY5y2uLge30+Eo0mPA6JxHawpZ+pQlDieP/jgg5PRO7bXIeOAoWPqCy64YOpjK7yxyZtkzF/sN6U7VAdj/xoJXNTNt7/97XN9FLDQQgs1EGkhg0U929ZWavq7mO/Scde4PLa30vhptswRYj5LBC7eu5T/Eh6x79W8IZLvh9SJUvxcq9W7bfHpuupgSQ9OksA1tE3RvofOJWr1xDhjAvXBffpXYR3be6w7Q8ZtteNElXvpt1ZfEteocXcpPa6Nsz7QFmdN/1NbdyKxss2jVdwOnn4KAtfuu+/erLXWWukVcq+dXKSPYKcBRGPvNp0Vx0H5+Fj9ouYlxKdrHJc+yojbQsa5Mc+XJNYb9W3xudj3DVkjIQ61r9hmuF4bZ1s/SZzIOLpA7bukS4m7ROCqGRfXvjt56JK2tY9YvkPGUV1p+Z4RMAJGwAhMHgETuCaPaTHGVVZZJW39pJsMnG644Qad+tcIGIEHEXjMYx6TCEac8iUtX+kOFSY6L37xi9P2a5ACWAzEEwTbtpxwwglFTw1dabDozARulEA+YFDfV/j6hq/3MfjydeVMiCZ1KH8mYZZ5j8A222zTQLCSaGsInVOv5IGKa6Uv20dNKksErrgAwsT/Ax/4QNryBIM3RL8ttthiyrCHkY7FCBnr4oI+i5ksIGOsHyXKZ5yox0WqG2+8MS02yMMNLviZyLIgjLC4kXt8GZWm79cj0Ebgoo5Qlgi6hEUBdKzKrZQiBkbcgstYLEIBxlvqIl+pYUSXsH3Addddl06lt3SPLa74o84RlsV5ffke20dtuPXWWy99WU169EHgQH+E4IWBOktfhcgr3DjvFxdPSotjk24jXeUXFxBLxo2ScWacd+/KSwJ4jH8lXONiugwxSkL6CT134IEHJn1InY5Gcu4xjpAnQHQzZET6coSvcyHOIG0LiVqwjAaBqFNnghiQMjjiX8yTDLeR6ALZGK964Ihg4OYrZLajQyA5QXbqklgeQ4wbGJ5IW/pEC+CkhQGUfgPDAXLrrbemNpxO5vyLdZztsfASxLt0CTqGbSIRSKIY99F9lDuGN3lf436JpML1NpnkGCD2rdFjCfUYkot0KmNa8i38Yrja+s77dRl5olFYWLQZRHS/9NuVBs/X6G/CxS+lGctDEoBsTF2DwLzYYovx2FQbr62DT3ziExMZR9hT/772ta81iy++eMOX8ZSbBH2jrQVr2wpxSbepnMk7/aP6XK7Thtrk4osvTm2EbaxZ00AwWkPC0diPcebWW2+d6hT5pg2KhCGdRzj0AsZe+lbygfELo4okN1Dpev4b9VM+Dq3NZ6nPiOnGNGdCT8c+hLksOk8kLnQP/RRESCSSl2K4eD2+S9vxOHODOGaMZQJZkA9CNG6C2AfBr08+RQwgv8RJ/woWjPl22GGHNF/hXuxPOVedH2JsI5wEL6vbbrutTtNvyTg5XXUivneJiDpXxsJJF4FLmORjnBD8IYfx/aZrHBCN62SAMiOvfLBEnYHktNxyy6W8SYdxUquDa8bUkRB96qmnNuecc07KD//IP2QU6qS8y3FdeMc8RwIXz0BmBFd+l1hiiTSe5L0Q0iAtpK2t1PZ3KdKWf13j8jh+ms1zhJjPSOCK1/uO/2Lfiw7SFmU1daIF8jSXFaFp6JpMW5xcVx0s6UHqGTYJhPUXeclOF+b8i4Rs6WzuKc5Yr4e2KdYMRJrvO5eo1RPkuXZMoLFL7F/a2mIJl1h3hsxxaseJvGub1OpL4hs17m5Lc5z1gbY4a+ah49SdSHjCCyhY0IeyfkWbfd7znpeyGtfWIxGXm3i5YpzPXAydzUc/YEM8eOiifUbdFHXWUAIXxDCt8TNWgsTF2gTpobPZshyJ+U0XWv6NGhePM2dU+4q6hGzUxtnWNuOr1eoCte+SLiX+EoGL60PHxbXvTlpdEutX7PvG0Qtd6fne7EGAseV73vOelCHm5tThcWVonMwlWB+kvqE70X2sETAXZG5VErwbsq6wzDLLpHUCjbFZZ/vyl7/c6jEc/YuHVNbsOWZMdc899yT+xTe+8Y0pe1aeZk246bIT53mrPUfvv/zlL2822GCDhD24/+xnP2tuvvnmBixq6kJtnDMdLseMMTtlzLoIa29twvox69l8sM86P5ix1sW6F3afn/zkJ3MFZd5NHe0rfERF31cj9Nu0H7CkPfDHsc77xrnAHIb53C4z+oZ8hDwXO25e2QSuR0jB+zUHI1Dr2UcJlRZcdY9fFDDt8bbbbouXO49ZONTWCV0PQmKI2yp0Pcs9fQlBnmSgGRVm3PsiNPSdtI2bnsOPRiB+DcTTccGW8/gVF+csFuTExlGTyhKBi86fSb2MP8TNIpq8IHDOgpXuswjAgOeb3/xmGgBq0Y/nkEic+euVh/5XPuNEHZIlgxgGIAjtgfcjHzEvbdsaPTQVX5kUAnHsEj1wEb/KUmnFRW1dy3/50u9Zz3rW1GUZ4Rh0ItQLDAcI9QCiO3VUeivdCP94RmG5DEkXz12S2nCEjx53OKdtkBbtRkI/ApFDUvt+oxbHpqONtJVfXODpa5zh/WvfnbBteeHeOFLCNS6m53U2vjvpRj0VvRVxj7pHnSA+CZM7FgbxmoC0LSRqwTIaBKKRdCaIAcpz12/Mkwy3PJ8bIFkYYVEaQ6vaI/hAzMgnuXl6sTyGGDeIZ7vttms23XTTqShJE50S+w3K4t3vfveU9y0eju+lwCXipO7xy2SexS+9H9dif5nXB9JljAcBaJRMcgwQ6yzp5nqMfCJ6D845zsPV1HfiHWXkyXUyWwXiHXGIjEqDuPL37qO/qYu0TfVBxEO91tiEc9osxhaNwWrrINsNacsV4s1F5cKv5gfjtBXpWJVzNPrnaZfOtQUMC18sgEW9R12nzalOET4nOKy88spJN8ZnYvuJx30JXNH4rDxrHFqbz1Kfobj5jbpjpvR09O5AHqiT1Is4Fsm9U7X1PYQfJePMDSDZMBaIbSZvQ9GLcJ98rrTSSonAk9cd8qlr4EF5xC0WVeeHGtuED3VcW8xyjTT22WefKcKinpuuOhGNxmwLqY8IlG7bbxeBq2uM0xZffL/pGgfkxnXlBcxVxlzjHCwg/kpqdHDNmDoSG0kbnYXuY44c8xg9RasOSu8SLh8/cQ3J3xVvb5AeGdMgXW2lpr9LkXb8U971iMbLsQ7N5jlCzGckQ8T6rHcbNf6Lfa9wIGxNnVCa+e84ejePK56rHEt6MPah4xK4atpU3hZGzSXG0RO1Y4LSfK2tLQrr2N5j3Rk6x6kZJ8ayLx3X6Evi6TPuLqXHtXHWB9rinMm6w0cLbGsrPY+uRvfrAwjyWOqbMJpDPpDwDKJ4OI4fH7XprKEELuKl72AnBUmeNud8YMV4bJSMGhcTvnbOqPYV24zyUxNnW9tUnPzW6gK175IuJd42AtfQcTFx1bw74bqkq++r1Qtd6fne7EGAnQHQIwh6ZhIfww+JE6+1bHffJvHjVz2z5JJLpnW3uKane/ot7a7Bx004JYjrFHqeX3mAjtc4rg03XXbiPH8158zF0fO8W0lYt6Z/xi7cV2rjnOlw+fvwoS0e/5EzzjijlTS4/PLLpw9p4jpGHhdk7DPPPHPqcvyYY+pixwFjXfrVGjGBqwa1HmGYhLEgDyMfQgoDhSgmcEU0fGwE/g+BSOCSh5b/u9t9FL0N8CSGM7wZ4cUHBrcmTBheWITVolh3rE3zwQ9+MIVnsoPCbROMe32MdQpvApeQ8K8mhSCBsfm+++6bAiV6cGzz7AHJ5fGPf3xxew8iKhG4uL7MHLY4A89osOQ6beO0005rrr/++uRhQYOY8847r+GrHYTFjOc85znpmH88j1ekLlE+84k6i3IssLYN0vlKgLAsmFtmDgFtIUOKeBmK+o2FKRYXVTfwIoiu7BIIHtRvdHIUdCt1DZ3PF8AsKiOQP9CrMvqju1lkZtFc+lzxQPBjAB3rSG04xRk9sugav+SX7cjY3hTjpKT2/eIkmK+Y5dFE8fI76TbSVn4Y6jG2I0OMM7XvTjpteeHeOFLCFR0DaQSJhhjOWezcd999p7yEUN/0RTz38ezJV7Oq81yTYFBER8VFkThZFLGA5+VVMerB9ed43iFupERqof2tsMIK6X6pXAjDu7UtLKaAA//FPEXDLXMc6klcHI5RQ3JhsT/fsi8+o+NYHtdcc00ymOuefuNCNIv2cezWVSZsZYgXpXxhgjkZBs+nPOUpSiKRS8CuS/h6bccdd3xI+VOOlD1bLO68885TUZBGNDRP3SgcTGoMEOsUyaAnWbBAf+TC9n0s6jDGyMPxbBe2pfpOmKOOOiqlmbct7iHRIFW7gDEqjb+mNLdHLV3jt01/c48FP9oaHoNyYT5BH0W9itKFU1sdJDxfydOW1d9xDZ3Dtnz0kbQv8ioC1zhtJR97xe3sSHeURAM340XapDzcxbCUKV/Snn322fFyOoa8zXgzX1RFX+DJiz+kL4GLZ7vGoTX5LPUZpCOJOnEm9TT1ZO2111Y25vplC3M8mURp63viM13H48wNqLssCMd6TVrUZTwAHnnkkVMk57755CtYjA75XIV4aTN4q8q3tVWdL217SLg2Yxv3JIw/SRuJxDPd53e66kRcEM7nhTH9/HjXXXdt1l133XQ5Goe5MGqMk8fFeXy/6RoHRGIG+o96oY+HlCfK+ZhjjmkYJ+RSo4NrxtR4y+RDwXz+QX6YC/A1O3Nhiepg7F/R53hA0TwbAz+6PQoGJuZTcawzqq0Mna/E9ErHbePy+WWOEPMZCVw147/Y9+Zjm6F1ooS1ro2jdxVH/qs6WNKDkcCFbsbzdZTogYtxlwiyijPWa8INbVND5xLj6omaMUFpvtbWFku4xLpTM8cZOk6M5dd2XKMv+467S2mOsz5Qio9rM113usYgtC36JkgIuUCaQOeT3yisVTFOxjOXpE1ndRG49KFlvl5BnDvttFP66EDx65c5MfmFUNhHRo2LFUdXvWqbM3YRuIh3aJxtbVN51G+NLlD7LulS4u0aUw4ZFyuPQ99d4dp+R/V9Xel1zWXb0vP12YEAa1dxbWoSBK4hcWrMCxrMA1mvZ74GsZEPdCTMeaRD8XqEDUJjY9YJGBfT9lgLjV6749iEuNROOUbPkhbrDqxZKL58HDdOuOmyE5OncSWSy8CedSzWOiHTiZ9Cf4BXNvqQPlIb50yHi+/Cmh5re1p/aiNw4fmUMZ/W+fEUy8dTzEcZn+s6WLJmddddd6VkqANtJLmYD80diRd7Q42YwFWDWo8wbBOkAi49bgJXCRVfMwJNIjyydRsylMBF57PiiiumsLlippPCmK/OCuVM591HyAeKG28wudehPuHbnjGBqw0ZX59JBOirID8ykGZAwdcM0WsKW6YxWGbQA8kmkktw88yAmMEgxo1IZhn6DixusDjHYhVfcjJAYaBJuxvi2W5oun6+HgH0KZMvBv2Uf6wbXbESBtIAi3qQPNDFDIYR6hn3iBsPV3iWiEQsCDVMwCBx4RmRusEiVKmO1IaLeaf+r7rqqqkN8MUl6bB9XheZd+j7xfS6jifdRmrLryuPte8+HXnpymfXPeofWLPgGAlZhOGLWwhu/FF/qZ8YHCO5sSvuh9M9iPGQCvhKjn4El9xMdOlD1J5n4n0pB3SBXKVDJmURaFSZMBlnkYiFydjndeUZgxeL6Szm0z/xriwoScCCPpHFJvTSOH2i4pzELwsLbAHJohdlBOkD49somXR9j4YI2o2+6h+Vj9r7NfqbtFjcR+8//elPT3WDbThp621SWweJD11DfaIOagEN4iFbKtaS3NryOcnrbJ8AiZ92pDHgVVdd1ZkE78oYjzEl5AT6/z4eALoiHTUOrclnV3rz6h4Li+DNPJd6gcdR9FyfdlyT53HmBqRHuTCmpw3eNWeRk/wyrq8V8sOXsfS9bGeKHuPjjr4k2Zp0o+eJaFyoiWs2heka4wzN5yTGAZGYofUbDJ7rrLNOGnPRp0N+6OpPa3RwzZhaYwB0GGmyBQxbs5C/2voNuZX+mT6Avrk2ntr+rq3Mp2NcPlvmCDXjvzacJlknxtW7bXmcqes1baqvDpmEngCH+XFMMOlxYo2+HLcO1bb9rnRnsu5gfGUMwhiW8Q1zTNYo++zqgQGZ8Rt9GEQExkPxg8OudxznHvWGMRP9Fds2QWYg/emSSc8Zyed0xKn3n2ldMHRcPB3v3tX3zQu9oLLw7+QQgLzJOJ81H5GWFHstgasmTsZGfJiN7mRdkI9tov2VLRUhiiPoJX3IFT9IYO2FXUDi+D/ej2SYjTfeuNlhhx1SfKyn8K6aI7PeB+EVnYjEjyxrwxHPdNmJiXscQdfwvgjYYxOh35GwY4k+ZD333HPTLju61/ZbG+dMhyP/2tqSd8w/CNI8M3/PTTbZpNl+++3TZdb9hB8XqMP7779/spdyfumllzbHHXcch73kwAMPTGuKlAVrfLX9MPND6jD5Yb7AH8c675WZOQ95C8UMKX2lkF2eOjWBawoKH8wnCDDA3WijjZLSogOFyMGexRi0UXZc48tDGfDp5BkwsLh8yy23pK9BWZxi0ZXFLq6xd7GMfBA22KsY4xfPIBjHMKyhZKNHojbI+HKMxab8izA9H7178YU48fYRtechYbriZSK1/hxvG0xkyS+CEQMsMWahlGGro4i5ri/eYpwMeDD4sBdv/NqTr2TA/corr0zYEw/EHHCEyS5CA8qfL33opPgil44VAwyLkLj/p4xKArmCgRETWHX6msD2xbMUr68ZASNgBNoQkN5iMhY9IrU9r+u14RTev0bACBgBIzA5BOJ2cGw3yrjzkSZ4cOFLZ4T5Re45iAV75jPMBRi7s2BkMQJGYGYRYA6Ol1Xm1NFAMLO5eGSkViJmPDLe3G9pBIxAXwQeSXrC48S+taLfc4+kutMPET9lBIzAwxWB6ME3f8daAldNnNh211tvvZSFCy64oPnCF74wV3biPAvbJN6+EXnm47htrUg7D/DM7rvvnmzRkZQEWQx7cxT61W233TZduvrqq9Mcj5PacISdtJ2YOCch0fPZZZdd1hx77LFzRQu5DzsJArkXD1mjpDbOmQ7He+Dhig8xS9JG4Io7PpR2RIlEND4ie//731+K/iHXYr3Dzn7++ec/5Jm+F0zg6ovUwOdgj8JglsD4REFJTOASEv6dHxBgz2y2jYl1WPnG5TaTIiR6zFJnxpexfEWOQSIXDDe4HMRAwdYzMKFL0mdrNuI//PDDU3Bcc6pDivFBVoJshrQp7vg8x1FR45Kzr7vhPJ54HvMRr/9/9s4E1rKi+P/nHzVoABlEBUQUEWUZR5BFdhj2TQhhkXVkUQkEQYRERhYBEZRREgIJCZDAzIAwA8g67MuAyCqyCIMIqMgmiKMYQhQTzX8+/eN7rdfTfe7Z7n1v5lUl792zdZ/uOt3V3VXfruKYSQrKYpWf9/HemDTRiHfny5sBOzgBiglxrEmRBTQAhmC3ZkyA8WbOnBmQxfbeKgt22TPpE+jM3uOY78nkbpC7oeN3+rlzwDmw+HPAyi0HcC3+39tr6BxwDiw+HGDOyOYONgugHIHsTsvFp6bVakJ4s4MXhDmD2IRBeDTtLGUnHesi1h7QjTfeWFx33XXh2P85B5wDg+eA5JVdq8ehCAdfivH1Bjeuj6/v7bV1DjThwHiSEz5PbNJC8mnGU9vJc8HvOAecA+OBA6xf8AYvwpmDbLFNAVxN8sR5BB7gcdhx1FFH9Rx9qFy5XzbP4DQCL4U4p0iRwhtzDw9d2JOJgoa9OpcOj2CKpiSHHzzfJB3vHYSdmHy7INmKyev73/9+MtKAnJ/wTAqwxHVLTfMcdjrKTKh6vDiL8Oo8YcKEcJrDAdA2aCPoLOFHTBavgPd4PGn1I7yEnnnmmaFdVk1TlqcDuMq40+E9K2DI1gFcHTLXsxooBwitdthhh/XewYA4f/78EHZGEwHdTAG4dI+BG0MFxgkEmcBgiiMOChiPUHjiQnBCoIERoAC4AIr1I1zQA37iPXHYI8KhYBRR3oAsq7gutOFeAFIRW50JEQBNvILhOhlPZHWIAWWXXXYJExrKCyn8FwMByuO2AK64PHjJIvavgBC6z3ch7BG7ixmUBDzl+tSpU3shiOAb4Cy+HwRYC+QxZSXEjNIRNogJmoxReo//OgecA86BphyQ3HIPXE056OmcA84B58DocAD5HW9kYtcac8jxSCglzzvvvKCcpP6sq/B6y4YL1kBaH+ERl40tzMednAPOgeFwgA1okPoh3rHZOODr2sHx343rg+Ot5+wcWFw4MJ7khM8Tu22146ntdMs5z8054BxY1DkAgIfIPVBTAFfMgyp5AozCdoiOgw18hJ/lD9shoREJ6UcouphOOumkoBPBVivnFPYZ8gTkhadydCR44MIOjK0TKvOOdP755we7JWs60hFetkk63jMIOzH5dkFypsIa9vDDD09mia0XvkHYoAEYlVHTPIedLlUHQnUSwQrKAbiIOLbMMssEOzch7WP67ne/W6y++urh8u23317Mnj07fmShc+GAaKf0gRijsFCCPhccwNWHQV3d1odTfg7gEif8d6xzACMD4CcoFnZHHHFEsf766/eqkANwAerBJSZGCggwEG4tBV664oorijvvvDPcs2EObX7hZs1/AKWYIOA5itjzApwRrgQkbBXae++9ix133LH0USYlxHdOCfqyhExOKBcCHdeSIuvSsqkHLvKC3xdffHHxyCOP9AxAAkJwnwEdZPyLL77IaZhg4QlAiH3LJ74fEy5o3rx5wbVpOFnwj8kT4DiFVCTkoxTgesZ/nQPOAedAUw5IbjmAqykHPZ1zwDngHBgdDkh+6+3M95n3j2dCSURoRK2vYl44eCvmiJ87B4bDASmaeRsKfrx7P/bYY8N5+Th9ixvXx+mH92o7B2pwYLzJCZ8n1mgcfR4db22nDzv8tnPAOTCOOFAFbFWXHf3yxMkDHp4gAFXYYfGCFBOesAiH+PTTT8e3suc2PJ6cgWyyySbF17/+9ZAmZz/lpg3PyPNN05HXIO3E5N+GBHzD6QYRnlJkbfkXXnhh8fDDD6ce611rmuew0/UKbA6qALjM4+EQz3XrrLNOcHICnkCRq2JsQ5xO5wceeGCx1VZbhdNUGEs9V+fXAVx1uNXiWQdwtWCeJx01DgCAErAoF+5ErgYppAVcWQUoIKHnn39+RD3sQgrvU4CZoC4BXKlYzRrkRxSm5OSYY44pJk2a1HsCsBWewZiELLnkkr3rhDLkWQAGVWnQAK4U360hjcH0gQceGFFcJluA9th5LGQ6u8BQYHMth+K2k7S6IIsRBfAT54BzwDkQcYDdDssvv3zwFEh42KrUNF3V/P0554BzwDngHCjnADsU2eHIZge81r7yyivlCcbRXXjDGoMNEHjCfeaZZwJYhLWKk3PAOTB8Duy8884FG6nYlAVwy/vi4L8BXrzx+A4hA/Ea7uQccA44BywHxquc8HmibQXNjsdr22nGLU/lHHAOLE4c6Ae2alLXfnkCQD799NMXyhoHE9hTsS+KOGdTG165yohNb4CRVl111d5jsneydttzzz3D9TKwDJ6mVlhhhfAceW222WaN0gGMGqSduFfBBgcAjbCDQ4rClMrGAoyIeHXzzTenHgvXmubJt2hSlqbpcnVoAuAiqhQALku01eOOO66vboD2jbc3vMVhPycko5zZ2PzqHjuAqy7HGj7vAK6GjPNko8qBKVOmFJMnTw5luOyyy4q5c+cuVB7iIe+www7hegrABbAJdG+KBPJSDGKeyQG4QMBuu+22qWyKyy+/vHj77bcXukeYRMI14iHKEgAsPEZVURDaQZ5QiTNmzOhlBTgAz1QKH/jss8+GEIWAmRgQFW6wl2DBAW5CUU5CgwRwUUcGl5gE4GIAycWUthMywkbgclWuRfHmBUo+RXx/QsAwsAn4l3rOrzkHnAPOAeeAc8A54BxwDjgHnAPOAeeAc8A54BxwDjgHnAPOAeeAc8A54BxwDjgHnAPdcMDa9oYVQnHixInFscce26uAIgPJyxOh6k455ZQQso6H3njjjeKEE07oPR8fEP5ut91269l1sTded911xZw5c8KjgLcAcUG33XZbceWVV4bj+B/2YXkCgxdbb711o3SEwmtiJ47LM4hz6kc9ITZNwucU7b///sU222wTbsWRtuLnm+bJxvsmZWmaTu0hLn8TANchhxxSbLTRRqHN4chERFsmhOfjjz+uSwv9Wt7SFmmTXZADuLrgYoU8HMBVgUn+yJjjAHFa11hjjVCu0047rXjppZcWKqN1O5kCcM2fP78gXmyK5L3LAn5yAC47KMd55cqm5xC4m2++ebHPPvsEgBHXn3jiieBpSs/kfldZ4C6RAYud8XgOiGm11VYLiHGuyztVDnHOMzZe7iABXDb8Ie8VCcBV5k4TYBe7vaCzzjorhHkEqAfZcJfhgvnH5EDhFwnF+Nprr5m7fugccA44B5wDzgHngHPAOeAccA44B5wDzgHngHPAOeAccA44B5wDzgHngHPAOeAccA50zYHRAHCttdZaI5xJWDux6vfRj3402Bo5t/Zg3ecXWzS2SYWv4xogFvJ77rnnOA0EEOuAAw4Ix/fee28xc+bM9+6M/LG4DEIoNk1Hrk3sxCNLM5gzIkURPQkqA8ZZmy+OVR588MFsgZrmCRCrSVmapsvVoQmAyzIDxy1f+9rXet7bymzpOHGhzu9///t7+ACbV5tjB3C14V6NtFZQkAzgSgoMUiNLf9Q5MHAOWIQyHpUYWGMCGHXwwQeHy3ZglnctQifi2jJFgIMYuO2APQgAl97NuygLgC7r9Uv3m/6ee+65vXCKuEdcdtllky5Dyb8rAJfiN8cezhRjmAkN/I1JAK6yUJKgjXEnCk2bNi2ENdhyyy3DObGsc2hjUOxrrrlmeM4BXIEN/s854BxwDjgHnAPOAeeAc8A54BxwDjgHnAPOAeeAc8A54BxwDjgHnAPOAeeAc8A5MFAOjAaAa+mlly7OOeecUC85uUhV8rzzzisIjQhNnTp1RBjFgw46qNhiiy16yf7zn/8ET0aE+4vpC1/4QgivyHXAP4SvS5HeJ/tz03SpvFPXYjvxP//5z9RjnV+TLb7M5mttt9iNLSAuVaCmeQ47XarsbQFc5Akwi3CQSy21VHgFkbgAVMWEpzgwDVBZ9Ko4XZVzB3BV4VIHzziAqwMmehZD54D1wEUMWEBPMdmBNQXgev3114sTTzwxThbONYDacH45ABcDOwCsFL366qvFLrvsUghkBEgrF0MZoSsEN2CrLgZR3vexj30sFI08AVWttNJKqaIWf/vb33qCvo0HLnhNuMJ4QlQVwGV5HheU0Iug5iFCKOJaUzGlZ82aVdxxxx1xknBOnGu8j0E5wF+46f+cA84B54BzwDngHHAOOAecA84B54BzwDngHHAOOAecA84B54BzwDngHHAOOAecA86BTjgwGgAuCi7gTpm3IiI14d0IOvPMMwuiCEHYVL/0pS+FY/49/fTTAZT17rvv9q7ZA2yxchpS5nXqoosuCkCcf/3rX8WRRx4ZbLhN0tl3lx3HduIubM9l79M9Qvx94AMfKP773/8W3/zmN3V5xK8NAZkDI9kETfMcdjpbZh33A3B9/vOfD57eeP7GG28s7rnnHiUd8WvbZS4yFQ5PPvShD4V0MShxRGYNThzA1YBpTZI4gKsJ1zzNaHPgwAMPLLbaaqtQDFDMoJljsmHzUgAuBllAQDEByAKRjDcsO8jmAFxx+vgcl4YCcF166aVZoZsK2xjnpfP111+/513slltuKW666SbdGvGrQakMFDUiwXsnOQAXHryQGdDvfve74AXrvSThZ4klluihypsCuMiI75KaBIlHGvDxqgVCG8qFnuQ7XnDBBSFGcFymkND/OQecA84B54BzwDngHHAOOAecA84B54BzwDngHHAOOAecA84B54BzwDngHHAOOAecA51zYLQAXNiPsVviOeuwww5L1gvQ1vLLLx/u8QzPTp48uZgyZUq4hj0SQMyTTz6ZTG8vYoskbB1pCA/IryVsu0cccUS4BCAMxx5Qk3SDthOHgrX4d9pppxWf/OQnQw6AyIiKZYmQiHwfbLjvvPNOcfTRR9vbyeOmeQ47XarwVQBcxx9/fEg6b968gmhXKTrmmGOKSZMmhVuESXz00UdHPLbiiisWRDGD/vrXvxbKc8RDLU4cwNWCeXWSOoCrDrf82bHCgY033jh4UqI8FmSl8q277roBuazzFICLewyKuA+0hIcm8ofuvvvu4mc/+1k4tgCuHGgsPBj9I+Qfof+gV155pQBYFpONxVxloLJI7pz7yR122KH46le/Gl6V4lFcBnsuABfXiMFsqQyxDkL6i1/8Yng8BktV9cBFYiZCgOgsWTeif/rTn4of/OAHYeLFt2WAZyIE8Iv3Wtp2222L/fbbL1xKgc7ss37sHHAOOAecA84B54BzwDngHHAOOAecA84B54BzwDngHHAOOAecA84B54BzwDngHHAOdMOB0QJw2ag+119/fXHDDTeMqJAFuxDpiYhPkHUQcvbZZxfPPPPMiHS5k+985zsFtkzo4YcfLi688MIRj5566qnFyiuvHK5ZO3OTdIO2E48oeIMTHJvg4AQiAhSRtSzttddexU477RQu/epXvypwSNKPmuY57HSpevQDcAH8A7MAYefG0xbOWSzhgAbQH89CgAGJvGVp7733Lnbcccdwac6cOcW1115rb7c+dgBXaxZWy8ABXNX45E+NPQ6APl1mmWVCwV544YVixowZQZhtvvnmQTgRC1aUA3AB+kEgPvbYYwXxhhFsAJ8gzgmxCPgJ2nrrrYsDDjggHP/2t78tyLOKq8n3ve99AZ0NmhgCRAQIigGLMm633XYhDCDPQbfeemtx1VVXhWP+CTAVe9Hi/YQqhF566aUAePr73/8e8tx5552L3XffPQCbuF8ldjDPiWzc4dmzZwcgmwYKy3dcOF555ZXFcsstF3i+6aabKovGIRSVAXnjWYw6Acg7/PDDQ924D3gLEBfE9Q022CAc4waVCRCgNoh0gLoAeEHf+973ir/85S/h2P+Vc4D2w84EUPH0D4hQoYSthO67777itddeC8eD+EdbAp3/1ltvhZjiekeqXLo3qF8mQ8Q5/9znPhfc2VKmP/7xjyFkZzw5GlQZPN/uObDeeusV66yzTpD1F198ceUXTJw4sbcIQ1bHu2gqZzRKDxLWd4UVVghjGxP4sUp42fz4xz8exspceNyxWnYv1+LHgdyY1GVNAZwzn3rxxReDgqfLvMdzXrvttlvPZfg111yzENBfvEE2LrXUUuH0rrvuCjvUdM/+Mh9AuQc99NBDYQ6+xx57BMXJc889Vzz++OP28eRx0/EnmZlfdA44B4bCgUGtg5A9hOqYMGFC8ctf/rK4/fbbh1KfsfySQfF6WHXucg6LHqPuGDOIehJ6Y9999w36mNtuuy1sTOQ9ZXX91Kc+VWy//fbFJz7xiaCPYce7KHWPd+y6667h2QcffDCMr3p+WL+joWsYVt14z5577hlkDZ4eMCY6FaG9jYU+1vRbMCdF/wsRnWGQOrqmZfR044sDY2XcGiTX8Xaz0UYbBf0wG/UJg5Yj9OqMLZDGNq1Prb6dkFkK0fbzn/+8Z6wvG2dz72xy/cMf/nCw6/D97r///t44Pww9SL/yfvnLXy4+85nPFNh8kHOrrrpqsANht2N9L3tVv3z8/tjgQFUAV84emqpFlTxp49g0aeO0HQBc6MTR6dOmsIXSX6HLLrusmDt3bjhWdCMiBdE3+hGh7MgTGxIh60S8j3UeoQQJI4gzDwiwGB6nKBPUNN0g7cShYC3+Yf/GQ5TARjjuuPzyy4P99itf+UrBHxTb4rkGaI51MsRaAhs41DTPYacLhY3+9QNw8bht08g+olKhK4YABtKGpL9k7nfyySeHe/bf6aefHtZhXBuETdwBXJbbAzx2ANcAmetZD5QDTG6JTczAmyKEvu7lAFxKp0FSz3MdYBLKKRFCEWFpKRdf1j7DsQV/6Z4tn66l3BnKc1Xs4nONNdYIkwtbZiYIDESWMCTh3rMOpcqr+MO4DZ28wH1ojlSvNh64bN7KT9fi+jC5OueccwoB5HgOXsEXywsmWXVAGnrfeP1Vu8MYCQAQsu0Cz3R4qIP4Bp/+9KfDMcC6VPjLcLPGPwEFFQdcSVPl0r1B/DKhBumuSbx9B4vEmTNnVprA23R+PFwOfPaznw0hVN98880ACNXbLVA19jSoZ1K/Nh0A0tjrXyrNsK+V9cnzzjuvYKcGCwB29gyayspS9m6V0+58Knt+UbmXa4+LSvnHazlzY1KX/EChwVwGhYQ1cnb5jvGYl1VkpTzvwpOll146zCXFn3vvvTeM7zq3v+QhxRPrAjZ1aIcgxwo1Tppcf7fjSJ3xx5ZjWMe5Ogzr/f6ewXEAUAXrWxRvo2XwbTpHGBxX8jnn1kH5FP3v2B3dPB2vc/vnsGg/kWuDg+D1MDmVm8Muu+yywdiLzoYNiFUII09ujKmSvqtn0D1pt7w1auXqyoYX1hlWV6XxLncPT+7onKCUZ4Su6lKWT07X0HQsbJqurIxt7skYivcIDGJtKdeH2+Y7zPRjpY/1q3OO11Ze0n4feOCBfln5/ZocaCK7a75isXp8UelTTZlOpBNt/CePfnq1lVZaKWxC51nAR1dffXUAfGGv+MMf/lCcccYZ3AreUwCGQbK/cJwbZ3MygTRNyIZzs/awYehB+pVXIc/k2MDapZib4CDBadHhgAWmoBfBcUOKNCeL7aGpZ6vmuf/++/ccEyif2O5ooyexqUQ2MT3f7xdHErKLWaBOKh3vJgLRb37zmxG3m6QbpJ14ROEanqy55poFXtDs2iDOykbC0j10bgIqWYce3G+a57DTqS76td8Xr1ipzfVsaCekZxm/yI/+QT9iLIoJcDFjjWRnfL/tuQO42nKwYvpp06aF3d56HNdr7MZxcg4sChxgICXeK95EJNAQSoBI8BqERy2Ido7nK0hKC57Bg5ViG4ebC/4xeIKIZsIak40tyz12Rdx8883xY8nzDTfcsDj00EN7Rh/7EO9kJ8b06dOD4LX3NGFJCVt2Z+CCEhR5TORJHZrs4mX3IxNkPEGIUMBpMCB2MzsgLPG+X/ziF4GfTBpiAJf4Hhu3lMdPfvKT4iMf+Ujx6quvBi9ZeMbRN9Uzd955ZwFoLibACMTxVTxle59yzZo1qyCtU3UOqN1VAXA1DS9aVprcIjFVrrJ82tzDBe2PfvSjXjukD9IHllxyyR6gi/YFol0e4dq8z9N2zwG72Ird8DY1oNt0YxXAVdYnpQDqp2jq6muUlaXsHSrn4gTgKmuPZbzwe6PPgdyY1GXJHMDVJTf/lxdKXealUMp1PdfxgIP3BdH8+fPDJhGd69f2YcZ/dr0B5koZ1+2zXY0/KsewfsvqMKwy+HsGxwHtJkZpztxmNKjpHGE0ymqN5HYjS9OyIDsUGoE8UICylka/MF4o1wa75vWw+Zmbw5500knBgwTjxze+8Y1KxRorhvC6AC42QMmTCGvod955pzj22GNDnXP3xiqAq+lY2DRdpYbR8CHp5LoCcOX6cMPijUqysdLH+lU+x2srL9GVOYCrHyfr328iu+u/ZfFJsaj0qaYcx24qGwxRWdAFY8/Ikd0oBHgLEJc2BFn7COGv6gC4cjIhV45+18cygAvPMauttloAxQCOwZukPA+OVZ1sP36P5/tEO8LjFcQatB+AK2UPjflXNU/SbbLJJsXBBx8cNnrH+eAN+ZJLLuldxkv+fvvt1zuvcmABXDxPNCc86cU2TupFWMVf//rXyWybpBuUnThZwAYXKR+yTlGolAVrI7zgp2y+VubiZSreeNYkT9477HSqK794HAPEBeUAXNwDqIs+U2MO1ywRHYgNq6k+tMoqq/S8cr3++ushyphN28WxA7i64KLn4RwYRxwAvINiSELrwAMPDAMkLLCIfCkttNMBFC8uYQGBMfF+9NFHgwI1xzoUMbh8x/hNHuygrEp4VsBzGO4weR8GItwfokBB6DUlwi6QJ+hcBjJAN+Rbp2ypdyPs2W1E2EGAVZZYlOGyEc9Lf/7zn4t58+a1qoPNm2MQwiwgqBeLmqeeeqqvpxsGtkmTJhWUG29mpAO4BxrZqR4HUkAp3LNj5IQABsptqTW+WG939d448umcsTxVrpEpuzvTLh9yZCKJe1fRYYcdVgDKhOgfLCidxh4HrOL8kUceGWEos0As7QivUgNCXzHRh/Dq11bOVnln3WfK+qSMSqMB4KojH1jQME4iy6+77rq6LBiTz5e1xzFZYC9UjwO5Man3QAcHDuDqgImJLKxhK+XpliR2xybnKJBQBqNUs7TTTjsVe+21V7iksR9FnBSAzIXZkAGV9fem40/IeIj/yuowxGL4qwbEga4NP02KWTZfaZLfINPk1kFN32nBKugfGAPGG+XaYNe8HjZfc3PYJiCA3Bgz7DrlAFy5umpzHuMphhrrsTh3jzFHYaasrmGYdU3pGpqOhU3TDbK+0oU6gOt/XB4rfex/JUof5eSlnec6gCvNu7ZXm8jutu9clNMvKn2qKY8lR//xj3/0gMllecEP0kBEcMDTszxE43iAYygH4MqNszmZEDJr8G8sA7hwKID9CZsfoebwgIYnNKiOLrcBWzzJYsoBAER4YcKGyDF2VBzasIF4EISNc+211y7WXXfdoGMCeHPffff1wibm3tk03aDsxLly1r1OSFTC0GKTx958zz33tOZ90zyHna4urxhDADxiH8fuzZoK/AL27jfeeKNudp0+7wCuTtnpmTkHFi8O4IpcuxaZ/N56660LVRDPOQCaUBqxQ55fSJNtAbgWSugXnAPOgSKlvMyxZRDGl5yxvE65cuWtel2yAnfMgEAtAcY8//zzww6KKu58bVo/Hh4HyhTni4oBvQm3yvrkogLgalLvsZ6mrD2O9bKP9/LlxqQu+eIAri65+b+88CqrUOKsBbR++N8TRW8XtL2WMoIRuh1lGISS6dJLL7VJRhyX9fdFZfwpq8OIyvrJIsmBrg0/TZhQNl9pkt+ilAZv1ni1hm644YYQMm5RKn8XZR0LbbCLelTNY1EGAeQAXLm6a94Ue2Xn+bJ7ufyGdT2la2g6FjZNN8i6Sr/hAK5BcnkweefkpQO4BsNvm+uiLLttPfy4Gw5onACA8cMf/rBSppK96JDxtqOQYNbbTw7AlXtBTibknu93fSwDuBQF5q233goh2HDCQISb3Nq+X139vnPAOeAccA6054ADuNrz0HNwDiy2HLDGGHbHA67Am4iIiRwTOghUKjFyRZo4O4BLHPFf58DCHNCi1IZQxMvdNttsEx7G0EDfI4TnSiut1AtFSqxwvFLgAtS6NWXXAC6OUf4SthRl7rPPPls89NBDYadDXAIpdv/1r38VRx55ZO92qlybbbZZ2MnAxGHGjBkFAE8UWeymAHyFG3l2NvBOFKl4EQO5TuhNyohnJkKWWALVTmhE6PHHH+8Zf+0zKiPX8Mi1KHh6w6PelClTwjdjpwMuv3GlyndAeZAivO3ts88+wWW1wqrixYQY7XznmAbxPew7Nt544wIvWCuvvHIA0OFtECUI31FAXZ4/6KCDwvdea621QnLaB+0ZLweE0YoN6OyGIW+8JOJV6+WXXw48id0ZY3Cjbf373/8e4d6XHWCE3XzhhReK2267LbRBPHXRP2jHKHjYTc6YlCLaHO6Zab+8/4knngjtknbLDjOuAVbmm+UI9+z9+qQFcAFGYKxEWQM/2dFGOXHrTl9OEV4XcVmOdyzGYvjA+EsbuuOOO3oeyaqUJZW/rrHTkPbGbpw4BC4ACsKdwe8lllgivJ++TEjj3//+98qi76/aKu159uzZYVff5ptvHsLbILPwMki9aFspqtM3+rXHVP7xNfiN+27aCH0ZmcPOTzwEzp07N348nOPhA7lNO0Tm8Y3hFSGa5EVRCakzikMIcDy8hB/0C1w2syONdyndlltuWWywwQYhb/LlebylyRMq+QwiT/IVtakf8mC33XYLgBwA/4wXqgPywpLkvcYk+iRu/KEbb7yxxxObBp4hbyHeRf8oozIAF55fd91115CcsZe5LPLOfq869VE5qvKPHa98b4i2huHPEkpVwoEhr3ADb4kxX16r6KPImLby0uZf5VhKcp4988wzR8gJvqO8aCJ3CeENpYybkp/cP+OMM4InXtq4vjPjC164+vX3puMP74XqyB6enzx5cjFx4sQgn1NefpCl2sV8/fXXB/nfrw7kmyPJ1i7nZXpX3bozZiOnGKcA3HFOX6J/XnTRRUGuKW/kKl7W2OFIWHeMBHxP5pDMbZsQ74P/yGDKzq5eZChlsbLS5o18YXcu4yz9CjlPv7nssssWmgPgdZh5LeMB4R4ot+QTIb+Z41GHOXPm9F6x/fbbh3kwcxR2dGrOwdgQz+sYbyg/7yE/nnnyySfD/MjORwhFIe+kGlN7L3zvgDAByANkGH0NT7ZV1hBxPvac/JgPMIeZMGFCCK3Ct7r77rvDXM8+yzHefXgnczzWMsxn+EYf/OAHi29/+9vx4yPO43WQvl9deUY72H///QMvBAjFm/Xzzz8f5hzsphV1XT+1F8YPwkBSFtZMtDfaAXNe5ly0OdoeADvmvMzH8BzO2u2aa67pzfVUTn7Zvc7cjbz4FrQt5gbM40hD2xH1a4M5Xit9Xb7U/UZ6T9PfeA6L/KUfwUvmchBeNxgzWduWrSFTY4wtF99viy22CPNCnmUuQ/ueNWtWI+/ozDeY+7FW5hvSLllLs3lJG5qQRZp3xnWlX7PeZ+5I+6KtIUMl+3L3rrrqqt46nfqxNkfuWaJs9GH6O/Lo3XffDZ56ScscSERZAWtTfvgsz5i6zy/lZk2DjLRhS2NdQ9OxsGo6eFRnnW3rkDtmfCNUD+2NY8Yc1jLoNBj34EtqjkN+VdfZ/fqwLVvVcYQ0rIcVUobvihw6eEGYI/o8ugq+lZ37drVWiedxlEVzGY1pzIXrrBPJo07deT5F/XiNDCDMEkT7ZQ2PTEAHonGbaAbMwXPURTnjvKvmWXf+QH9mXiOqqpvQ8/qtIk/ayu6u26rKzm/TOSsyGp0T4zryh7kc4zt6JuZ3luJ1fNW1Zm7catunGD+lU+aY8rChBhnHda4Rtl66CluX3HEdGcyciPkN/QuCd7wPfSS6kzJC9sIXPFAyl5InSvSD6KEg2gt6OYg5qfQR8TjbTyaEDN77V7Uf8ngdAFfcNrqQxbbc8TEymnmUPGCju8UTF/MndPFOzgHngHPAOTB8DjBOoQNgbcG4wB/HOq9aov+3wB3b/7ndqZrCn3MOOAcWCQ6ceuqpQXlDYVEMvfnmm0GxhNEY4QFhaGO3jBSsXHMAF1xwcg6UcyBWXvK0VQ6h9GfBTJjBFKFck5IIhdvUqVOzMZtRKF5wwQUjsomN5bqZKpc1hKJgRgkUE0pjPHAce+yxQREY3wdwc+WVV/YuozyUlw4ACYCVLKFMZMcTkxKUx8Q4H+uE8Z2FL2VOEYpcYotjwBGtssBgB39lbNB1/WIQJY1Vkgzie/A+lPHkjRIiRSzmAevKoEib4jvFBMAKD422nCirUAKkCAOIBV7YdITYwvgCybCPIYAy4IY3Jng7bdq0YAyx9+hbGPNS3wZlu4yi/UIQYvDq1ydVTvoEZV9mmWVsUcIxihBADiijLGEAwzCcI8ZcgBC0iyplyeXDdVvOo446qvcoPMcYnyLmAhhk+atC+pbUF0MUyswUpWLS1+0b/dpj6r32GsBCXMfTD1KE6+QTTjhhxC1kGIaYFMErjGO48RehwESuQfQTFOJciwklIwptDBkxIQ9pAxh/oUHkqXe2qR/KVuoAqC0mFqHUgT4iisckG6YbIALtNSbbX3ifeBs/p/McgIvvcOKJJ/bmtvfff38I32p5W7c+vLMO/2hHGhOldFa5AbtQPhHgZys7MCoh4yCAVIyn6t9N5KXeU+cXGSJgllWUk8chhxwSjHQc//jHPy54FoOXwHpchwCLsmsasspi+x0I3f3Tn/406dGLdG3HH/KoKxIoM/QAAB2xSURBVHtIw/fhO0GpMBP0ZeQLBIALGdpGZkm2kl9X8zLyalJ3GUOQeYCYBIQkPwBPAq9jtMAAoDUk90WkBbCAkagqMZ6zAUHjd5yOPOnzFiSOPGK+DMAoRQCfkNnIAJGdm/PtqB+Ks5iYs5911lnhsg0Rbp+L57MAhnPjIuBt5lKa/2EIJRSp5jGsEwBRiTAYysBC3ZEFxxxzjG6P+LVriBE3opN+81o2ayG7NS8kuWQ5IBcAeuoX3Ev1Da6LLK9t/erKM3QV55xzjrId8Ws3jQyifrYOANT5LvpmKgjfh7kq863UHBFwEG2I50SsE84+++wgJ3Ut/rVrrX5t0JbT8po8m/Cl7jeKy173XO9jHsGYYucDcV7IXubNOUqNMTyLrGKuzuakFCEvKAfg+6oE33PrEWQIc1HIArjiujJ/EsA9fi9jYO4e/c+GFdVYpDxYf+y9994LtVfdByQ2ffr0cAqQFeM4xBrv3HPPDcf2H20cAAOyTCAL7se6hqZjYZV0TcY0W4fUMXNG1gOpNTBgSnQc9PkYwFV3nd2vD6tsdcYR0lgAAXMRxjTJKOaPzMM0nx7EWkXzOMqiuUyTdSLp69adNCnqx2srL+nvqfUZ+XIPvU1MXZXT5lsnz7rzB+ZJgOqgMtnKfaub4FxUVZ6U5V9HdnfVVlX+pnNWa0NRXvaXOR3tTWTHnzprTZuuqz6Fbg0QMeuxmJjvIU+Z/6Z0y/HzOq8rg5E/qfczhmsdpbzjX40JzJWRxXjuIly01TNpzUJaC+CKx9l+MkHvrtMPSWPlL7px5m2Q5s52bWy/cdftO7w0+gewFyAbANVTTjklzN9Z38ZjeJTMT50DzgHngHNggBxwANcAmetZOwcWBw4wYWTyixIiRUwuY/AWzzmAK8Utv+YcGMmBWHnJXascQpmO0RpAFIYIvDlBGEMwlGB8wQjBdRRFMiZxjwUru0Hx+CCFHIs+DEGi1CKRe6lySbmmtPyyi4zFHN4L9A7dB0QD4JPdU3YBjiHp7bff1mOlvygP2EEMxcbs0oSjdBN+w1N4gcGF+mNg5zrKeH0fjJMKCRV/OxQTeI/CSMPOZ/EOWYtxQsCvQX0PKdphIYpUFu8YL+Shguu0L9ok3/74448P3xjPOhCGSQx2tEvaZ6qcgH15BtCh2jRpeVZAYJsuBeDieRGgMrVDKdJjA6k1apIOgwvtl12VMVinH4ALg0W/PikFkMpIewD8A+8AXTG2Qpx/61vf0mNhJz48VRtCUUY5aQvwWEZveb2sUpZe5okDlVPGLx4BnCkDK20AWQIAhO+FwUKknY06z/3ab6lnaOd8a+pkgYvUXcriJn2jX3vU+1O/yFjkqGQZbYj+y7dCmanr1gU/oaMXbGIJ2fGNMbhSfpTkGKyU5uqrrw4e13jQKuJUDtojPKY9AmqJCX7Rxm2eeA+hv0KDyJN8u6ofvKEdUU6+ufjSb0yCHwB1oLhPh4sL/mlnLecnn3zyCO8jesb+pgBcfF+Moepf7O4V6C7F26r1qcs/3o/iGf7YPkn5MfbKOyfneG+54oorOAyEgZk+yhjBeyH173Dy3r8q8tI+X+fYytnYOMF35HvS1vEkhtwT6Mb2ewtik/KYMtjvICNFv/6ekj1Vxp8msocyNgFw9asD+eYoVb+287KmdbfGEPqH+jjH7IbHIydePS0Qn3kA8pJ2awEsMaAgV3+ux++l/shSZDBzL4hxjDFbO+ytzABgjfcZ5KtNQ7kxXCC3IDs3DxcW/CMtHp14D3NdkYBRgCrxzKI5Mn2TeQD1Zq4IWYAn7xTYkjQCvnIdUBgeeiBrKLJ1Y74IgELzIIxCGAL7zVdCppl/eHmyHnqRH/RL5Di6Ac1rc7I8ztbKp/iezi2vf2YAanXlGXML+iR8VPtiLMVbFeA8vKAOqn62DqoX70X+2Pm07vGNabvMR9VuuWcNe5zTJmmnEO2ZNsF3Z15p02ks7NcGbTktr5vype43ChVp8U/v03jJ+IGnPNa+6gf0UQhvjsy5c5QaY3jWApOZhyAv+JYYpJmTQfRDxjTkSD/Cowheo0SUHZlAXnY+zP0yABde7RhDkZ/0Q8kP+hhgqtw99Ho5ABde8hRulPfTX5jXstbAAC5CzvCOLgFcTcfCfumajmmqa+oXMB9AdI1zzGtY81tZo3QxgKvuOrtfH+Y9TcYRCyBQWfmlHeHlFF2QAFy63+VaRfM48k7NZaqsE0nbpO6kS1E/Xlt5qfToZ/B6iPy1IM94Y1qX5dS7m+RZZ/4gUAleAulntHfaRz/dhMpXR57Qd7qQ3Xp327ZKPk3nrMhmvFRBlANQLmMHYzcyU3KDzQ7yxmrHn5BwwT943W/tbNN10adiXQhlYJ7O2IQstVQVwNVEBgPSYkzUuMPYyhwJfqQ8HNtyocfD26uAf4x56K/sHMeuHcoAXP1kAu9t0g+t/LXzvJRu3n5j1bOL9q284l9AlwAqNa9nPsV4p3lW/LyfOwecA84B58DgOeAArsHz2N/gHFgsOICSClfWKGwhDIcY6KXYjiupMAksaJn8OTkHnAMLcyAFlLLKIbvQxJU0YaigGGBy3HHHBfft3MPLgLxXcI6BBYW/FNkALqXMTi0SSZMql1WusZhHeSuPWSiJLQgFRTSLZSnK8QaAYVd54ymiH2EwwoAAsWgHzCXjW7+0o3XffiMLAKA8ViFiF8AoDVCkQ7ipl1GPc4z5AO5kIMCVPeBYaBDfw7Y9lCR42hLPUR6wg03KSWtg5Zo8TsTKnLicKINRDItoj/KiZRVZNl0OwIXygraOEQGCxxhmKSskAxbHMvJwbHfgcW6VOJzH/YtrKbLfO05j34fSDt4prA1KMMAMKJcgdnBj1NWxvJ/hZQHFv4h0tA8MfPQJyi0qK4ueSf2qnLZN2m8Sg7Sswik2SqTy55r9lsiOm266qaew5L52RnKMR0GM31DTvlHWHkPGmX8YufEWBcXgExR/yDRIuyIBo8AfFLEY7+irNgyNBaJgUEORSH+KFXG8C/CNvMxZcAHvmz59egiLwzEGAtoFxjqMG9qFOog8u6ofBmvkA8YtyHohog9bY2FqTCKt5A48tnNK+KFd7pYf4UWZfzGAi/7GLnMBEfBmw9grinlbtT5N+ScgFu/HcEzfhGw/4ZxxnL4q0sYJ23bVv3mmjrxUnnV/mWcg4+kTtGdkN4TMQlZzXeBTO2+w/d7KY3vdfgdrpCjr77HsqTr+NJU9TQBc8KesDtzPUVy/LuZlTetuvxvlBXQPvy1g37ZHQOz3GE9bhHs5eEH4JtpIPL7l6o8hDGM4aRhbkJ02LJ4Fu8hAwnvwBgchMxh/NVflmgVdMg8iBDJk50ecM/7Bb8ltjDw8A9k+yDl8oP0iO/hmIr67QAD0T+TbiwtAwyJkI+tpiLEFGSCy4wTGVMApdtNDXIamcwQbFlU8VBnQCeAJUOsLPPCiG4Akyznm22AIZt7It+1Hltd2HWTbTx15Zo3IFhBDOQZVP1sH3mM9EjJXhT8ac6gL349vBlnZaL0aAUIjHQSQjrYLb0V4Tdpxxx3DqZ2jcyHXBm05La+b8qXpN1Id6v7qfXYOSx6ax8IfjKtVKDfGiHfM81ibqs+Tp5V7ln9l70NuCKhv2wVpCM+O5zORba+5uqqvxTwgj9y9HIDLls2Ov+RlQQnaQNAlgIt3NB0Ly9I1HdMoT47seoE5DfM2+jGEvoNxhHEJsmsl29/qrLPJR+2wq3HErufIH4++eCyUHLL9gftc73KtYudx8Vym6jqxzRhKnXKU47X9fqTFw/KMGTN62dhxVnNdbg6inG3yrDt/YKxpopuoK0/gVVvZTR5dtFXykczluM6cVd6jkAnoaKVLIx90Xlq72Xld3N+qrjVtui76lNUpE6qPeanmGbR/6zky1vlRvxS1kcHSSVs5mnqHvYbOBp0Z4xRlZ67KWEWf1Hzfjt2UT99I3zweT3MyoWk/tPLXzq01ZkvXRL3sN+a8q/ZNXikCTAnYkDEK3Q1zVTYiwxOND6l0fs054BxwDjgHBscBxinsg6wvkMv8cazzqm/2EIpVOeXPOQecA84B54Bz4D0OaFFqQ65Y5ZBVBlulkAWLYCSWh5KcAXvy5MkhrB+vBTyjnUupRSLPpMpllWupBbuMx6RHMSGQGOfs3MNADlnvNeFC4p9VYKP8ADghgEvi8TFzyRrbCEeAAcUSux7xKMbObYyOLMj5Fky6rLHbpkEBgVEMsh5oBvE9eA/vQ9mBIRDlkSUW9Cg2IBS9KLAhlBdVAFy27YWEC/5ttdVWBWHSINuubP1yAC7aHKA2SwBaFNIABRqARgukscoym84aq2z/ss/Ex7k+yXNSAHHMN7agNa5ZBZl9H30Hb1t42WO3a0yA6gj7FBulysoS52HPVU6rqJKnHJ6zIDil49tggERuIaP6kf2WGHfVnpXOeiNQCLw2faOsPeqd8S8AE8lFjHQAZ+hvljBUyxspngTxoIChFbLgQ5uG5yZNmhQuXXXVVcWtt946QhHHd+QZKQ550AJeU+1VAB/bBqxyr6s8rYekNvVDTjA2WbL9Td7euJ8ak3Djjzt/6LHHHgsAy3Cy4J81WMceqfRM/Mt35nujhLz88ssDQEOGdBt6Suli3latT1P+2TAi1mBw4YUXhkU735cxg3aqUGlrrbVWkCmU+ZprrgkgSY7VvzmuKi95tg1Z+UHfZxyxfVzeiagDnpn4BdyKrIGsYQmvaAAVIPsdrJGirL9b2VN1/Gkje0YTwGXHz8CwBf/qzsva1N0aQ9jcI8CrykKIQOZAUGoc4Lo1iqfGTZ6xZD3jAAaTZ1M9Y8OOCnAA4FOeilLjG2mtfJJhx87NU3NtO4YwfjPGi3KGHwu4SvVP0ssASr8HPKJxCYMUYwH9B2IuxFwHYt7MXIhxXdRkjmA96lljtPLkl81dAO8gvEEBSIMkyzmOQSpcKyPLa7sOairPcgCuQdbP1iEGXFB3K+disBX3kY2MS3i+0Fxw9dVX7827AfYQys4SBjdAi1C8iSTXBm05xes2fGn6jWw96hzrfXYOS/q2IAA7xkiO0veQBxbAxRqcORzE/C5ek4Qb5p/lbUpO8qgdQ4cJ4LJrJdbcgDYs0R6RywA2ZVwe6wCuNmOarbs9Zi2MXEb2ImvxKsl8zJLVB1jgQdN1Nnnn+nDTccQCCKgHc1bbtu2cq6t1hc3T9jE7V0vND+wcUutEeNK07qQtoxyvrby0ehDlhTdEdA+QlfuDKGebPOvOH5roJprIE/jWVnZ31VbbzFnR6yAniaIg3QJ1E2k9x/qGdQ5k+wZ1qLrWtOna9imbV2q8o5xWz5Zad/CMJfKEB8jLurpO8pFO2spRm3/TY7tm0TyfvHJzipxMaNoPrfytA+Dqqn035Zuncw44B5wDzoHR4YADuEaH7/5W54BzwDngHHAO9BalbQBcVkEyd+7cEG4hZq1VKOEdi8UmJAOLFLFKp8WyLZdVrsWegUhnd5tZjyrcI7wDhibo6aef7nlMCReif3ZXMArR0047LevpL0o66qdWwUhhMHjdcccdAbxjlaIqKLuZZJgpU4KgCMJbE4t27STv+nugmMdgBBHCB0N0imT8tGWpakAnLd/fEoAYgDGQ9UBm65cCcFnwgs0PL3UYKiGBH6ZMmVJMXgBihKwxJFx4758FTVhAlX0mPi4ziEoBZPlk09ud7CqnvW+PAUuxm5BQffyhBIvzLSuLzSs+Vjmt8csq5/Ac9dRTT4VwbSjPmpD9lgDQ6BeWbPtRG2jTN2x+Zf3KlsG+L2W44lkU3gAw4Qnhl/CYgWzjW6AITPXx7bffvthnn33Cq1QWqyC1xlmVh7ALtA8oBSiyXmUEfhpEnhYo1rR+Fpij+vErBT3HeCqAp1BqTLLem1Ao2xBs1giOEdV6+wkZJv6hSCZPgLQY4gSAsEYhm8zytk59mvJPoQoog8bLVVZZpQdwwmgPaAJCduKxh3ZA6FMopYiuIy9DJi3+WQ9GUkpbBbctn3jE95fHCoG6Yi9M9jtYI0VZf7eyp+r4Y2WB+myKHalxeTQBXF3My9rU3RpDBJ62fLOARr47O/tjsuF7CbmIkauM1H54BuCQQgzaNPIIzbjDn8AY1rhqn+cYr1uAZSAM7hiUreE2Bxa1suXoo48O6fmXM/xoPqX230tgDshn7bXXDldUFt224FZd45cwrLRdS03mCHYzRW5eRGgcAG+Q5alkOeMjfZvfqmR5LVARaTVfqSvPcgCuQdbP1iG1cQWj7WqrrRZYwphOWEpLfEMMwChrkZk5Yu5O+ETGCIDOAie2AXC14UvTb5SrX7/rep+dw5JGc4x4rlyWX26MsYBO5iCAtO66666eV92yPON7FtgT92c9SxixPfbYI5zaNUuuruprMQ/IIHfPrrUFILTfPQWIJT/GCEKAAlLFC/dYB3C1GdOob4qsh7zcHMF6yxPwoM06m3J0PY5YAEFq/mv7wyDWKnYeZ+dqVdeJ8KTtGEoeKcrx2sp1vMlLX2bzkOxmjaF5wCDK2TbPOvMHWz8d99NNNJEn5N1WdnfVVrues7K2Q5dDeDo2F0A5AFedtabtp237lN1wK7kVCmr+WaB4Tv6Zx8OY0VTXST7qY7ny2HfVObZrFrsuzY2zOZnQtB9a+au1MuXXmG118/Ybd9W+6/DKn3UOOAecA86B0eeAA7hG/xt4CZwDzgHngHNgnHJAi1ILlLLKIWu4yBlfcGW9zTbb9DiYM5LISI3raJQSUGqRyPVUuaxyzS50eR6SMdMqrP7vTj0Alw0TxaI4Bvwoz7H4i0tTDIoYlGPCsEUoFHZoK5QYShyAQ9AVV1wRQDJxOs4tYEMeI7r+HtalO+/s1454BiAFxoyqBvRUu7Eh0KxyxtYvBeCyig3KItpuu+2KfffdN5zK8GHBAwACU+6/reE4Z6jUO/Sb65PclwIoBpworQXpqJy6x45+Qjyh4AO4p76r+/zGRqmysth08bHKaQ0/gJLgEwojS7wTz3oY1jFeyQ29fSZ1bL8lsidOZ73MqQ206Rtl7TFVPq5ZoFUchjaXRjLPytT4WQtepd2JrygCISv7ldbWfebMmcGThu7xa+VBCsDVVZ5d1E9eb2z5ObZ9Ei9SGOSh3Jhk68wYgWcyjAeEe4GsEjxcKPknkEX8SMqrDs9YxWmd+jTlH+9UWhnv8VzEbnD4BFCNfgsJSCIgW1wH9e868jJk3OKfDZFJGD3KJoC36qPsLXCW8R4vEAKYEyqT0B0i+x2skaKsv1vZU3X8sf2v7rg8mgCuVP3UjqrOy9rU3RpD8ByAdwpLdm6Xm1/wvMY7tR2bR3ys+nHdAkHj53RuwUZl+QOEwbgJKYyYnZvPnj07eJVSvvqVdxU7nnIvZ/iRFwaeyfFE/OCZVHu0fOUZQBUCVHEuajJHsEC21NitvGWwpv9qA4dkeW4OpLSpX8truw5qKs9yAK5B1s/WwRrnVF8MmoBLIDZmxN9fPI1lJs/juZZQiRMmTOiFr+S6pTYArjZ8afqNbNnrHOt9cZ9rCwKwYwwbpZhr2b5IGWnvzEUAVQPqr0J4JsSID6VkNtctuMquD3J1VV+LeUBeuXv2HQJw2e9O+4tBoOQX01gHcLUZ0+K66nz33Xcvdt1113Aqz7q6Z38FFta6ps06m3y7HkcsgMB6blUd7Jyrq3WFzdP2MTtXS401qXUi5exiDFV97W+O11au57695ru2Pw6inF3kWXX+AG/q6iaayBPe01Z2d9VWLW/i8ZlyijQuxHNKwFCE1kYPwKahFNm1q+0bddaaNl3bPmV1aKkNZNTBAlGrALjaymDppCVHU3xscs2uWexYnBtnczKhaT+08tfOETVm23W7/cZdte8mPPM0zgHngHPAOTB6HHAA1+jx3t/sHHAOOAecA+OcA1qU2sWYVQ5Zw0XO+GJ351dhp10QphaJ5JEql1WuCTRg3ydDWsrYYEEM8ihi09pj5RMbou0zY/mYHfMY21H44/ksRVdffXVxyy23FHZ3Ym4nNukt71MAri6+x6abbloceuihqeJmr0nhUdWAnipnUwBXrn1Y5ZMMH1YJlzKUUUEbBog2SKi2fpTrk6STAihXzhyAC0UXYeGkEFQZUB5ihEeJDVCQc3lj45mysiiP1K/KaRXNPIdRhrBIeJzifTEBJAFcgfGqH9n2m2oDKcV8m75R1h5zZbXvU4i33LO6LuOM9Tqie/q1SreXX365OPXUU0cAggB1EorFklV2Tp8+faEwSRbMJH7a93SV56DqR13rArgswFKh8KwXu5whxfJVxzGACyMschu6++67FwoL2o+3pEvVpyn/yA+QlkJvomTGoEHoVACUHMsjCOe0B4ytyIyHHnqo50mRfNS/c3IoJS9J14Yoh7xoIbPwVAewGYrBkXZugAcI5g+AEyCr0ObcfgdrpCjr7/1kT2r8sbKg7rjcD8BlgSQymlO3sjpwP0f96qf5VNV5WZu654whKrvarM77/eYMWDad+lhVkBBetTAqQmWGJzsPZ67GnM1eY46M942Y6gC46CeUvw7FfYK0tg1wngtX2GSOwBiP58l4vsF7LEnOpABcds1h05QdW17bdZDeU1ee2X6neSHvH2T9bB1S7cUCuDSOW56kAFwYgZGlGNJjgvfwhbkb1AbA1YYvTb9RXJ+q53pfPIdtCwKwYwxlYS7Mhim8Bsfzc+7D+6lTp/ZCnHItRQJbcy/13bm+ivG4adtrrq5ay8c8IK/cvRSAy3o0rOrRtAqAS2WIvWqmdA1Nx8JcujZjGvxLkfUyeskllxR42EuRDPsCHrRZZ5N/CkDQZhzJAQhUFzvn6mpdYfO0fcyOY6l+kVontqm76pj7TfGaZ/vJdZ6JAVyDKGdXeVq+U/bc/KGJbqKJPKEMbWV3V221zZw1p5tlLjV//vzeRs8cgCtVB3iTWmt22ae0WYh3peYtXK8L4GorgzVOSI5Shi4ot2bJjbMpmdCmH+bkr8ZLO3e23zjVNprojbrgoefhHHAOOAecA8PjgAO4hsdrf5NzwDngHHAOOAdGcECL0jYALnZ3oVCC8MbRbxcwIaYAHECpRSLXU+WySp6Ucq2uoZD3pEgGOULd4EZ/UaYVV1wxhLVCUY7xnYW+iDB1m222WS9U2qxZs0K4Rd23v4TJwmgACYDU9ffA25PCJgI0ufjii20RFjrGWISLdyinOOdev3KmDOhxupQHrjoGPKvwOuqoowqMHDEddNBBBSEwodECcOFRCC80aid4bAKQAZhMXtvw4ES4nNig2sQ4S11ziiruiTBcbbjhhsXEiROL5ZZbTpcDH+FnP+rXBlKKeQtwq9s3ytpjrqw2lIQ8ruSe1XUp6cs8cOFpAY8LEN+RtjUIRdwg8hxU/eCF7ZNVPHDRJzBoAyaUUnPatGmhPdIXUMSmQljyrpgE4CIdxlFAUAJ1cA2QLNdE/XjLc6n6NOUf+VnFLuMxYErqLuOKjAMAKTESUn8oDiOn/l1HXoaMWv4TAIBsCO0sUBZgRZTPlsQnysj8hLESig3I9jtYw19Zf+8ne1LjTxvZ0w/AZT393XDDDQUgLqisDuGBzL9+9as7L2tT95wxREW3QGoMe/36K2FCUmO18uNXbccCh+z9+Nh+79hbgn3WGrHoX3idrGK4rQPg4n0AHTGGAWygz/Qjwuwp3CzPrrfeeiNCynINGQao89VXX+W0R03mCDbMX8orijIXUAGva3hfg3LrC6Up+7W8HiSAa5D1s3VAhseAvyYALsl9eEf/AczMHJHQoYAYkZ1aN7UBcLXhy7DHHL0vBi+1BQHYMca2VcDe9LsNNtigwMMKXv1EVQzM1gMX8lvrceXBLxuAtEFjmAAuO35ps5AtV+q4CoBL4xBtlvWcKKVraDoW5tK1GdNUzvjXemi89tprizlz5sSPhHWcALpqF23W2bwgBSDgetNxxM4zU+BgO+fqCjRg87R9rN9cJrVObFN30pZRjtf95Dp5al5iZVLTb1RWxrZ5Vp0/NNVNNJEn1Let7O6qrTads1qgJvOxefPmhXGaDayscSABtMcagAtPz8x/oZynWXRCrHmhso0Q4YEF/9rKYI0TkqPKt+1vbs2Sm1PkZELTfpiTv6m5s5WbXbXvtvzz9M4B54BzwDkwXA50BeD6/wAAAP//JaFXhgAAQABJREFU7J0J3G1T+ccXka4KCXEN1zxkCmXI0DUWEkUliQYRKipSZioqqSgaZGxEyJA5YzLPQqZryJSM/2TW/35XnrfnXXetc/beZ59z3/fe3/P5vO85Z++11177u9Z61vA8e61pVl555f8EiQiIgAiIgAiIQGUCRx55ZAx7xx13hO985zvx+1prrRU+/vGPx++//vWvwwUXXBC/b7zxxuEDH/hA/H7YYYeF6667Ln5/97vfHT7zmc/E73/5y1+CxRkPvPZvmmmmCZ/4xCfC9NNPHyZMmDAU5/e///0w88wzh+effz7suOOOQ5dYHD5du+yyS1hiiSViGLvf0AUTv5CmN7zhDeFf//pX2Gmnnfyp8La3vS0ccMAB8dgtt9wSfvCDHww773985CMfCTPMMEMg3PXXX+9Pjfjvm2yySXjrW98a7rvvvnD++ecPS+/rXve6mMdvectb4vHjjjsu/OMf/whwRW644Ybwox/9KH73/8i7n/3sZ4HrX3rppfC5z30unm47P4j/5z//eYz78ccfD1/96ld9Moa+r7/++mHs2LHh3//+d/jtb38bj88222xD5feqq66K6bULuqVzpplmGioPt956azj44IPjpf46nplnR2A044wzhmeeeSZ86Utfisf8v3XXXTdsvvnm8dCvfvWrcOGFF4Ytt9wyrLnmmvHY4YcfHq699lp/Sfy+zz77hPnmmy9+9/VrkoDuQKlOEqRbOjfYYIOw6aabxtgsneutt1746Ec/Go95FvHAa/+snv3nP/8J22yzzdCpTmkZCpT5YukkP7/whS+E6aabLmy11VaBcnfllVfGeugvo/x+97vfDdNOO208vP3224cXX3zRB5nku8/LnO4YM2ZM+PGPfxyvs+dG13AdUrdudCqPMcLMv7e//e3hK1/5Sjxz++23h4MOOmiSUDvvvHNYeuml4/Gvf/3r4ctf/nKYffbZA3nxxS9+MdaJ9CJ0OTodOeOMM8Ipp5wS9fBPf/rTeOzGG28Mhx56aPxu/9773vcG9CByzDHHhEsvvdROxU9fVo0nur3tOL/97W/35fl4iF133TUsvvji8Xm23Xbb8Morr8TvpTaJk74cEQ7+CG3aN7/5zfi9yr+f/OQn4fWvf324//77w3777RcvIa4ll1wyfk/1Xze2XJR7nqb8iI/6hT6mHpJO002kl98rrbRSgBti519++eWw3XbbxWP2z+p3HX1p1/by6XUu6UKvUE8++9nPxk8ft2f36quvxmc3feTD+Xy47bbbwve+9714ulN992XG6oqPM9f+9KJ79thjj7DgggvGW6BPeQ4v9I2WWWaZeOi0004Lp556avze6Rn89en3bs9n7UXVflkvz05b8M53vjMmkefknl7Qkcsuu2w8RB+QPl4qc8wxR9hoo43i4csuuyygizsJ/WbYIdThp59+elhw2it0OfXonnvuCd/61rfCL37xi/i7VCeIAH0y11xzxbjQ9fTVfN+cPjL97VRox2jP0vKLbqb8Pvnkk0PtGtfCgTJI3aA+U/5TWW655cLyyy8fD59wwgnh//7v/+J37sP1xIsQt/Uv4WD6MZ6c+K9JH+GTn/xkWH311WMUcLv88sstuqFPyjvlHqHvu//++8fvnXR5DNDhn2ftx0FN9dmKK644pButv8Xt+/l8/hly5WW33XYLiy66aKSQ0030+9Gbvu5afaa80Ed/4oknhlH0fYeLL744MM4wKZVBn05j3QuXpnlk6az7afdL69yee+4ZFlhggVi3fF+5U/y5Noa2l3EFcvLJJ8d65uPwbfGzzz4b+4L+fPqdes41CGOodKzI8U9/+tNh1VVX5Wvw5bX0rFbXUgZcXzpHO2Rjddoh2qNPfepTYbXVVuOyYtpMZ73wwgthhx12iDrH2uK//e1vcXwQI3jtH+N5xl2IH8PyOzfX0LQtLF3XS5tGGnOCTv785z8fT/m+iA+71FJLDY1RbVzTyzibuEt12PKkbjtCe027jdC2nHPOOfG7/fP1oR9jFc+uW18mN04knU2f3Z6x9Fli7fVlTq8TH+Wdcu/rYz/S2UucdfoPTecmmugT+PWqu9sqq037rLTNiy22GI8yTH/HAxP/+XEHfUv6mEi3+kYYP16ysbO/rtc6tfDCCw+l5+qrrx6aV+DeJsxdMYeFpHN+FsZ/9qqDrZ0wPerj7uV7acxSamdLOqFpPSzpX2uz/dy8z+O2yncv7HStCIiACIjA4AkwJ8BYgnkt5or547v9rpqiaeTAVRWVwomACIiACIjAfwnYoNQ7SvnJIZtMJ7Q3vngnlFlnnXXI2YDJUQakTOJ5+eAHPxje//73x0OXXHJJOPbYY+P33CCRE7l0dZtcM8OCNzbEm0z8V8eBy64ZjZ84H80yyyzRCMfESpoP3rjyu9/9LpAXcKPTheGOyXBzVLLnX2eddcLHPvax+NNPjvcjPywPuZk5Clg6+Bw3blzYe++94yE/6eQnztMJn27p9BNZfnLGX9erA9cqq6wy5Oz06KOPht13390/VjSOegdGOJiD5LCAyY9SnSSYTQCVjMQ5By6cdigjyJ///Odw9NFHx+/2D4cXJu8QypY3SnVKi12f+7R02kSzdxx55JFHhgyz/lo/OZkz1PuwfPd5mTNU5ibmmfxuWjc6lcc0bfYbhx70KnURtpQHDFQmlFP0JedxWEPP+sk/nKyOmehs5QWWhxxySHQ45PjXvva18Nhjj3WdpPV6gjgnlwNXv54PFrlJaI6X2iTO+UlgJjZxGEYwdDOJXFVyDlyUwR/+8IfRYE48ZtDke7eJU8LknqcpP+JDvBMJv72DFoZ9JpMpjybmoGK/+bT6XdJDOYdXf33T796pw+J4+OGHo2HGftvnGmusEbbeemv7GT9zjt4+H7yRolN976Z7cu1PL7rHG95xwMMJ1oR+EA5ElmclB660DbXrc5/dns/a9Kr9sl6e3Zf3XLvgjYDeedI/l3dOxfGqmwMXDoET56BiFFdccUU44ogjfHTRQcEc5qxOm44hIH02+h1eKE84X5JP3oDi++Ylw203B66nnnpqyFGYe/q21Bx8fVqYpCMPKfv0EXHQNCcvjH4Y2xAcp3hJAmdg8hBJ26QmfQTv+ETbRRuWii+D3gnAOHuG6bWl3561Hwc11Wf+ObxDjD/e9vP5Z8iVlyYOXObURR8ldfijvKJf0DNIyYErLYM+nca6Fy5N86hUFrodt/tZH9bCmxMAv3P9TgvnP3NtzAorrBDHZoTjhRDKTyqmZ5977rkhx540jP32L12lDp2Eoc7TF6WNR3x5LT2r1bWUAdeXzuUcuHCwxWkBwTnQxhvxwMR/OHfhlIF4x/lOTrG0BaaD6zpw1WkLS/2AXtq0+KCZf29605tif5E6R13EWZu89/KNb3wjvuzEMT+2tbLC8TrjbMKbA0Fah5u2IyUHAu6F+PrQltOAj9P343w7kquvuXEiaWz67FzbSUqsvb7M6XXizDlw9SOdvcRZp//QdG6iqT7pVXe3VVab9lmp1/PMM08sXvTL7r777mFFjRdqx48fH4/5uTRfN3LPwAW5saa/rtc65ePixSrmQvy8JC8J0D+29qmKA1evOtjmpL0ejfB6/Fcas5Ta2ZJOaFoPS/rX2mzfd/b5kisbTeaNesSny0VABERABAZMQA5cAwau24mACIiACIiAEbBBaRUHLj9pxACdSUCbMGQlIt72RHC6wPBr53AU+dCHPhSNUUw0MtC0t7Zzg0TiyKWr2+SaTUpWNRRyn1QwptrKNxj4MEaMJuGNXN7MRW6++eZoTDNjG6s5wJ5nRODJ5D3OSe9617viMYzs++6779AqEqy6gFOXGXxtJQi7HocGJDfZ2SQ/3vOe98TVl4gT5xVWt8AwiLBCEc9nxkHe7MdAhLAiFhMeCPnP5BtvovPs3cpNzoBOPP66Xh24iM/KOt/vuuuu6MSIQwSrS7zvfe+LbzBwDoFdFQeuTnXSJoBKjhM5By5vKMJJCAPAQw89FPniyMcKfFYWqMt+NZtOafnvU+X/Wzq94cfeJuSK3//+9+Gss84aupjyTZnEOcnewB86Wfji8zJXVksT803rRqfyWEhiPOzfFibf0D///Oc/A8YanP7MMGrOJd6IQwTmIMB3non6bisneR3fj4m4fsTZr+eDT24SmuNWT/3EJcdNzDBiv71Tkx3r9plz4OIaX4eoX6QRHd2NLdfmnqcpP+JD/FvO/P773/8ecHAxYdULW3GHY7nVPKx+l/RQyYHLG/DT+9r9u33iwIQx2uTMM88MJ510kv0c+vTOk3YQ51WcWL34fPBGik71vZvuKbU/TXWP50YZpg1Fh9PGb7HFFkPOnDyXd+Dq9AyeQfq92/M16Qc0ffaSMcSnGYdW6gXCak70NalrlBPalXe84x3xXK4fGU8k/9CzOC3RHiEwPf3002Pfg/4MhjIzsrMiFfXAG00wUOG4aU5crKhIH8ycQ71DktcPpLvOClxWD3lW4qdM0D/yzmKc++UvfznUr6Ju4+RDmhBrd/junR6Jh2djZS7vlEG4Aw88MPZ3+O7Tn44hOF8SezGB81xH/wBuMGfFUPomSKqzu+nyeFHhn0+rORUR1Dj2os+8Qwxx9uv5/DPkyksTBy6fVhz0KC/kBY589K1Ywc4kNawau7QM+nR61v5edfLd7lM3j/z4o06bY/fzfVgYeN14/PHHx5Wn6S90klwb49snrkdfwAPBiM2KgfaCFIZ6W226032sbhCGa2BNfxrdSBvPi1kmvryWntXiSxkQR+mc1xW+7+pXNfR6gtXM6OfYGJAXwXgJyd+D7xdddFFczYnVqBlb2UpinKviwNW0Lex0XdM2jTSXBMcGWxmR8S7jNZxA0Yu0ZTYXwPXe8cCX8zrjbOKx/E/rcNN2xLeFvq3jXoivD205Dfg4KV+2epuvr3XGiU2f/b9PWP5fYu31ZU6vE6ONU3x97Ec6m8ZZt//Qy9xEE33iy0IT3d1WWSUvm/RZvb7hxRUcnujT0pfDGc70BvF7J15fN3LPQPjcWNNf10ad8rqNdOO49MADDwRehqStsz486Un7GRzLiWdC34B+sK2Y22muk7ioZ4jXo/FAj/9KYxar+77+cis73m/9a22271P7PM6VjaYOXJ4BKzDSDkhEQAREQARGJgHaZObMmNtivMEf3+131VRrBa6qpBROBERABERABF4jYINSb9z3k0N+Mp0BMxMJXsxgixELowoDPBMmSplcpkE3SZ0xcoNEwubS5SdUcpNrTQyFli77ZDKKyR6k9Da8hR2Jn29+85vjZDmdKYRBPhO05IO9rcZxv0Ubk+EYBDASmGCQSTtibCd01FFHWZBhxok288O/OcjNMFiQFm+Iv/POO+OE1FBiJn6xiQ07ZpNI3cpNyYDur2Pix94AtPvUNQ6xVQ0ONb4+WFr5JK/sXFUHrk51sls6cw5cMMYYbcZj0gV/X3aY0LHzlBMmW9iCpVNaiKcklk4/UeUna7mONMCf+xojjrOK3HnnncfXjuLzMldWSw5cTesGibHnsoRZebTfuU/0JzoRQ5AJxnGrzxyDP8Zy6jXy4Q9/OBqp4o+J/yhH5IvPM9jttddeUacRrh8Tcf2Ik7T24/mINzcJzfFSm8Q5xG8/xO9rrrkm4JBVR0oOXMSBw4MZwc2I3I0t15Wepwk/4kP86pr8xpGSNtzErz7EMYyGVi4tjNWDuvrSr7SBswnlt66kbQkOCzhE5iR1RmNSOd2a1edDWp/tOS1uO99N95Tan6a6B13mV3Kz9NgnusHaUu/AxfnSM9i1uc9uz9ekX9b02b0hILcCF+nnRQO2orV2BH2JfvT9H46xDR9O/FVkww03jC8pWFiuR+wefPcOCvz2K33xGz3PdZY3HEv7oL5vXjLcllbg8vlE3L69ZYVVc4LiHOmgzaW8m/BCBi9qwIoyS32xtJ544onh7LPPtqAxnL3QwX3gTblr2kfgRQG2F/Y8SR/3t2OkGSZ+i8VuunwowZkvnrUfB1kdqavPvGOld4jh1v16Pv8MufLSxIGLLcw322yzIWJpWad80PewfPGrKJXKoE+nZ92US9M88s9Wp82x+/k6BSD/XAaspJfsfKmNSfsdtE3oDHSlsSYv0CsPPvigRVf8ZLtm6rNdS0Df1/fffXktPavVtZQB8ZbOlRy4FlpoobjSnu/3pv3gdLVPv6IM90wFNjwr5ZPxnEluroFz9pwWztpz+136LF3XtE0r3YfjtPWw9W2X14v2zIRNHQ/SvhHXwcd0OtfkxtmlOkz4uu0I10wJDlxNn53rOkmJtdcrOb1OnDkHLo43ySOu6yR142zSf+D+TecmmugTz9ievY7ubtPBpUmflRe3WK2+pN/RDdR56+OhF3nxEYcmnKWQ3DNwPDfWLLVbvgzXmXtBpzGnbHNN3NeLb5+qOnD1ooOtnUj1qE9Tk++lMYu1I2l76nlyP3++bj3k+pL+tTZ7EA5cfoVM5hKxK0hEQAREQARGJgE5cI3MfFGqREAEREAEpgICtu2An5wcP358XDWAx/eT6fzGELP00kvzNQqrWbCqBYLTAQN7W/ElHnztH5PNbM/yxz/+0R+ORiDe8veDUALk0oXTApPOSG4iwAyFOcOK30Lx+uuvDxi5cuIN1qnxLBd+JB7jrVsmqL0Dh0/nX//61zgRxgSICXmHMceWXLfjfDLRg6MMA2sv/cwPvxKRvydpYWUqJpiY0PeCEXWTTTYZcnZhlSucIbqls2RA99d5By4mEd/4xjfGiS7CpFJaUYZwOAhSh+acc86hiTXygVXGeC6cLZDvfve7ge0qq0ipTnZLZ86Bi/vNP//80VjqnYg4jgGW+n7DDTfEFfbMuHLuuecG3lBFSmmJJwv/LJ2pDmD1HwxqfgLSoiDvMRjnVtOxMP7T52VOd5QcuIijSd3gulJ55FwnYSKUushqA6ng0IOeY8sDL7zNjAHL8sSfIywreTHgMkE3sBUSknNA8m9Soou9UZxrvPOB8exHnNwLafv5iDM3Cc1xc+RJyyPnEFYyZIVJE1ZGY1vUOmLGldw2bsTPSgpW7skndEOn/OLepefhXF1+XGNiE8n8xokKw7YJhgWMwIh/i9vO82n1G8NAHX3pt53NcfL3KH33K4j5iehceN/mlNLqy7jvMxFfqb530z2l9oc4m+qeueeeOxpjvGGX+Nj2iFXJcCRGUgeu0jPEwIV/3Z6vab+sybOzvR+OMgirddoqsGnS2YoZQ0jaxhGOvMdRndWm6gj3ZUthbwDnetp3+r2wTgXdyXZmObn22mujEdaf831z9AGGq1RKDlxsf8zWZBixEBwtcbg06aQjqH/UY+o4ssceewS2KEX89jvxwMR/tGGEt/Lnt2Nu0kcg3k55xrOwglC6TVA3XW7pzX161n4c1FSfdXLg6tfz+WfIlZeqDlzpmCp13DV+tIOs/sQ2rmxdhdBvxPkIKZVBn07Pmmua5HvTPPIOXOY8TRq6id0v7TPQr8RRhpWgTGgv4VmSUhuDXkHXwjAntG8Ymaus3GvXL7LIIrGvb0Z8O05c9HGsn+MduErPagbflAFxls75/kP6chf9ILZLxenTC2NAxvC0YzgdePH6344TnlW6mAOAXerAlZtr4NombWG365q0afYcpU9e2sLxgrGlF56bFwW22mqryDDneOD7POm1pXF2qQ7b9XXaEa5h5RtW20HshcD447V/vj70Y6zi+3Hd+jKdxokkt+6z++fMfS+x9voyp9eJy8YYufrYdjq5X504m/YfepmbqKtPetXdbZVV2CKd2sFSn3XNNdeMq+6m8wKUCfQ4831bbrnlf28w8T9tFeO7JmNNX0/bqlPESR+VOWd7BvQa/VD6uTioIelWzfFg4V9THdwvBy7fZvkxS6mdLekEe9w69ZBrSvo313f2edxm+Wb7X1v5uLRKtj2fPkVABERABCYvATlwTV7+ursIiIAIiIAI1CLARCFOWkxC8xZs6kgzduzYOOBmssXCXHnllbXuocC9EWCyH0cu3nBmgpfOFgaJm266KbCcekl83rFSCRMxOBJhhBm0YNjGCMTKVUy6k24matJVUXy6mHTjrX0MepTNkgHXXzO5vjN5xrYXZhRlIo0JNwRnDNtmtEr6utXJKnH4MEyWrbTSSoE3V0kjhmL/Vj95Q75g+MGhy3NuMy0YbzB8ci8MzzBhGX0mj7xDkk97v743qRu9lEcmGin/8847b2TPKok4lZYEPmwzyh/3xYERJ4TSikOleEbq8ZHyfDglYBzBwerxxx8fcoYZqdwsXSOFn6VnSvzspb534tFE9xAfRiv6AAiradAmdpN+PUO3+5bON332Unx2nPrLtm/woc1CT9I/glMvguMtBif6xRMmTIj9J+8sn8Y988wzx/CLLbZY7NvQ30JvY2Trh+A0Qf8Qpyzre9h90G28JMEf+oL2hm0a225DmvYR6JeQZ6QPhwwc33khgWeZEmQ0PR/1EuMfWzLhHIJTje+bk0f0EylD9EW8dCqDPpx9HyQX6gCOLzkDpaWn7idjYV5UwtnR96PrxkN4VufE6RMnXbjgNEc9oI+OcbuuoAvQgfSxGefhuFOlnah7n6bh0RWMZSlL9957b9Snnfr+jEkoXzg8MGZEP3QK3yldTdvCbtf1o03DoZbVVBBrQ6qM25uMs7lHpzo8qHaEdIw06cezd2Ld9Pn7kc5+xJk+Hzqv6dwEcdXVJ23q7vRZ6v5u0me1ORT0IXOAtBM4fJnQjjCfi9MuK26l87kWbnJ/km+UL3uBiLkpXp5C0tVtq6S1Hzq4yn3bCtNJJwyiHrb1HIpHBERABERgdBFgTMXYkT4JfTL++G6/qz6NtlCsSkrhREAEREAEREAEREAEBkaAySZW50BwQvPbDVkibOs0jDCscNDEGGNx6VMERKA/BPyqTr/85S/DRRdd1J8bKVYREAEREAEREIGBEbAth3Mrjw4sEbqRCIiACIiACEylBFj90V40wKE6la233jqu+sbxgw46KNx+++1pEP0WAREQAREQARFomYAcuFoGquhEQAREQAREQAREQARGDgHeyLZtO1mNgxW2/DYqbDmz6qqrxgTzNv3+++8/chKvlIjAVE6A+ssbymydyvamrE7DaiPbb7/9VE5Gjy8CIiACIiACo5/AaqutFthWjj46242ykq9EBERABERABERgcAQOOeSQoe1z2YKXlRRN/HbU6Za4FkafIiACIiACIiAC7ROQA1f7TBWjCIiACIiACIiACIjACCKw7777xu3wSBKra7G1DEuds8UlbxoiOInsueeek2xtFE/qnwiIwGQhsO2228atPFke2uTMM88MJ510kv3UpwiIgAiIgAiIwCglsMIKK4R11lknsLKmbdU0Sh9FyRYBERABERCBUUlgs802C+uvv/5Q2tnumy1hmS/jBSqEebRDDz00bn0+FFBfREAEREAEREAE+kZADlx9Q6uIRUAEREAEREAEREAERgKB6aefPuy9995h7Nix2eTIeSuLRQdFYLITwIFrpZVWGkrH/fffH/bbb7+h3/oiAiIgAiIgAiIgAiIgAiIgAiIgAiLQnMA222wTVl555eBfnLLYcN467LDDwvXXX2+H9CkCIiACIiACItBnAnLg6jNgRS8CIiACIiACIiACIjAyCMw///xh9dVXD3PMMUdM0N/+9rc4CfXggw+OjAQqFSIgAsMIjBs3Lm5xypZKt99++7DtHIYF1A8REAEREAEREAEREAEREAEREAEREIFGBGaaaaaw3nrrxRcfx4wZEyZMmBBX3GLeDCcuiQiIgAiIgAiIwOAIyIFrcKx1JxEQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREYRkAOXMNw6IcIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIDI6AHLgGx1p3EgEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREIFhBOTANQyHfoiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIjA4AjIgWtwrHUnERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERhGQA5cw3DohwiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAgMjoAcuAbHWncSAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQgWEE5MA1DId+iIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiMDgCMiBa3CsdScREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAERGEZADlzDcOiHCIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACAyOgBy4BsdadxIBERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERCBYQTkwDUMh36IgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIwOAIyIFrcKx1JxEQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREYRkAOXMNw6IcIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIDI6AHLi6sH7zm9/cJYROi4AIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiEA9Av/3f/8XL5ADVxducuDqAkinRUAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEahOQA1dFZHLgqghKwURABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABCoTkANXRVRy4KoISsFEQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAQqE5ADV0VUcuCqCErBREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEKhOQA1dFVHLgqghKwURABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABCoTkANXRVRy4KoISsFEQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAQqE5ADV0VUcuCqCErBREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEKhOQA1dFVHLgqghKwURABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABCoTkANXRVRy4KoISsFEQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAQqE5ADV0VUcuCqCErBREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEKhOQA1dFVHLgqghKwURABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABCoTkANXRVRy4KoISsFEQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAQqE5ADV0VUcuCqCErBREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEKhOQA1dFVHLgqghKwURABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABERABCoTkANXRVRy4KoISsFEQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAQqE5ADV0VUcuCqCErBREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEREAEKhOQA1dFVHLgqghKwURABERABERABIYRWHnllcOcc84Zj5166qnhP//5z7DzuR/TTDNN2HDDDcN0000X7r777nDzzTfngrVyjD7OWmutFbjnVVddFR566KEY74orrhjGjh0bnnnmmXDBBRe0cq+6kQySQ920dQuf4/fWt741rL766vHSK664IjzyyCPdoul4ft111w2zzz57eOyxx8J5553XMeyUcHLxxRcP/CGnnXZaePXVV3t6rEHzW3jhhcNSSy0V03z55ZeHRx99tFL6V1111ZjPTz75ZLj44osrXaNA3QmgXzfYYIMw7bTThuuuuy7cf//93S9SiKmGwHvf+96ob2adddYwZsyY8Pzzz4enn3463HPPPeGPf/xjePHFF6caFvagSy65ZFhkkUXCs88+O8W0OW9729vC2muvHR+Rvk7Vdrnt9twYT02f0sG95fbMM88cxwrEcuGFF4aHH364twhH4NUjfQy14IILhmWWWSaSO+OMM8LLL78cv+fGAIPCu8oqqwT0Wpvjx37E2SaP9dZbL8w444y12ibrkzOWOP300yuNz9tMM3FRTkjHE088Ec4+++y2o+8a35Q+vhg3blzgGZHf/va3kyWPu2bCKA/Qy1zN+9///jDTTDOF22+/PY7D+oWipKfbLP9N+5L9euY24u0lb3P3nxIZ5Z5Tx0RABERABESgTQJy4KpIUw5cFUEpmAiIgAiIgAiIwDAC3/jGN4L1I77yla+EV155Zdj53I/pp58+HHTQQfHUHXfcEQ4//PChYDiDMUn973//u7KhcejizJd3vOMd4ZOf/GQ8g1OMOWvtv//+cVLthRdeCLvttlvmyv4f6sSh/3fv7Q45fjhvbbrppjHik046KVx66aU93eTAAw+MjgX/+te/wp577lk5rhlmmCHMPffcMfyDDz4YyOPRIDvssENYdNFFY1J33XXX8NJLL/WU7EHz+8xnPhOWXnrpmOYTTzwxXHbZZZXSb+l87rnnwte//vVK14yGQPPPP390nsJw9dRTTw08yW95y1vCPvvsE+97/vnnB4yvEhHASXSrrbaK7WyJBgZf9Pcpp5xSCjJFHv/Sl74UMIjipLDLLruMqmcs9Z18u/yb3/wmOrJXeTB/XRvteZV7DiIMTkE4p/GywYQJE/p6S+ng3vDSn6BfgRx//PEBx/CRKk3LVdtjqLb7HYyfGEche+yxR3Qg4ntuDMDxQYj1ldscv/UjzjZZ/OAHP4gvIqVtU6fxzic+8YmwwgorxGS0MaZo8jyM3WabbbY4pt99992bRNHTNVPq+MKgbL755gEnUKTqHExTXWX3nNo+e5mrsXr7t7/9LfzkJz/pG7qSnm6z/Ps+YZ2+JA9d6p/2DUjFiHvJ29wtpkRGuefUMREQAREQARFok4AcuCrSNMNrxeAKJgIiIAIiIAIiIAKRQNvGBxy7mFBhJRBzPugFtRy4eqFXvjZnvPETV20YfG3isa4D1/ve977AH3L00UeHG2+8sfwgI+iMGZBIUhvGlkHzkwPX/woTDgJ77bVXPHD99deHY4899n8nB/RNzgMDAj2KbjPLLLPEcvm6170uphonFiZMMIS/8Y1vjA6zvJFuwkpcU8Pqh/a8o9mBq9R38u1yHaObv66N9twYT+7PL3/5y2G++eaLDlzkdz9FOrg3uqPJgatpuWpzDNWPfkfJMSA3Bugtt6tfjZPazjvvHC848sgjW1nFuR9xVn+i7iHNESR14Oo03tlss83CaqutFiM3Xt3v1G4IOXC1yzONrYkDV1Ndld57avndi5OP1dup3YGr1D+d3GWol7zNpd33m+v0t4lrpDLKPaeOiYAIiIAIiECbBOTAVZGmHLgqglIwERABERABERCBYQSaGB8wEDPpiBGZZeWvueaaoTjbnsAYyQ5cnTgMARmhX3LGG5aOZ9s+5KKLLgp///vfe0p9Gw5IRx11VLjpppt6SsegLl522WWHVrBi4q/XLRQHza+pAxdlhrLz+OOPh7POOmtQuPt6n34YUusmWM4DdYlN+eFZoWHeeeeND8qWmj/72c+GVlTh4Bve8Iaw/fbbx1WojAarZrAi5tQgcuD6Xy633Z7/L+bJ+22Qxmvp4N7yWg5ceX6lsUM/+h0j0YELKt/61rei0zHt2Pe///08qJpH+xFnzSQUg5sjSCcHrnS8s/766we2SsZRu9/OqqWET24HrilxfOFZy4HL0+jP95K+rXI3q7eTy4GrzfI/JTon9ZK3ufyfEhnlnlPHREAEREAERKBNAnLgqkhTDlwVQSmYCIiACIiACIjAMAJNHLiGRZD8mJocuJJHH1U/cw5cbT/AoB2Q2k7/5I5v0PyaOnBNbk79uH8/DKl10ynngbrEpvzwBx98cHScfvHFF8PXvva1opOo6XeIHHfcceG6666b8uFMfEI5cE352SwHrtGTx3LgqpdX/eh3jFQHro022iisvfba0TmJrbeff/75erAyofsRZ+Y2jQ6ZI0gdB6411lgjfOhDHwqvvPJK3F6v0Y17vGhyO3D1mPwRf7kcuEZ2Flm9nVwOXG3SkXNSd5pi1J2RQoiACIiACIhASkAOXCmRwm85cBXA6LAIiIAIiIAIiEBHAk0cuKaddtrwkY98JMZ7ww03xFW4xo8fHxZccMG4AhFvxLH60C233BIeeeSRcOaZZw5LA1tdvPvd747b4Mw444zh0UcfjWGvvPLKSSbx66zARbowViB/+ctfwr333htWXnnl8M53vjPQV2K1sEsvvXRoZSnSQPxzzTVXXKGE8Kwg9NRTT8U4uv3LcfDXMBHE/dn2irDE++CDD4ZTTjll2Kop/ppO31l5ZcMNNwxzzDFH3CoLIz6d5WuvvTZcfPHFRWN+Lk4z8LP11m677RaDjB07NmAwQM4+++xJOMw555zx/AILLBANCuTv5ZdfHjBIrLnmmtEQ86c//WkoD70D0n777RdWXHHFwCpVc889d+TNm/eEf+ihh+I93/SmN8VyRX7MPvvs8djDDz8cHnvssViGKEslYRKcssR2jSeccMKwYMstt1zgD8GRgTLrhS1MeHZWkDr11FP9qVh2LM2ceOCBB+KKYMTDW/FeuMdiiy0WtzMjj1MZyfxIa1MHLt4QxvBIPl5yySXxsSmrHMfw9Lvf/S7WAcoIZYd8Ik+pj+eee+4wTJwnHEIZtLLhA33iE5+I27SyMhur/1GullpqqRjkiSeeCH/4wx988PidejPbbLPFskp94d4loSzhPEVeIs8++2y4++67Y7nxjjDoFAyGpJnt69B5hL3zzjujHuF7TqrqhW4OXO9617uGVnz7xz/+Ec4444zc7YaOWZ5QbtlSbbrppgukZZlllolpZ6tS8u+ZZ56J56gXiy66aJh11lkDXHkutuTLrSzHaozkN3VlpplmCrQBrPpE/eWatO5aWuqWD3uYuvez6/hExy255JLxuUgf7c6tt94adQR6AL2UW/WvTrtFXd9ggw3ibU877bTIk3KFXuN+HKsr6MdvfvOb8TLy+4ADDihG8YEPfCCstdZa8fwVV1wR66APzLYjm2yySeCZyF8E/XfbbbfF/PJh+b7SSitFZuQp9ZnrKDuLLLJIbCeuuuqq+FwvvfRS1AWUBerFmDFjYt6zDSltRUloo6vq2VIcHO/kwDXDDDPE9oVnR1L9Qhu9zjrrxGcin3gWyjw65p577onX8I/yzdZWSKd6Z/qO+kabVNIH3fpOcN50003j/VjZkT7EKqusEvWT9Z8ov+eff34MY/9K7bnlpenKJZZYIvZTqJPUK1be5JnJs5ywyptx4ju6kf4W13GcY1xLOr1U1Xv+Gv998cUXj/1GdBL3QKin5BNc0CUmb3/722M9py9BGaTcooNOP/30SdJl1+Q+m+jgXnUb6aDu4vxEXw89jU5GL5144olD/SvCWRnj+/HHHz+sjLECG+0eQjtKefdC33f55ZeP+hx+9CerCNe85z3viW0kdem5556LOoC+3F//+tdhUaQOXE8++WS8dp555ol9YvoBlBX6ryWpk5evf/3rw8c//vHY/lj/II13vfXWC9zf6m6dcpXGxe+2xlBV+x3cs46+ZExEXiN77LHHUBnJjQGs7LbdR7C+Bv0CyjL6hTrMGIFjF1xwwSRtoqWlTh+hH3FGcC38M0cQc+CqMt6hrm211VZxTGHjNJLCmPLDH/5wHLvS98T5jfIMV/oXdYW2j60aqQu0fdRT+oP0lXfZZZfYd0aHsppnKvQFqo7n7dqFF1446ibGDbTLjGPRz7RhjMO95MYX/nyd/hzPSX1A6I/QdrHCGelhfMBzc3/mAUrt9cwzzxwdD3lu8oHyzHiCftY///nPGHfuH7qQeojuYdxNm02/CX3EPAHCCqu+HUvjqaOr6ujN9D7+N3lEHiCszE17u+qqq8YxF2WP+Zurr74627/beuutY5/CxlzoZrjB6dBDDx26DWWOtoq5AfpX3IO24c9//nMsgxbQt2n0L8455xw7NezT+qamx5gXSOes/AWM5Xgm+hZ8py9Du3TzzTfH1QHRUSUHrjq62N8z/V7S022W/yZ9yU79U9p++vlIm/2POnWMOt0pb0lbHR3RNqN0/pP0eKGeMgaHJfNP8LT+PeHou5133nnDxiAcrzPGbKoLmNOiXtCPpg2nnaFu0A8tzZNSv/vZHvDspOWDH/xgHKtRX5mTQKfcddddcR6Eei8RAREQAREYLAE5cFXkTcMlEQEREAEREAEREIG6BJoYHzDasNIWcscdd4TDDz88OgExyE8Fo9BXv/rVocMMujH+5ISwTOoxQWDChKdNuGL0ZqIfyRkffLomTJgQJ0rNWGvx8YmDB84ZGC5TYWKVbTiYlO0m/n7GgWswgmIoMcN4Gg8T+L/4xS86OpGk12y88cZDji3pOX6Tboz71nnOhfHHcvz8xBUOHji7mXCON8GZyEyFSU4mpxG/BYg5cDHxzzMzMZsKk9U//OEPo2MUBmdfVnxYnFNSA7U/78uxN1IRhrf7mfhFmGwmXV7YvoVJQAzacEEwBO6www6ByaicMFn0ve99b5gxlfBMACO77rprnIS2a0c6P9LZ1IHL8pkJSFgjvixhOMZIAeNUcIzwE/kYU9g2BsGojIEjFTOE2dY7GEQwfFjZ/P3vfx8n/u06M4Lxm4k96gnOKiWxVY7S8+iUQw45JB7GcYX8tnumYbkP29t5R7G6eqGT8wA6FF2KcK9f//rX0XiXpsP/9nmCQQUuafqJ60c/+lHUubn6itGEck84E4ym1BvqTElSA61PS53yQfxN7sd1OLvg4IOxKBV0FO0PBrmnn3467LPPPsOC9NJu8XyUa2ON4e7nP//5sPir/KD8UDYR9CbMSWtOYGROjRgpMUyaYBjfcccdh5xg7Lh9Yuz66U9/Oqwd9rqNOonTZCowRP+zhSNpTeXCCy+cxEG2iZ5N4/W/Sw5c8MD4bGUa/f3tb387tktcT5nYaaedAkb1nGDMO/bYY+Mp9Bh1gE/qQWn1GNMjhMH4XnKQ4VynvpOvK+gTjLg5wfkOnWPir/PtueUlZQgDaak/huMlhiMv8803X/j85z+frev33Xdf3N4TLp5XXb3n7+e/e6dEf5zve+2111DfByMxzq05IS8w3KdO3rmwHGuigz33uroNHfWFL3whWx5ID3mGodTaRd9m4zTuHaFw+iMtdh1tpBfrF8GEcznHXB+e7/RrMLKXhL47dcPEO3DRXlN+ckK7QruTrsJUNy99fuFMdsQRR0xyu7333jv2zW1sUrVcTRLRawd837ObA4bFkRs7mL6wMPbp+x1N9OUnazhw+bLbZh8BYzMrRiLeEchW08NZhn67F5+WOvWoH3H6dDX9bv1Wc+CqMt5hPIG+9nwwjuO8Zf2JND2MRWm/q9RnrqVtZNyVG68yrsSZhnbR55vds26/iOtw6rEXaiwe+0QX4ZRDfpvkxheca9Kf8/WONop5AOJJBd7MA/DMXpiP4AWSXP+GtDP2uOyyy/wl8funP/3p6KSRnuAaHNes/e+mP6rqqrp6M02X/+11OGN9+pW5ssdYjrkgyrcJY3sEZ0D6LTjrIXXKM/2Kn/zkJ7Ft8C8xdGq3vvvd78Y+CmFo52g30zmrmJCJ/+gP0/fDKSQVXvBAd/G8qQNXE12cxu9/l/R0m+Xf69SqfclO/VOY2NxPW/2PunXM12k/DwfbJjqibUalOS3Le/TJCiusEH+WxlaUY8ZP/sUfPzfaaYzZRBdQ3tFZlreWVvskPcccc0ys13aMz0G0B9RH9CT5nhPGxLS1JQez3DU6JgIiIAIi0DsBs0Exj0g/mbaEOSH++G6/q95pmolvN/xvtrnqVaMgnBy4RkEmKYkiIAIiIAIiMAIJtGV8wFjEZDMrBtBBY/KYNygxHDOZjPiJBCYAeGMYpw+ca1glAeE4hhxb8cJPUtRx4IqRTfzHZCL3wTCfm6ils8kENYYf0o3wFtePf/zj+L3Tv9LE0TbbbDNkPGcCHMMVE8FMFNokOROKTCyWjLr+vqxshlGP9MGHCU24YgDgrWGbTE6NZz6O9HsdBy7vAEM8MOX+vCWZOm3kHLjs3qSdMoGRjglrm4DhN8Yd8uBzn/tcNBbwZi/CZAznceBi8rgkfhLMG6sJ7w1jcGfyxwS2X/ziF+NPjNlMwCNm5OM71/AWJJxtRQyOky7CWR6aYZxz3oFrNPAjzd4YzBuWOUME4VLJTTD7SVALTz3DgY5xC2XHxDvnNXHgIh6cC231OPKLfMFIgMEAQ4xNzvOGKxOhnYTygKMHdQuhDjMZSPkjrcSJ3rSyj37BAE3dZnLRjBQ8L+XApK5e8MZonBe5N8KqKzh0ItQptsjDGNVNcnmCXuLZfLm2eIgbp0aexzu2pAy9UR/m6G6uQT/563CYsZW4cmmpUj5IW5P7cZ3pPL4jOOnSTlEWTfdzPHXg6rXdIk4TmLICoDkD2fGqn6y6Ze0YehgjO3qLlQGrCHqVsstkCsIkC9dixMUxw8o0ZZ72yYzAXrfZfSgb5Jm1+XacT67DSTJtI9iOiXuaNNGzdm3uM+fAxbPhHGDzJeggDHzoCSRlgl7HcEd4Vp+wsuEdKXCAw4kTyTk6eaNnt3a5W98pV1fIH+oS9Ys0mngjmr/Ot4m5vCRP0APoPHiZUFbM2ZV78dt4mH4gvLXXdp134Kqr9yyO9JOVLmw1CtPn5CWCwYZ88/0A0ke7Tfop2+hnSzsrcbFiVDdpooM9d4u/qm7bd999Y1+V67iGukle46Rs5ZfnovxSRllJ8LOf/Wy8TWq8NActSwP9IK8nzHGd9qvTan52/fvf//64whq/qTv0k+n7wNb3n1lV01ad8PXA4uF5yJe0fmFQtrECYZvkpc+vqg5cVcqVpT33SZ2wvOnmgGHX58YO3fodXNtEX5YcA6w9JD9sdadc2W2jj1By4PLlF2ePe93qS7m0VKlH/YjT8q2Xz9SBi7LabbzDOIm8YbzFKsaUM/INPYYeQLdRp9HNCy200FC7Tt+dPnwV8eXXdDqfvu0jHsqBX4GrSb8Ix+8tttgiJgsdgg5D/9CHQMeZHHbYYXH1S37nxhcct/LLd6RKf87Xu/9e9d8+NG0pbQrtn7URvr0nLCuEfupTn7LLYntJH4hrzDGckzhTeAc0P67iPOmkn8wz29id40g3/VFFVzXRm/+9e/5/TofT90SHM6/inx1HraOPPnooInPgGjrw2hd7WYqVh3AUMcGxnnhxqGdeyPqpPi/oy9lLGDgzpyu74ihGvUKs7+Xz3beT9Alw/rY8p0ySBuaj/HMRV+rA1UQXE09JSnq6zfKf06nd+pKd+qe85Ndm/6NJHSvlLZyb6Ii2Gfk+TS7vfX218+ha5sr8fALnGDfaWN/Pjdp1fKK7bYzp4+Z41f4weo68sPior6QHHW1jeuqKzbMQblDtAS/gWRrQpcyvokeZy7O5F9LKvI9EBERABERgcATkwFWRtU0aVAyuYCIgAiIgAiIgAiIQCfjJ226Th4as04QJbzlyPjWCMxmM8ZaJMib/WMnGG5PYJgJHF8RW1uG7n6So68CFUwUTiEz6I944xm+2gmLbA4S+FBPkTBhizCSt3aTEwRgwwYFjkt2f+PwkXbpSUOl+O++8c5j/tZWgeMOVCUgTjKdMUmF4txUF7FynT5vYKhlvvMHXOw1gmPPb3vmJHu5XcuAibRgMzYGDdMPbJlyYbGHSBWHrNv4QH188UPjnHbH8BC0rPvBGvhdvyGTpfd5oR8zBhK1EbIssJq4Iz4Q7Qp5jGDVHPD9Z7w3j3oFrNPDj2byhoW0HLia/WXXO6oJfHYR6am9HN3XgIv2+fuPkiMHMO1rYZD5huwn6iol9xDsj8BsnAtuaKrc9HW/c4ryEUM8oQ0hdveCN0ebAtfbaa8dtG4mPCVnqRyfHRsKZpBPTbIdi200yIUpazWCC7qK+kjeIN+J4wzhGDq5DMHxQj0mXiV8xwNeVNC1Vy0fT+7ENBKtWIDh6UFb4RNAROMhStxHfdrXRbhEnE82sukUZ7EUw4G255ZaTREF+oT9tq0+2/svJtttuG9hOA0kdJpgExzhmuo1tcVjZDfG6jfxlZRtWEkN82eA3xubvfOc7Q3y9EcGvqtdUz3KPkqQOXDi7YXS2CX8MxtRDc0wjHv9sqQESAwpbSJnDEMY0nIZYvXO77baLycgZC7zegSEsu4npB1/+uCatK6nO8e2l13H+Ot+e++clL1llyxxuuJ93/DG9w3FWVrNtZXG0oA9ndZ28RKebIdTrTHuutvpDtsIO9ya/TXAWpF6TBu5F+uhLmrAqF0YmhPxHv1u7bmHSzyY62HMnvqq6ja0tP/axj8UkMAmKLjUdxUFfj8wAzrPSP0Fvpw4WHPcOAmyPe/LJJ8f42TKMldQQn8fxQOGfrWoCW2+4I7jvZ/n+e6obKBc4HFu5QRdhALZyY32wpnnp88u3U/6RzPCe9pdL5cpfm/ve5hiqU7+jqb70Yw70u5X5bmMAnrWNPgLx4OhhL0qgL1lRx4S2gnEAbRaOOyZN6xHX9yNOS1fTz9SBy+Lx+jsd75hhGn3A2MmHZdtav5IgbRxlkbqU6gK7V/rp+0Vcg84wh13aPvSrjdF8nE37RVbHSId30uK3H+v7MVzOgcWnu05/zo/ZuSdtLSsA2zP7dp15Ctp+Ez+OS8dH6G62QIW91ytej6HzcG6y7bkZszPnYqtDc5+qczDGcRBtYKrDvTMVaUaH069ESI9f4dg7cNFu0BeiDTD9Txtnzt/piyE4xtEXsTbsyCOPjGMdrxd8WxMTMPEfzlu2Sqk5vPh892XLh2U+ij4D+Y6wKhJ9bWubvANXU10cIy78K+npNsu/Z0cyqvYlCWv9ON8/hU2b/Y8mdayUt011RNuMYNdJfL+OcOkLIYxbeRbEr3Lu9SXn0jGm1z11+sP0oegjkbfUU/Q0zvom6ERztiBEOloAAEAASURBVLU6O6j2AJ1gc7PpuIs+MCtn89wIjs/Mb0pEQAREQAQGQ0AOXBU5y4GrIigFEwEREAEREAERGEagTeMDEecmeTjuDZq/+tWvstt92eQNkwY4PjH49pMUdRy4iIOBvhkrSIOfKMlN/NmbnVzrDYNcm5PSxJFN1DORS5zmtEIcGDLM8Mu2YlUMu0zW4hDCxBn5lYotcV813VzfzXhjBl/vMJBjRlx+EtYbIGzikTDHHHNMfCuQ7ybeIOyv80YKf9yuK32awdIbGmxlJtjYRKw3ZGLEZfLJT7pjRMTBjGswCMPdC04BlFWESTMmuxBvGDcHrtHEr18OXAzozBkqgpr4D+MFBmHETwj34sDF5B71zfL5mmuuCfBHmJQnnygbVaSTIXWTTTYJTOBTPnBC9CsKETfnx48fH2/DSn42+VlXL3hjNEZ29KE5jnFv70QTb9bln5+Y9sztMso6+gnxzlZ23lZswdhleghHAHvbHYdUc4i1a5joNSOYNzj6tNQpH03vZ+0c3HgO7zxMWldZZZXw0Y9+NCbbs2mj3aLsYYjy7YDxafKJwyllzFbLysWBPiMvMAbYJDbtFXWO+kFa0FGpoPfQfwjX2cosXrddd9110QnDX2tlm2Pm5GTnvXPtlVdeGX7729/GU031rMWb+/QOXEzm47xlBkKcm2gjKAMmTPZTh5FcOeS4NwTxZjttGWIOLcRn/ZV4YuI/a4swnFD+/T0tTPpZ6jv5uuLbG7seAzsOC4gvu/46a88J4/MS508Mo158XTBHLN/XoVzQt0rLM/0a25bariNeKxtt9odwGIKp76f59suvAuWfzafR9yd9GP+9iQ723HNlqtT2mY7i/ubI5NPCd9/XMmcc6/9xnnKIQwOrveIMgMCJOo/jIXUTwVEMpwOE8o/TZTehjGFMv+WWW4bqgL/GyjyOvBjxEW/8L70Y4bdUs1VQm+alz68pzYGrqb4sOQZ0GwN4XWL53KSPYNeWPnF8mbhLRiyn6Ep0JtK0HnFtP+Ik3l7E9GDqGFRnvONfdMr10djCipWwGfdWWUXa6xzSx4sPXvyKOH5c1bRfhK4xA3tOxxEvjmj0mWmzEBtHeucFSze6rU5/zrdjXMuYAB3txetYXp5C/JbluTaTMN4ZiD4CfQWvx7wzJOERxpLcz8YsvTpw+fu11QZ6HU7Zpd2xPuV/n2L43IofX3sHLhzlbGV1rsNBijkZxDuexwOv/UMvUJcRriUOHDVsC2scsWFGXppYO+T7uD7fzYHLs0fn8OKN6R6Ly7dN3oGrqS62eHOfJT3dZvn3OrVOX5L0lvqnbfU/mtaxXN6S3qY6oh+MSE9J/LwkTrqUrVR8u0v5Zvzq50ZzY8ymusCvmJtbyXHcuHFD/W70NO3MoNoDr4ty85G8BMwLdggvDJpjbspTv0VABERABNonYP1pxvvMF9C3pc/GH9/td9U7awvFqqQUTgREQAREQAREYKogYJMcPGzVycPShAlxlCZ57D5MuOGQlBPexGf7C8TetvSTFN7gljM++HR5JwO71zrrrBPYBgbBeYr4vPg3y2zi1p9Pv/v72aQgYfwEMBOdOJKw1LytPpXG0+Q3k9wY6ZiwwEmFTjGTmN6g2SneHD8/cWUGX//2X/rWscXPdm5s64Z4hyubeCyly2/J4+OuY9CwNPBpbyXz3QyZtpoIxksMBzgomCETZkz+82n5xyCDY0inrYVsGXf/bN4wbg5co4mfn/Dz+RFhdPhn+ewNLL4sXXzxxYGtvVIxBwhvGOrFgYv411tvvbDBBhuktxq29cAkJzMHOjlwZYLHVQrYSgrjGROI9ta4d+Cqqxe8MZoVV2xlJO5v+jGXltIxnyfekcbC77TTTmGBBRaIP8kbtlvwgtGElYgwDmLEKQl1iK1/cLIgL+xFo5IDV53ykbtnlftRpwnny1oalxl+vOG6jXbLO7Ok9+zlN6sM4ITB1kkwRo+lwmQKk/8YprwjVac02eolJd2WrgLJPa0ucx9zHLG0sN2NlZfbbrst/OxnP4t50VTPWry5T3Pgop+BIwtGOoQ8hQPP5MU72JrziD/Pd+8gxTaDxIN4457vm/gtvKxdiRd0+VfqO/l6e9VVVwVWMUvF6qYv3/46a8+5zrdTZgzy8aFnzCnZVmnzZccbMv116D9zCvTlq67e83Hmvls778sn4ShjlDWOY4xNHcwIM36iYy3Oj0jOETGecP+a6GDPvY5uMwcPr39cUuJXVs3CiRWxNsD3l1iRBz1rjuvUA1blY7tPXzfJX/LZl5cYac1/tJP0Q+E6//zzx6tLDly5PjcX+LbW6kvTvPT5NSU5cPXSLy05BnQbA/SzjxALymv//KqafmvTpvWIaNuOk5VEKVupsBIhdbyKWP3uxYHLO9dyTwz5OMzgLJTTd93SZWnqpHOsbfe6omm/yDvQopvoD8CPel+S3PiiaX/Oj9lLzhLWvpAevpNOPzbCKYl8T8VvD8nW1r/85S+jcyzjTtok75zor/UrbFedg7E0DqIN9E4TtrKxTz/faV9w4kBMh/PdHLhy5cuvfu3nDbjOBOd7+g+Ij8O3g36FU59W72zv893S58OW+gJej1i/pxddbM+V+yzp6TbLv9epdfqSpLfUP22r/9G0juXylvQ21RH9YER6SuIduM4666xwzjnnTBLUHJI5YSv3+7lR39+2i5v2oew64kmdLi3u5ZZbLs6l4XhJGzSo9sC/YERaePGA/i4vK5njgKVRnyIgAiIgAoMlYHpYDlxduNvEeJdgOi0CIiACIiACIiACwwjYwJuDVScPSxMmxFGa5DHjOGGYdMyJN0Dj8MHErp+k8EbSnPHBp8ve1vT3WWuttQIT8YgZuvz5thy42Kpniy22mMSgzsQ9b40x2cDkWR1hEphVEzCSYVD2rCyedDLXjuc+c/z8xJUZfP2bdeStbanm4/QT134i1iYe/Uou/jrvUOcdhvyEoI/PX5v77o2ztj2mlTuMHGxVwRZQZsj0hnZbFc5vBcQ9qpRV3kDlGb1h3By4RhM/P4Hq8yPH2h+zfC45cLFNH2+fp2JvMXvDUK8OXNzDnPbsfmz1xvZ1dcQblXOTozgysdrBMsssE1f4YVI/J96Bq65e8MboNO6ScToN53/7+u11qYVhiyMcNRAcYdKyb04iOQcuVilie0f0lDmvWbz2WXLgqlM+LK469/OTvmyhZ6sVWVz2aU6Z3lBk+oMwKQ+7zuviXLuVbolh17X5SRrQbTh0saWNbXvEPcxpCidbnG0RS2f8kfzz7aCtkuF1GxP8fmVLLjenL1+XLdqcA1cvetbizX2aA1d6jryjr2Pbmdp5c3Sx393yGKcwnIMRnBRthTJzCua4Z5VuE8X5kpT6Tt3qLfHljOz+OmvPCevTZ47OHDfx9cWMln51hAsvvDCwdUoq3rDpdWZdvZfGm/4uGa+tDPo8Sq/1ZdFv3ZuGs99NdLDnXlW3eUN1upWcpYVPVmC0FQZs60PmASnbiNV18pUt0CiX9H1s+1j0Gc/NJzqjtJpMjCzzj/aBrTLhWFoBsOTAZX2sTLRDq7TZtjhN89LnV6mNNOc1v+oqaSqVq1x6/bE2x1Clfkcv+rLkGNBtDNB2H8EzS79bn807yDapRz7eNuO0vo+Pn++d+hNpWHOW6sWBi74Vq0pSTlKh38LqeDhKVll5xJe13HjZ4rf64tv2pv0i9AbOtYzVvdDuoqtwpEFfocNN0vGFb5868c/15/wcga0eY/exTz9mszkRK0uEKfUROGd9QdPh1qaXxsBcw7a+tJGI3S/+6PCvpKua6s0Otxq2imLJ6YfxEHUE8frfHLhweLPvdi/vhJXrh1g4q3u+3vDygq386+P2cVrflXh8vpsD1/rrrx8YbyI5XRdPTPxn9db6Qr3oYosz91nS022Wf69TS8+c60uSXivLfnzE8bb6H03rWC5ve9ER/WAEp5J4By5ebqEPlwrjOtte28bRfm40N8ZsqgvsOtKAnsGBtZsMsj1gW1NbWd2ni/kn9C7th6167s/ruwiIgAiIQH8JyIGrIl85cFUEpWAiIAIiIAIiIALDCLRpfCDi3CQPk5pMgtURDIQYCv0khZ9wyhkf/EROznjjHbjMwcenyRuue1mBizh5IxXjMEY0m9T19+LtBNgzsdtNMLzjeJbGw0Qyk+pMVjGxz++2V+Dyk2o5pw7S7rc58A5XNvFY2jqnbQcuP2mH086ZZ545tH0bE0xs52bbpPGb8sCbhHCzt6O9M1q3fLHz5tDgDePmwDWa+PXLgYsVY3IOi00duMwQkFtCnzzx+cDv3NYlHO8k3rjlnRG4hhVHWK0Ko0UqVp/NgcY7cBG2jl7wxmiuZSIVHWB6oI5zCNf7ielcnngHrpz+M0OKd+DCiE/5x3ErFQwu1H3bltEmnqukhTC58tHkft7gkmsXLN22UpAZKODcdrtl92rySXlg5UXKQc6R1sfpV/4z46Ff8cBW7/HX2Hdff8wI5o/lyoZN+vuyYfF5pxlzMOlFz1q8uc/UgYtVSegXIN7Jyq71q37asU6fxtLCmJGYNsRWfbJ6koa1a0qfub4TYbvVW8LkjG7+upIDVy4vfTtqRkvv6JbTHaSh5MDFuTp6j/CdpGS8NkOr1d9cHL6P2MSBq4oO9txLrFLdxqpaGJ+R0mognPNx/+lPfwqsWISY7rL6ZwY1jFn0g7gfYsatT3/60/E32/CiE6tIqa5Q9lkJwVaI9AZ8v8rJscceG2hLc2J5Z9fa77p56dvMkq63scNocuDqRV+WHAOMg9dTvnzlym6TPkIuv9Nj3kDNuAFd3S0txJHWIx9vm3GaTvfx872TA1Ea1sq0d0QhTN0XVuzlAYzY1s9M7+VXMkvP2W//AkunlxxsDGMOXL30i7g3dZRxGG1CztmfF2wOOeSQ+LIT4W0caS+INO3PEZfX/yX9kHPgMv1KHFXEnMMsz00v5671bWuvDlx2v7p6M5cuO+Z1uJ+DsfP2affGSR7dgthYLecgyDbMbHvfbd7C8t9evLL7WZ+TNhluCPWUMpU+v893c+DyjjNs682KgzmxttT6Qr3o4lz8dqykp+352yj/VXRqri9JGkv9U85Z/bBybszq9D8sDuKrIlbHcnnbi47oF6PSM/lyaHNJaVhW9mYMhti2hqW5UbvW6mNaF+w8n56d9YfturR/5K/z3wfdHnDv1VZbLa62zgp5ObE6njunYyIgAiIgAv0hIAeuilzlwFURlIKJgAiIgAiIgAgMIzAIBy5uaMuZMynAstzdhO27mJgrTVLkjA9+MiI3OTtIBy57PibbWaUHRyFWL2ClBZMqkwwY7Mkjc9pgyXIMfDfeeOPQW9YYjseOHdt1ItTuy2eOn5+4MoOvn8zmzW8m8VPxy7tPTgcu/1w4jrB1Jaty2cQvxg4mfREmFilbTADZqg8cHzdu3JATHBNaTOx2EowxtjWmd3IwB67RxG80OHB554KcA9eyyy4bPvWpTw3LMgwETEpjbKsqnRy4zGmDuJ555pnAdinUScoLOgtnS3QNkjpwxYMT/1XRC94YTTljYhWj5BprrBGjYUKfCV/uWUV8/W7LOOuN+jjLoJeodxMmTIjOqX6VojYcuJrcz5cZm/TP8bKJaz/h3Xa7lbtv1WOs9gRPxByrOl1rhhjC4PRCmbQthDutwGWr93CdOe163ZZz+jFjmhlwuNYk58DVi561eHOf3oGLLXTQ3xiGzNkyfW5WE6JeILQJOUdTfx/aFVaJMfHGV1ZbYtvdbbfdNp62bZQsbLfPkoGsW70lXstrM7JzzF9n7TnHu+Wlry9mtPQrcJVWlcIgT3uHpE6v8eDEf1X0noUtfZYcuIxBpxW4cL41Y+9NN90Ut3wu3YfjTXSw557Ts8SbOp74VSxs9RbCpcKqj+QFctxxx8V2h+/e+IvzAw7GCAZt+oxmHMUxhnaQvg/tBiyriDda055SLtDzbLFJnUDMaGtOWBzzxv/Sym2+327P3jQvfX7lxgCkyXQV7ZVt+cnxUrniXCdpcwxV6nf0oi992fAG4m5jgFzZ7ZcDF3yt/NCHOProo4fpr1xauCatRxzz0lacvIhjbYiPn/YO58UqYv2LXh24/L1YVZjVm1h1k76BjRMJs9dee3XcVgrnGZxoEL+KUTzg/ln59m1L036RizZ+pc1YYYUV4tbj5gDKCX+v1IHFt091+3Ne15T0gx+zmUOVObGRNpySyMNOghMTz2BteidnCD/usvt1iptzJV3VVG92up/X4aVtpnmxgnsj3qmxkwOX37K90wpcVocZa9nWztyHVc5plxCcg3EgxxkGOfvss+Nf/DHxn893m3dhe/f11lsvBsmtYMQJ75xifaFedHG8WeFfSU+3Wf6r9E2sDPk6SJKtLPvxkT2KT3vT/kfTOpbL2150RL8YGav00ztwUV/Qxank0lSaG7VrLR/r9ofturSdsnhzn4NsD/z9mU/FkRkdRb2kLJjYVpP2W58iIAIiIAL9JSAHrop85cBVEZSCiYAIiIAIiIAIDCNgk7McrDp5mJswsUhLkzzm9IDxh/vknA4YhPOH8KYnhqHSJEXO+ODTlZucHYQDF6s8mYMFKx94Yy/PxWS1TTKmE2ScTwUHpE022SQetgnENIwZpLq9yeqvy/Hzk0Rm8PUruWBUwbiSCkawueeeOx6e3A5cLDOPkwuCMX2OOeYYNqFs5R2nLQwYTNCy5QjlDeHtXSaMEf8mcTzg/rFdHMYTnGhwCkC8YdwcuEYTP29IaHMLxTrGNybUmVhHco4CflWF1IGLSVvy1ybxmGi2laHSyf94gw7/SoZUHCrRZQj1FwNYqsv8Nh7mwNVEL3hjtF9t5YADDggzzjhjTENpS5N4Mvnn63cuT5oYZ73uQaekxky/bV8bDlxN72cTzDnjA5gw0GJEQnyYttuteIOG/7beeuvoCMzlJUOaj9om4q1d8A42bLX0i1/8wgeP39GHtoqBd3Dwuq0NB65e9OwkiXYHzIHLGx/YbtO2kMOZFyMghnfEO6aU6hJMuB4DPjoH9ibe8QbDJXqGrSwRDG84zFSVUt+pW70lfstr36fw11l7TthueemNX9bnWGCBBYacgkrOWTgHsqomYmGa6L0YQYd/JeM1Dgvobco7jiqwSMU77J177rlxdao0jP/dRAd77jk9S/w5xxNz8KDfa44VPi1890ZOdBPOUojfUgqHLbib4zrnrW3nGGUfx3VekqDcVBHfpuX6Br4elBy4zHCe3s+v4GqrSzTNS1Z73HfffeMtcs4d3onf6zcuKJWrGFmHf9anJEivY6hSv6MXfemN6yPZgQvHexzwrdw2rUc+q/oRp4+/zner375t4vo6K3Cx7Rs6iZcFLr744mG3p4zQtlmf94QTTgj0uTqJpclW90nD0pdGR9AG+ralSb8IRwJWAUV44QHnTy/UXdJv25HzYhKOT6kDC9c07c91myMg7pwDl39xoLTNGeNJcwi6+uqr43Ze6CJbgZZno3+QCsfNea2q/ijpqqZ6M02T/+0duEqOfn5ew3Q4cXRy4PIvfv36178OMEvFO0vRrtE3NfG6khW+KP+Ep/3nZQfKjonPd2uH/HOZ47CFt0/frlpfqBddbPHmPkt6us3yX0Wn5vqSpLfUP+Wc59S0/9G0juXyljQ11RH9YkSacuIduEqOhNZ/43rr95XmRu0eTXWBXUc8OZ1F+4Jeo00wfTCo9oA6y/iKuo2DVirW3nOc9uWnP/1pGkS/RUAEREAE+kRADlwVwcqBqyIoBRMBERABERABERhGoE3jAxGXJnm8Aei8884LTFR4YVIMoxaTMThEsKUdn6VJipwDkp/ImVwOXH4FoJKR3ZwQOr0ZZ2w23njjgBMEwhL/6YpQ3ihvhnq7ttNnjp+fuDKDL2+3bbnlljEqv1KVxc3qYrYdD8faduAqOY3Z/dNPP+Fr57yDlp/gsfOwwFnLxPKH35RnjCVe/Coe3ljpDePmwDWa+PlJwpyR1jPw33MTzL4s1TFi+4ngm2++ObDVmwmGHfIKJyokdeDyb3Q/8MADcRsWnJ14Oxy54oorAlunVhFvHDBnBK7DUdFW7Xj88cejw5iPj7QxuWmrNZgDVxO94J0Hzj///HDGGWfEW6V1nolqnrebdMuTJg5c9nY8uid1zGWCl1X7WIEJacOBq+n9fDtneeJ5+Ulr78DVdrvl71n3O9tGbLbZZvEytr2ivUwd5ixOVuSwlaDM8IrzAteQL7StGLlwYvCC8zGrSiHeAcLrtjYcuIi/qZ7l2pLkHLgISznEmRfxesPXMVhgNKYse9lwww3DuuuuGw/ldIiVHa6DK32Zug6jRF7qO3Wrt1ybM7r566w9J2y3vMw5cPm+FQ4WODv6soNRBw6m90xnNtF7pLGTmPGaML4s+rY9l0+0HxiazPkVnYAO7yS+fFTVwZ57nbbP+mSk5/DDDw8Ymr3QJuHYRf31295ZGNON9tuvhMIKsDiAeuEFA5zYqohf/S+3SoR3VPd9Im8kp27QHpvTmd3X6g+/Kcc4lvWSl+aUwkQycXtBJ6IbkZIDF+d8ueJ3J/FtS1UHDF+fzKGAe5T6HZxrqi9LjgFW3nxZ6lZ2m/QRSHsV4YUIHBQRxjn023C4ROrUo3jBa//6EaePv853K5edHLi6jXf222+/6KBFXaKspW2Vd5hPV5vMpdUM75zzYzcL61f8s34E55r0i7gOHYX+4uUadEEqPl5zNsyNL3ydq9Of8/UuN0dAenIOXP5FqtSRyJ6BOQscZxHb3tw7pVibaOH5TMerVfXHINtAr8Np+ykzab9zn332iY6FPBPl/L777uNrRwcu3yblxlJc7/Pi1FNPDazi6IWxFg5ypIv2nbLl+3cW1ue76VtW7+FZuIZ6hO5hTsaLX43WHLg431QX+7jT7yU93Wb576bfSVOuL8nxUv+Uc0iv/Y+mdSyXt6SnqY7oJyPSlYp34OKFT3S81+v0xykDVk4Z4yCluVGLv2kfyqcnt4qw70PZSndeb1ed3yWdddsDxr/oDSTXD6Vvb+1KySkzXqx/IiACIiACrROQA1dFpHLgqghKwURABERABERABIYR8JMcTDD6iYNhAV/7gRMBHTQmcxCbDHvt9NDbssRDGLaXY7KZN0wxptgkBA4i9nYwE3Bf+MIXovGCeG677bbAW65IaZIiZ3zwEzm5ydlBrMCF0YHJPZ6TiXqeg4kEBOMmb+jaW7r29lo8WfjnDaC8dYazBkyZqGCiibe3uRcCcyZ2u+UhYXP8/MSVN/haWK5ja7Tjjz8+PhsrJ8DU3pjmvDcC2MRjaUUJVuuwLb28w5BPB+WLONOJVe5VEt7SNUMyYcgPjJlI6nCWe/P83e9+99Cb4hi3MBCYg8yiiy4attlmmyGnIP+WuzeMmwMX9xwt/LwDF0YKjCydhC2Ubr311qE671n6PKxjfPNvN1OOuZZJxIUXXjjgzGgrvZEuP1G/yiqrhI9+9KMxuegb3hyl3HlHFk6yvQNluJv4iUBWLKEsY8AiTdRBq3NW3kk3ZYsV4MxhjHvggIYjWhO9UHIeIF5f1qo6i3TLkybGWTMokiacJqjHGFNYsefjH/94XOWOcwirLrDtF9ItLYTJrVLT9H5+hQDSx9u7tHdLLLFEYDU9M7xxX+/A1Xa7RfyIfw7b5uy/Z8r/KXPoEhv38xysHEj5wgGV8zwHOtkmuYnNJtn57lfxsnbcVoWg/DLpb2Ub45Y5W/jylnNuMIMWdQXDq5fcFoqcb6pnfdzp95IDF6tjkC57tl/96ldxCziu32677WI54Ds6j/pt7Q1tFA5cXEfdh39qvKT8bLTRRlw+JDkjxtDJwhdrL7mP7ztVqSs5o5u/zrfn3fIy58BFknHUpowg5PMxxxwTnW1wUqZP47eHNmN1E70Xb9Dhn08/qzTiKE9fyxtkufyss84K55xzToyJZ8LIZPWcFTuqbOXdRAd77nXaPt/XpW7Td6T/g+DYg3MhTpgI9R7HdC/egYDj3nGd9gk9Y+Wf8ziD0UZWEa83WFUOdpQB0kW7bOWCuLz+9MZ/znENTtSsAEh+UabYWhzBUd7GFb3kpe9vsRoMrMhH9KJtucX9UgeuUrkibCdpcwxV6nfQp2mqL0uOAcZpEA5cfru+XBthfM0hgzJGve7VgYt4+xGnpbfOZ8mBy+uLbuMd30dnnHzEEUfE8TXpwFmNcbS94MBzpytAp+n1/SLKGA5k9CcQHCqo26YzvANX036Rryunn356YGVZE3QF/Q/Gk4x10XeItYt+fOHTXac/122OgPt5pyHvUGXb0BKGsQ+rRtFWo1tJ91JLLcWpqOOsD0RfjXpmDNHJzKHAev75549tkh+r+vvFyAr/SrqqF71ZuNWwbXAJQ7+R1Vtx0qI92n777eOzcM7nEb87rcDFed8PZp6EFXPIT8oAK4/bauZeR3GdyXvf+97AqnRebMzlj/l893NWvk9D+cahBGcy7k+eUiZNvANXU11sceU+S3q6zfLvdU2pb5LrS5JeS0faP7VnaaP/0aSOlfK2qY7oJyNj5T+9wxTHeXGGvGFMx9iJ8Rb1GvEvcfr+Yq4/2FQX0FfGCcrm9Rg/4uhvfRCc9W08xDwL+mBQ7YF/IZK2jTaV/ibC81KHeMENKa1mFk/qnwiIgAiIQOsEaA8Q5hboG9NW0Jbwx3f7XfXG00w0NA1/pbLqlSM8nE3kjvBkKnkiIAIiIAIiIAIjjICfUK2SNCbY7r777iFDi58M43o/schvP6Hn3+blHBNBGN6YgDHBcIqjl63uUJqkyBkf/ETO5HLg4jnSCRkmo3lWjJl0XhF+M1FmjkXxYOYfHWAmU8xwRxCY+UlfJjftvBn10+010qhz/PzElTf4YmDDMGBpT+PiWeycObQQxib86jpwMRHDRJ6Xk08+OVxyySX+UPG7X4kJHkyKmzCIsLf+OJYrJxz3K07wm3gQ8sMkNQL7su8duEYLP28csmfs9GmTiZbPvq77slSaKM456HA/jB+2alN6f1/WzIGLcRDGKsubdDLTO2iQRgzXlp9p/P63PZcdszc6vaMT54jLBqf85h5jxozha6znOLlhaKurFzo5DzDJSh2xZ+bNdN5Q7yTd8sQ/V85JxxwjvQE2dV4hfxDTB+hxdJX9ZtKVvOqWFuLIlY+m9yM+nmn+iUazbuIdEAjbZrtl9/aGCvSROYja+dInWz2im4ynhfP1wo7xedNNN0UHWDtGO0Gb750MKb/ERxk2SbcT9LotVzaaOHBxryZ61tKY+yw5cBGW7ZswvCGUS/QAbSd1CSa+H5KWW65JDc4cQ7iOttzyhLzIrebw39Dl/54xoUyfVqkrOaObv8635/4+ubyEB3UP8UZLygycrK8RA7h/lCPTR+bAxem6es9Fmf3qn8sC0Gaglz7wgQ9ERx07Tl5geLJ0cZy85fm6rb5F2CY62KevbtuXGkFJO8/g019arcQ7xJN2nhFnfxP0rm3nVeqTWdj0E8dp0mZlnPM+v0mj78vDmPaT6+hXpEL4NC70IKtvmTTNS78amMXlP+3epNFW0+S8zzcLb+XKfuc+2x5Dlfod3LuJvsSoyTgK8c/TbQyQK7tN+gjc129R7B2BOOfFt+8YYHGeRXJp4Xiuj8BxL/2I08df9XvJgavOeAfnLPLN2mrKMuNL9IMfE1pftUracJQaO3bsUFDiRKx+Wn1J861Jv8i/kMQ90CHUQ9oUux/H/ephVh+sPeQ80qQ/122OgHhLDlysDsxYwtIJF9Lu+1IcQ4/xAoyJX0WbY4TxbZLx5VxVB65Ouqqp3uT+OUmdcC2MTzfH+M0Ywa9Y3c2BC2cL+iPGlHgoE348Rbw4y+E0lwrs7YU5znnHPx/W57ufs6KvQ33yeejv75/R94WIu4ku9mlKv5f0dJvl35ebkk7N9SVJq+838jutj230P5rUsVLeksYmOqKfjEhTKmn/2M77sscxDON+RdHS3Khdz2dTXcCKw9b2Eg9pQXw99S8GcW5Q7YHvx5IuxnC0f5QDE9oqWFGXJSIgAiIgAoMhIAeuipzlwFURlIKJgAiIgAiIgAgMI2CT+MMOdviBAxdviDFRh/jJMH4zIccKRWbkSyfUWCkHA4tNQHONCRN/ODr4t4Z5s9+26PNOPPbmpp9EYgLb0nXjjTfGt4ktbj79Clx+BQ4L441nOcOmhbNPfz/PgcmEz33uc0Nvgll4+2TCgQlJjOtVhC37mDwzpxC7hskJ3uZlNQMMxsb0oosuCqxM0Uly/FZdddWYN1znDb78ZqUFlk5nGyqbxOH+OB7wHEwUIX47CxzPWFGgtEKQn/DzK3ARj3e64TfPyRZGVcRvNeZXdbBreWuQNwaR3Nu6Fo7VlFZaaSX7OfTJpBHPfOyxx8aJeDtBnjMBiXgHLn6PBn68cYyBpaqYA5fls6+LvizBCYN+KiXjG5PpGJVYtcELxhLKCVu8MfFuDlzmtEFYv32TXcvEHg4zNklv6bbzpU8mMXm72+oVK3exghe6jZXuWPHAC+WCuogTI6sXYZhDbCurunqBbcmop4jfvisemPhv/MQVEnhLHeHe3ZxGuuVJVeNsOpFcmoBmy1UMOFtssUVYcsklYzrRGRipuqWFwKXy0eR+8eYT//HGPgZdKwtwo9yyIhcOPpSrXBlqq92ydFA+bKWM1NHCwpQ+Z5pppoCxx1auyYWjjWGFy5xTHzoZh9y55pprkkvhQduROgB73ZZrG82BKy0b3MCvwJVui8r5unqWa0piBhucSWjPvdBuoausHfV9BJjQxtoKTf46+jCsqMVfSSjTtNMIK1NgrK8rpb5TlbqSM7r563x73i0vSw5cPA99HtoJVq4zvUiZoc+G3jNnab9dal29140b6aN9sDac8BhrbOKwU12lbpM3OHtVkSY6mJUf6OMidds+rkFf+pWiOGbiy6wds0/vCJIr/z5evxqiXd/tk/LEikiW7xYe/Um/HX1iz805+uLoKtvKlZW3aK/IPy/kBave5RzqmualXzHM7kU5vfzyy6M+oq6lDlzdypXFk362PYYq9TvsvnX1pWfBlmC2umC3MUCu7DbtI/g2IHUEsufiE11BuUFX44REPiG5tHC81EfgnEk/4rS463yaA1c6JiaOOuMdHGrIU++w5dNx++23xxWS0AFVBNaMsf1qQ3Ydq6AxdkHf5/KtU/3MjeeJl1WX6YPZONLuxSeOTbZKlR3PjS/sXN3+nB+zl3RpyYGLe9LGs+IU/YVUGOfiFAP/VNjakjFy+syMDWBserOqA1c3XdUpX+q2gd6BC8d+xog2v2PPSZnm2W+44QY7FD+tzHdyKCzNbxAB8bKd8L333hvjy/3z8za5Ld+4xue7n6vhHP1wxnO+P8Fx2otjjjkmjgtoW1MHLsLU1cVcU5KSnm6z/Ps+YUmn5vqSpLnUP7Xnaav/UbeOdcpb0lZXR/STkbHyn348y6qxpDftY9E3Yu6VMbVJaW7UzttnU13AKs5bbrnlsJcHiJMxPGMhHLhS6XSvttoD2iP6IfTNc4J+Y/VazyoXTsdEQAREQATaJWDzMFqBqwtXOXB1AaTTIiACIiACIiACAyVgBj7eRLUlri0BGM8XW2yx6OyCcwWTE1dffXVcMtzCTAmfOKC8613vioYtJmTYngkeOLMwOVhHuH755ZePW5MxkY4Bzq/eRV9woYUWim+hYSQ3I02de1QNy9vaGN0sXzfbbLOA0xSCUSjdYqpqvGk4JmpYwYEBAUZxJvcHLXDF+QRnCQwibO2CEZCJ5aYyNfFryojrmFiHPaug4KzFBDqTh4MUDCVsm0l+Y0Tw9YoJbfQc+gyHPowUVq/Razjz4SzCioXeON2mXhgki073YoUNDD08GwYSdJCvI7AgP+FQZQvLTvfiXK/3I19xRkUfm17BiQA9W3LAGWntFvqRMojTEdzRybQx8CUPugkMKb/zzTdfzBeuoawOuo6Rzn7o2W7PnzsPE7ZdhSkTUNR52tpu4p1fMR6wvVVT6dR3ahpnP67D4EmdsFWe5p+4up059/ntC+3ebes9DH2sKIXBxveFuB8GZvQ2f+hg6gR5wrY0o0FwfKIc0KfD0YiXJmj/6PtNTsFISz+U+sFqWfRlbQtW0oUTF302HEhZ3dR0q08zegtDH2Ew+HcztDXNS2uDSSv9NvhVcdzrVK78c/Tze6d+B/cdKfqynwymtrjrjHdwSqO/RVtFv4q2Cj1MnaOsNxHuj9Ge/ra9mFNF3zTpF5keYVxF/WbMiD5BH1Spo+nzNenPpXFU/Y0TFluEw542kDaFVXZZkbmT4PSF0wX6hWt4Vj8u6HRt6VwnXdVUb6b38g5cti0ZL6/gZIyOpW3FGS6n69O4Sr/pd9N/YJyAsyf9cvQ1n4OScePGRec08pf2Fke8Kn3hkaCLB1n+B9E/bVrHOpWVkcrIO3Dts88+cV6NcR0rbKF/qQeUx16kF13A+BBdR12gXSAtnZyDB9Ue0HZYe0V6aP8YvzIPIxEBERABERg8ATlwVWROx1EiAiIgAiIgAiIgAiIgAm0SYFKVt/AQVtXg7ehU2IoK4yzOK7zJak4sabip8bf4TY25rmceaQRwsMGxAGcIVvhIDTNMmLPyAnLFFVcEVouRiEAVAhibDjrooLjKAwaX3Xffvcploy4MK/zhOMCb9EcfffQk6d98883DyiuvHI8fdthhMqRMQkgHREAEREAEeiWg/lyvBKtfn3Pgqn61QvaDgMp/d6qjhVHOgav70ymECIiACIiACIwsAnLgqpgfcuCqCErBREAEREAEREAEREAEKhPgrUW2KUF4y40VtqyDzjG/LQ9v5x188MEclrxGQPxUFERg8hPwWxT57d1IGStP4JxiWxIdeOCBjVexmPxPqhQMigC6nRX5Nt5448AWSQjbiuS2FhlUmvp5H7ahZeUUhO1c/FZRvAnPFkBIujVdPKh/IiACIiACItACAfXnWoBYMQo5cFUENcBgKv/dYY8WRnLg6p6XCiECIiACIjDyCZh9SFsodskrOXB1AaTTIiACIiACIiACIiACjQjsuuuucfUaLmZ1LbZ9YOsEtu5gRQ6E7XAOOOCAoW0V40H9iwTETwVBBCYvgYUXXjiusMVqSQgrJT333HNxezW2tjG56KKLwh/+8Af7qU8RKBL4wQ9+EM9ZmcJxabfddutpK6HizUbAiY022iisvfbaQylhC2VWsqMfwLYpCP2DI444Im4pNRRQX0RABERABESgJQLqz7UEskI0cuCqAGnAQVT+uwMfLYzkwNU9LxVCBERABERg5BOQA1fFPJIDV0VQCiYCIiACIiACIiACIlCLAM5aX/nKV8Kcc86ZvQ7nLVateeqpp7Lnp/aD4je1lwA9/0ggsMoqq4TNNttsyOk0TdMll1wSTj755PSwfotAlgAOXOa89eqrr4Zjjjkm3HTTTdmwU8pBtlNeYYUVhp7bPxfOW0cddVS4+eab/WF9FwEREAEREIFWCag/1yrOYmRy4CqimawnVP674x8NjOTA1T0fFUIEREAERGDkE5ADV8U8kgNXRVAKJgIiIAIiIAIiIAIi0IjAvPPOG1ZeeeUw++yzx+vvvPPOcMstt4SHH364UXxT20XiN7XluJ53pBFgxcD3vOc9gbo4yyyzhH/84x/htttuiysGvfjiiyMtuUrPCCawzjrrxDJE+4fT0jPPPDOCU9te0ph3Gj9+fHToHjNmTLjvvvti/bnrrrviClzt3UkxiYAIiIAIiECegPpzeS5tHp1hhhnC8ssvH6O844474grcbcavuJoTUPnvzm6kM2I+jdXCkKuvvjq8/PLL3R9KIURABERABERghBGQA1fFDJEDV0VQCiYCIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIlCZgBy4KqKSA1dFUAomAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiJQmYAcuCqikgNXRVAKJgIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiUJmAHLgqopIDV0VQCiYCIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIlCZgBy4KqKSA1dFUAomAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiJQmYAcuCqikgNXRVAKJgIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiIAIiUJmAHLgqopIDV0VQCiYCIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIiACIlCZgBy4KqKSA1dFUAomAiIgAiIgAiIgAjUIvOENbwgf/OAHwzTTTBMuuuii8NBDD9W4WkFFYFICq666aph99tnDk08+GS6++OJJAwzwyEhKS6fHnnvuucOaa64Z3va2t8W6+L3vfa9T8L6eW3jhhcNSSy0V73H55ZeHRx99tNL9RgvrSg+jQKOGwOqrrx7e+ta3NkrvBRdcEMaMGRPWXnvteD2/H3nkkUZxTW0Xrb/++mGGGWYI99xzT7jpppu6Pv7iiy8e+ENOO+208Oqrr3a9ZqQHQF+r7IzcXGqzzNFH3nDDDcN0000X7r777nDzzTePuAdnznSttdaKfYirrrpK/fnJmEOjobxMRjwdbz3LLLOE8ePHh//85z/hjDPOCK+88krH8FVO9iPOKvetGmbcuHFhueWWi8Gvvfba8MADD1S9tHG4FVdcMdDff+KJJ8LZZ58d46EvRZ8KueKKKyZLf6hu3yImdoT+6xfPddddNyy00EJh5plnDldeeWWcvxmhCAaWrH6xHtgDdLjRGmusEeaZZ57w+OOPh3POOadDSJ0SAREQAREQARFom4AcuCoSlQNXRVAKJgIiIAIiIAIiIAI1CCyyyCJhxx13jFeceOKJ4bLLLqtx9cgKijEZRxjkwQcfDC+88MLISuBUkpoDDzwwOkU899xz4etf//rQU88555xhxhlnDP/+978HZhQopWUoUSPgC0bm7bbbLhpdLTk777yzfR3452c+85mw9NJLx/vW0QmjgfXAYfbphtJ1/wP7rW99K7zxjW/834Ea34477rh47aabbhqv+s1vfhNwfJB0J/DDH/4wBsKB69BDD+16wQ477BAWXXTRGG7XXXcNL730UtdrmgQYZDuDoVtlp0kuDeaaNsvc9NNPHw466KCY8DvuuCMcfvjhg3mIGnd5xzveET75yU/GK3CSxCF1tAqOCRjkceKZMGHCqHuM0VBeBgm1Tp9lpplmCvvvv39M3umnnx7+9Kc/9ZzUfsTZc6JcBFtssUXAoQo577zzwh//+Mehs/1q0/bcc88w22yzxTHZ7rvvHu/n27STTjopXHrppUPpGNSXun2LQaWryX36wXOXXXaJzjyWHpyJjzzySPs5xX+W6kM/WI8UmPvuu2/ACfXZZ58Ne+yxx0hJVjYd888/f5h22mmjY+hTTz2VDaOD9QiM9v5QvadVaBEQAREYeQTkwFUxT+TAVRGUgomACIiACIiACIhADQJTkgPX+973vsAfcvTRR4cbb7yxBgkFbYtAyZEH4ydGraeffjrss88+bd2uYzyltHS8aMAnvcPUyy+/HI0pe++994BT8b/b+fTIget/XEbSN+m6/+UGhl6Ms01EDlxNqP33mrpG1jadaTqlepDtjDcYyvmvU65MnnNtlrnR4JAzJTlwffnLXw7zzTdfdOD60pe+NHkKUA93HQ3lpYfHq31p3T4LL3+wwiEOADgvtCH9iLONdBFHJweufrVpcuBqK/fK8fg+QhsOcawA6VdoxqGHlZJZqW5qkVJ9aJv1SOI5Why4cLrea6+9Irrrr78+HHvssSMJ46hNy2jvD41a8Eq4CIiACLxGQA5cFYuCHLgqglIwERABERABERABEahBYEp14DrqqKMqbetUA5WCViTA1g4YXljq/6yzzhq6qjTpOhSgD19KaenDrRpHiTPbW97ylmio/OpXv9q3lWmqJrCpA9doYF2VwUgP542hU7uuYxtgVvZLZaeddorby3Cc1QlYlTGVZ555JqyyyipaRSkFU+F3XQeuZZdddmhlP5yd+rWF4iDbGW8wlANXhUIz4CBtljm2xNt8883D6173unD77beHa665ZsBP0/12cuDqzmhQIUZDeRkUC+5Tt8+y0korhY997GMxid/+9rdbWbW3H3G2xXCkOHAxdqMvj1x00UXh73//e1uPWDmeun2LyhFPhoBt83z7298ett122/gkvKTGy2pTm5T6eG2zHklc5cA1knJj8GmRA9fgmeuOIiACIuAJyIHL0+jwXQ5cHeDolAiIgAiIgAiIgAg0JCAHrobgdFltAqVJ19oRTWEX2ApCbCnG1mKTW5o6cE3udE9N969rDJ2a2Nizss3I7LPPHn8efPDB4YEHHrBTwz7lhDMMR+UfI9XIOsh2RmWncnFRwAEQkAPXACDrFo0INOmz0G7jMHnTTTcFHNXbkH7E2Ua6RooDVxvP0mscI7Vv0etztXH9csstF7beeusY1dlnnx34m9pkkH28kcJWDlwjJScmTzrkwDV5uOuuIiACImAE5MBlJLp8yoGrCyCdFgEREAEREAERGFUE5p133rDhhhuGOeaYI4wZMya8+OKLgY7htddeGy6++OJJVqfYeOONA0uT33vvveGCCy4IGA6XXnrpMOecc4YXXngh3H///fFt2ZKRep555gm8fbzggguGaaedNtxzzz3hqquuCq9//evDjjvuGNnV2S6NC0jDyiuvHGaZZZYYJ1tdsMrJKaecEljWH1liiSXiCid8v/nmm8PVV1/N10mElQ1YReVf//pXOOGEE4bOV7nHm970pvCRj3wkzDXXXEMG+4cffjg89thj4cwzzxz25jbPvs466wQc1zDu4zRz5513xtUUYJKKcZ8wYULki4GMZx47dmx8M/kvf/lLfC6uI87VVlstjBs3LuYfby5fcskl4a677kqjLf7u5/14Xsrc3HPPHbcc49lh9Oc//zmWuzRRlpa6ZY63tymrDz30UHz+8ePHx3JHeWVVAlZeueWWW2K+kD9e1lprrViuqRdsFcEKOZRtyubzzz/vg8ayv8EGG8Rjp512WgxPOeI5r7zyysCxNC0Epu5x/JVXXgm/+93vYvldc801wwILLBDLIExYYePcc88ddj//Y4011ghLLrlkmHXWWQNljfvdeuutgcl1ygZpxuj0/+ydCdgtRXH+x7+JS/QB44qoeBEia1BBFNAoiIIgF1AERHABEcGIe6LgHhU31CCIgODCIotKUFBEFkFAEJSgKIsREVcEBfExmsQk/u+vue+X9/btPmemzzl3rXqe75s5Mz093dXd1TVVb1ePok022ST1GY5st/PnP/855fOHP/whld2fnTZfPO/8vBXAVeI14+3FL35xegXbfNx4443dtttu26299trdAx/4wO6OO+5IMo1IbZIZJEYu7bnnnqm/wMdStJNtttmmQ67deuutafsQzrkG0cdOPvnkJBvThYX/4DX9kK0qaSPGZx/qI4cmlXXwhPHJ2LnnPe+Z5oNbbrmlO++88xKPKOfyKOsYx89+9rNTX+ebnrb59a9/neQi277Q72dBrQAu5jCicq2zzjpJHvzqV79KY5t2qNGqq67abb311t28efOSPEFuIUMuv/zyVNfac6OuE2nhCU94QpIn5Md2JMyfzHP0YebKCxdEyMiJ55BPpEO3QJ7Qj84888xiFDI9v/HGG3dPfepTUzRA5NEf//jHtH3V+eef333/+99XsrnjUCcrshGeoq+gIzhNo4/Map7xcubnyIVddtklXSYC19C+AzgBuUmkKLYBZX6kvZhTvvSlLy2iuwCAYG6BvvCFL6QIl+mH/SMaHUAACJ3la1/7mt3tusc//vHpXegAEPoi8vWqq65qGoeM5/nz56e58z73uU8a28hxdKpcpi9SkMIP9FPmVOrPvMxYgr/oVeiW6KvMs+gtyEj4xpxNH6d/Mz6YY5xKfa51TuI59Ezo6quvTuObc+kTyDG2yKIvU+6NNtoo8YMoKcwzjGHu0Y6PfvSjk+5w++23J17R1qWIdEP6B2VpBXD1HX/oIbQLdM455xTlCX1i1113TWmY7/meEfWZR9ddd91uiy22SDyiP0P0UdqdMYbeJhoid73d0dt/vOBbCl2eMUGZkdcXX3zxXMQhygA/kaP0SdLTp+mLfWhJ9hdkPv0NmY1sYFxoDqOs6OLnnntu+u4rlX1oP/M86M8bbrhh6s/kw7x03XXXdWeccUbSs4boLJ4v53vvvXeSV7T56173uvx21/KNMos8FytYw4USgGvInFZ7Jf2Q71LGFd9G6NzIJGwNr3/965MeTv8++OCDUxbMMegPEOAg7+995QTPSkZzjp7C+H3Sk56U+gpzBXoV+kwus0k/SrcYMufIroBsZSu3kq4pecN7+Rb+wQ9+wGmS1dPQW2v8HNp30cnQNfhu4lsBgofM83wfuZ1h6Lc+8yR/2KD41uac7zR4fcIJJ6RvM+bbpT3HjRsPNV4nZi34Bw933nnnNIfx7Q4RKRx5xRyck3QS5mnkGd94zBnM+cg6bDzwHt1jKA3px+T99gXbyGJrQ79661vfmmQfuhH1oN3QPSkj47tEQ+vOWEVfp81LUd4Y3/RhCDnBHMN4I5o4z0GUFR0AfQkdsw8x5z/nOc9JbcT8juynfuhQl1566VwWLTYCPQwAkvZDBjLvY2tAr+G79CMf+Ujq/6PGA8+IeA5dge2esWEyJrFvoavmNivXQfraQySf0BnH6UMqUxyDA8GB4EBwYPocYC6C+M5hDsFmg1znj3P97vvmuy1QKGZjAe1bghmlQ8EJCg4EB4IDwYHgQHAgOLAicACjB4CRGuHgfNe73pWMFkrznve8JzmqACtgFMBYkBNGyiOOOGIxIz3GWwx/KJY5AVoC1AX1BXChtOIclwEszxNwxLHHHpsMI+T9yle+MiVB4X3zm9+cJ08gDAzJ0J133tmxpdyQd2C0Y+u5EgEQkOMdoybbauFQKBFGOIy8TuI7SjvtgvE0J5yVgHdkzPL7GL+OOeaYZCD067XzWb0PAxNOtVIfoCw333xz97GPfWwRg5PKMrTP6TmcSQcddFD3hje8ITnB8joDWlS7Yfg68MADi+l4DiPeqaeempy4ysedlRgQMTirfrQHfM/LwrPudOc5QD98fOXE2MCY50Q5X/Oa18wBBf0eThDqhJFV/djv5+f0RRxtJXr1q1+dLs+KL6V36lorgKvEa4zGrJaGGF8CxehdOmLoffe7350cpVzDCIwcgACPfPzjH0/n/g8jNjJI/QjnEnkAfoJwxBx55JFzj2AMJk+1NQAvjKyjaIgcapV1vB+DMgbjEiFDcJbTV5c3WQfAGMcr/aBEjJMPf/jDizgJS+larrUAuDDMYzAvEU6eo48+erFbyKEXvOAFac7Kb9J2n/vc5xZxPORpSr9r/YH8ABIgNzhHFjnhiNh000390tw56XEaOEBaN4n8J1CPrvkRR9Whhx7ql0Y6WRdJuPDHy1/+8gTK4Cfvw6ELTauPzGqeSYWs/PO5ZGjfwRlD9EUcUTUCrA8QGdp///3n+iZgJsAsOXmUG0AvilrDO+D/vHnz8kfSbxxXtG/ucComXngR5yF5as7N09LfGC/u7MrT+G/vH9QP8GJOzLHU6YADDiiONwBrAFhEnqf6XOuc5M/53OJ9AIcgQMicJ/Di8MMP716ln1hxAABAAElEQVS8AMwMUC8nnK/wn3Siof2D51wnot/Qf8bRkPGHDoljF8JJiZ6f04477tgB9IKkUw+ZR/35PO+3vOUtc99FQ+Wutx+gBwDfpXkJJzh6Cs76nPgGQMcAjDeO/H2z7i/MP4DTodrYoW8xPiRPVP6WfsazfEcx/+DILxHyne9YwDrS8/N0/n2W3+M3W6LxDQEB6gQQ7SSdc8g3yizy9DK1npcAXH3mtFHvo23hfelbnb5MG9GODuByeQYgFVAjNEROkB6QOd8SEIu3APnlcpF7fGehp2M3ENUAXEPnHNcBTzrppOICMucxQBXAbUPrqnKXjjV+Du27tBPjqUTwl23CoZZvfeYl5CkygoVLfEuL+E5CfxBQfWnOcd5WKh9HfQPWeE0aQFcsVmRMlAj71FFHHTUH4CWN9AfsD4D7WOBQIsBfAGT70tB+TL4CcFFXxm7JP4oNEP0+l5MtdUe2sxgRkj0i/Vj4j/nxZS97WfrFtyl/im7o6Thnvj3ssMPyy4v95hsaPbemE7O4jvkXarER6IWSL4x1vvdkM8AOgcwYNx5kNwDgWesTtBP2I9+C1nWCvvaQvvqQ6hbH4EBwIDgQHJgNBwLA1ZOvJQWl56ORLDgQHAgOBAeCA8GB4MAywwEMFABVMGRiLAM4w4o5HPOAg3B0QLnDVMY+rwhON4wEGKT1nAxZSscq7N13310/56JqYPzIDVl9AVz77rtvMsaSKYYkItlgAMZIJEMxBi8M75Tnve9979y73vnOdy4WOcKdDxjBMIYNeQcOAow+GDg5hwAG4IzEQYBxk+u8W+AN7hHlAh0TB4QMy7mhKec7bYazALBIyXFBfXk3fFCbwCMMj31oFu9jZf4+++wz93r6DaslAbTRd8STcXUngz59TnUQgAvDL6sHiaoFnzEyYoij32MwhWSc5BynApFB4BtONn0HwPv3v//9qd1I585KfotIx4pPwHh5WUjjRl49wztZOcm7APqIcgeTtjvUfRx51Idn1Ie41wfAtcMOOyQnB+1AG6hvkd/73ve+9IpZ8UXlLx1nBeDSu6gnEVMYQ8g88c37X6txFsO4IgryPkB8gPkgN/6zWh2w6zgaIodaZR1ABUXNQW4il5AxjBf6v+ijH/1oGn/Lk6zD4YRchhgrzBXIReZBGc3dKK+6TuPozru+Wyjqvcge+ihldzlP5CiPKEPkJCJ6iIhWwWp9+rUDNeTgULpRRzkPlAY5imOBOUWymnuMIwdw+TzKPWQ80QUAZvGsxhmRuIiqJUIOEZUSov8xNujLPMc41HM49jxiopwgJaCr8vajnGFcE5iG82n1kVnNM5SxRqW5pG/fcdAc7QsfGRPoghozvBf9ib7oAFE5mvJyAXBRf9VzpBHYlXPamL7BOFSkS66jE5GOth9HlBN9So425BUgJPoZzm+NbeZV6tmHvH8oPWOJPKQ76DpH5kn6N3OvysF1FgngjIU8T/U5d6KlRAv+9ZmT/LkaIEf5oQ8jC5y/use7qBc88nYGeOYR04b2D/J3nagvgGvI+IPX73jHO1JV6CclYI7rLOqDQ+ZR9FVF80RHgNDNIMC+9NMWuevtlzJb8A/QCH0X0D1g+Zww1tP/XA721R38fbPuLy77VQf6IHOrj0fuoRfjPBa19DOedV1YegtzFSBR6g4xPtGzxuksKXHln/oTfQB93kn6vV/r840yizy9DC3nJQBXnzlt1LuQ0f79hNxB/vg3L8/3AXANkRPk6QAufkOMN+YexpvrRwApPMpPSbdomXMAVwCygH68APiufNOFBf+QLywwQcehbFpINrSuyq90dB3BAXFD+y72GvQ9Ik6Kd8gn2g7QJjpd67e+65z0D+l8nJ944olJNgrApToujTlu3Hio8Tq3/6AfYGOAp+i50h/Q3bBboVtArj+o3jzL3I6O7za0km1Lz/ixpR/zvGSW8qJtkO+0A1EipXNxnbIw1qHWurcAuFisSd+ENxD8hFfY4LCljCJ0Icqtvod+wfciYxQgma4DngJU2GojoAy5HFC54Bnz2rjxAIjRF8vAc/QI5j9saoxRiOsA99HtIdcJ0oUF/0gzyh5ChNBx+pDyimNwIDgQHAgOzI4DAeDqyVt9ePRMHsmCA8GB4EBwIDgQHAgOLJMcYCUbBm6IVacY9kUYgxSRIXeOuLEPQyPGTm1rhOGDyDIymstpQr6HHHLInGOCbQxYXS5iOxhWa4r6ArgwePIujPZvfOMbk4NDebjhg8gjrFp0wzQh0HmPk+qGIQPjGQ6aoe8gP48+QZQGIlCI3BCXG4txcGC4lbOI8shhpLKRD8YoAEQ4JaDcaeKrYLnvBjfyp93G0Szex4pFAdtyJyGGNngusBmreKkH5GUZ0uf0nABcqrPaNAc3sU3BHnvskZLxcUR56QMi57MMbNxzZyW/AYgA2PEVj6WyuJGX52644YYUSQJHHYSRmDQQTmnKDRF5QlsDUT7aV+UkIh7ATI3BvI4pg8o/OcLciULSWfKlUpR0eZYALvjCqlSNIV/F6w6USYyz3l8YswB5aDs5cpCtOPk5jiP12VnKute+9rVzERUBaRHRT+R93J3Ay4OsQ7Yo4mIO0gKIxJwlsCTgOtpqmtQK4GL1OtE+RM7rHFjt82s+fzJ+2VIEx0M+nyvv/AgABhAOzzAf4mgA5A0hw4lmRhqI+wJwwUfkEc/RV1ntjvNDRFQuHA4QzineAQgIYk7DgcVzAHh0nXvINfonRH4f+tCH0jn/5ASZBMA1iz6iMZvL4FZ5Olfhwkk+l/TtOzi5kPsQgAPmPNpT5KvuHfyn+Yx0OJl5VkT/UHQC5kLaEiIC63Of+9x0zvwJmFFtzHzF/K8+5e9KD1T+4VBiu1corzPXAPYARIOop5yJ6ULln+to8IKoiwLf5mAAwN+AnDX/uswnMhkObcjzrAG4+s5J8Eq6gMvivA+4no1uTv0FvGSMMYbQKyCvl0eabO0fPl/0AXC1jD8HAyJnAF6LsNnihIVwdEv+a0z2nUd5XvOiyzmuQy1y19uPPGgDZJj0PuQnwBKRR3yiXgDXaEevl9KWjv6+WfcX7/+UJY8I45HTXC9v7WeuCzO30Sc0FplLkE0CRShqms+j+fdZiX+6RjQ35CHEe5BtIpeHQ75RZpGnytR69O9kLWRSXho/+Zym+6WjtxHfFsh96d1886I7CPTh3x4uzwQ4apETLtsony/Q4Dfbfu63336cprmPPqPylXSLljnHAVroPehPPs+yVSTbtUGKKNhS15RB5V+JnyRt7btE6iVCK5TrnK3f+m63IV++QQB6IusgrwO/l+Ycx/tr48HLqb5LevoZ/Q0iIqgWjvEb2wffCtKB2NaTaG2Q6w/0G8alL2JAdwKwAxHpfRxIiXQt/ZjnfH5izuI7GhAaxLzEfKkFPw5WbK17C4CLsgAOReZDisCZfoz5B8hX0Y/d7sJjLA7VFrp8J/K9OImNQPKFvNFJaG/KKtkwbjxQR3QbvreYd9CD1Bbk+cIXvjBFYuXcv5tcJ+BeX92TtKP0Ie4HBQeCA8GB4MBsORAArp78DQBXT0ZFsuBAcCA4EBwIDgQHlmkO8BGOY4sPdzk6vMCKEpM7LdzYx4rIb33rW/5YCmWuLT+0DQBbamDYh4jqoqg+/iBOCTnRc2Ogp/NzVsHLKY3hSw4Q0uAAUVh1tm/BGOYGHXcskh5eaBU/q9AAn0FD38EzNQeBRw1A+ZZxiWdEbuwmetOnPvWpdMv5njsd3ECNE4O2kwGIh91pgsGwz1ZC036f94EcfJAquODfgi3YE9CA3+6M97L07XPkoefcUcT1mtGVcSBd38GHPCNywzR9Dge0OysxomFM9b7Is6WyuJG31B9wPAFqgNxhonLSxjhh3WBHWo92589xbxTVAFx6H89Omy+jyjMrABd8wwGnD2CVwdtWWzVMYpxFNuEMEmgRZ8Raa60150TvOxYp31A51CLrXAaX2pmIYgABiPyBUwJaHmSdy0c3YqcKLPjHVmM4MyC24pLzLl2Ywr8WAFc+P1EMnJyaO31ce3SHHLyr4rtTgjmFuWUU+dgrgWkcqOU6gj+XR8rS+5iXpSM4uIO64cTCgal5T89w1LYogIUYVyI5QXzO0L3S0Z1hAtPMoo9Me54p1UXXfC4Z0nfWXnvtFJWGfAC651ve4IhTNJBvfOMbc9teOujer5PP/Pnzu6233prTtA2SHI3IFBYH0F9wAtKHnYg8BCAGKtXB0+p85513TsAw8kR+ydHr97fccsv0kwg8yK5x5P3jqquu6o4//vhFHpEs5iLzukD2/PboZIrQwHXPU33OnWhD5iR/rgbIcfnA+yF3vJbGNLoEDljkn74JWvuH60Q+xu8qyeL/W8afO6B9m05yd+AhkQqJWAip7dCT+3wz8EzNYdkqd739aHccsAIy8j4HQZXmK80nPCvgLM/VyN836/7iZffvKC+b90NFpGztZ8wD6CQQQAJFGNH7fGspLRqp6Sx6pnZkbkKmM0ZysKj0e54d8o0yizxr5e97fdoALv9+YPwJCK7yeBS7cQCuFjnhz/B9xvjJQfreb7/+9a93p59+eipeSbdonXN80RxzCnOLyIHGGhNe7pIcGKq3uo7goKLWvlsDcE3yre+AlZKtyOuwtOc42q6m43k5xWvkMN/zfBdiH0APyAn9CD0JEkCIc9cfSjq+f/P3BSu19mOX3w5Sp5wQdaBPUU/Aisyfk9R9SQK43OYCoOrggw9eTFawWBTgKcTcTd1YBAQ5+D1dWPhPYPN8AY3kC8lK89e48cA3OdG+odK8w3WBzNEXKDv9ynWCIbon+dX0Ie4FBQeCA8GB4MDsOSD7NTYPviOYb/k24Y9z/e5bkrstcMD839K9vk8tB+nk1FkOihpFDA4EB4IDwYHgQHAgODCIAxjCWWGGYZBw2SiAuZNAxj6MG1qJ5i9x47iAWL4KzCMr+XPujNFzfr907oALjBKAyS6++OIUBryUnmsOUuCcCAqQG3AVsYvrLe9wHjjYCp7utddeZJsigvGenNxJT8h3jGWQ+J63B/fQT+VwKxk9iVAhgMKpp57aXXbZZTw2kqb9Pnf4Ok+8EB65w42zKsuQPke+eq4vgEvOPX+3l4/zV7ziFR3OJkh92Z2VNeNpqSxu5HUnY8p84T9FpXHHhhytfs2f4Vxgh1F1yZ+pAbhmyZe8DP7bwSB9ZQLPl3jtBsuaY1GGSfLgHOPzJAAu8kGekhey1ImtD0444QS/NPK8RQ4NlXUOrKHu1113XdqmD6dvjZYHWedOEeqBzAd0ghNWRpBa/aZxXQ538pJzrpSvywMi9+AcyenQQw9NERp97Ps4wSHAqvecfHvMPn3Py4xzEcdDTiWQt55jnuK5HMhKHlsuANTgPIJKAJl0Y+E/gIiMIZ6ZtzBi6CwAXLPoIzXnXqs8db7k5619J8+H3xgA2QqHqGfbb7/9HKjZgVoORs8jAUnu0AfkMCJP5i2ILV4E1EoX7J8AGSU9x5KNPEWHYjsitktG78G4CbUAuPLotOSjObmkj7ANMWMAQn4effTR6dwdsCUA15A5yeeyGiDHwWOpAAv+vepVr+rWXHPN9JM6sIWYk2QLYCLVwe/rvE//cJ2oD4CrZfzhbAX0ydzqzm7K6U5mnKwC9rXMo9IL8j7ZKne9/RwsJ/6yjSzbyUIsPoF/TgAqFeFEQHO/n5/7+2bdX/w76uyzz+7OOeecvDhpkQaLNSD/1soT9ulnkqUCCuR50EcAlEA41wGG1nSW/NnSb43jvL9J5yzJBPLxd+a67DTzZI4EWFgigCQaB6X7ujZtAJfaaNS3iGSq6zU+pwkE0yInHAgFeIzy5ORbnvsYEcBiHDi8z5zjQDXPz6Nzef1b6prXy3+X+Mn91r5bA3BN8q3vgBUtAKzVYWnPcZSrpuOVeO3g7pqdgDyZ0+hPPt9IRnC/9A1B1C5AQlAe2Std7PmvTz/W3FqTdbwK2yB6O8R3AvoY2xpCQ+u+JAFc3kZ5tOZU+AX/iIyHvGDO4TuNxaKTArhqsnHceBA4tjb/UeaXvvSl3QYbbJCKL5uV6wRDdE8yqelD6QXxLzgQHAgOBAdmzgHZLgPANYbVAeAaw6C4HRwIDgQHggPBgeDAcsMBnHBsGYfRF8NNDjKgIm5E4reMfbkBm3uQr0qXodqNOTiGfLX5XU8tuo2BntO92pHtmDA25+VmlS0rVnHQawsb5YFjBAcJ5CtttUoNQwgOEo5QyzvcWO9gJbZIYKsEEbwtkerDViA4QKFRfHfgU8l4NwmAq9TOLe9z4BN10jYnef3lRKQNFfljVN15vtTnuK7n+gC4vE5ESmILuRI5L7VVgTsr8y1jlEepLG7kZUvRCy+8UMnnjopaIsO+G/VLYD09KEd4zTCodH4sAbhmzRd/f37uDtK+MoE8Srx2gyVRWHDm5+SrWZFZGKgnBXDxDt8Kk9/IP5zKtfFPmpxa5NBQWQf4ANANvHKinESZAWyDzPSxu7zIOoCzAGhzQjYw3qlXn+g8+fN9fgvURNqS80V5uDyogR5Kjk7fPmVUn9K8Mkq+qSySO6OcNHIM8E5FgpHzyecu5amjg1zyLUpwoDBeSIPzvUSzAHDxnmn3kZJzbxJ5WuKFrrX2HT1P5E+iZqETCvCkezo6gItrRBAFYAcxd7A9oQO7iAxJf4d8C0x+1/qp+ihpcPyhf4wjHOBE2tloo41StEOAHyVqAXCV9FX1cc3J/i7v20MAXEPmJJ/LHGwwrg/gQGV8QYzXvA2ke5UAXEP7h+tENVnmfOO8Zfw5mAmAIHo/C1EUoS+XFS3zaM1h2Sp3vf0cyCF++LZ6p512WgIb6x5Hr/O0AFylNmrpLw7gArzIGMjJt5DNZcqQftYSsY+y1HSWvJyl33wri+dsrQooDJLOWfpe4n7tG4V708zTdT7ydmJ8+7bufs/Ppwng8kiwpb6u9ypCjctUl2cCcJF+qJxwAFcNGM8cAn8glxk1AFfrnCMZ63qVby2cR3YbWtdUgcq/Gj9b+24NwDXJt74DVugTAC6dvA7TklnKf4js0TMlHY97Xk713a222qrbaaed0qNEhGTRVolcvqOHA7BxAFfJfuJ2gRtuuKH72Mc+Vsp6sWst/VgALhzIpSjyvMQXbRJZ6pGPfGRz3ZckgIuFIlpc8p3vfKcDRDiOJrERSL6w6Ebn/r5x4wH9Wvp6rs8pH9ep1e9cBxmie5JnTR/S++IYHAgOBAeCA7PlQAC4evI3AFw9GRXJggPBgeBAcCA4EBxYpjmAMYltRvzjngJjBMCIikEIwwC/5Zzlvox9edQF7kElQ7UMPtyX8ZtzJ1brKaLXELAGK+EARhHSPK8L+VNOVqnJEYjRn1X4pNU2PUSbwFkIlYzMQ99RcxDI4Z1e1OOfOwNG8d2dwu401CscdDQ0AlepnVveB2CFlYt5f1IZdVQ93cCta6Wy8Fypz3Fdz/UBcBFVC8MzNCoqjBtmzz///O7MM89cZAvFklGZPEtl8bxKWxHwnIAUcmy4I7wWrp/nFGliUgDXrPlCWWs0KwBXjW+tAC4B3/LtEVSvLbbYomNVuqi24lX3a8ehcqhF1mGM3n333dMKYxmGvTyMy8MOOyw5yrm+PMk6HGXbbLNNt8oqq3iV5s4dDDF3cQonLQCumjwoAbg01vsWtWaw9+flGKjJXNIqYpvL9D7RNtx54ACu2vxI/kRNI8oA5A5WfsvxUZq7uZ+TO8MUDUlpptlHSs69SeSpylg69plLSn0HkBz9E+BVToCoaX8iDEA52MKBJpdcckmKpuMgdY+g4RHg8vfUfpfAU3la9EYiS+GIzEk6H4sToBYAV0lfFYCrBHRqBXANmZN8/LjMGtcHHJBTqpfABV6v1v7RAuCijYaOP+9X2nLcQSwlfWzoPFpzWLbKXW+/Urv7uDrllFMW29bUHfyldoSPTv6+WfcXB3DVxi+R8ZDB0KWXXtrxvdfSzzxayk033ZT0Eq937byms9TS59fV7r6tnfT72nxZ+0ZR3tPK0/u+8tZxaQC4iPzCvA5de+213THHHKPiLHIUGFLfOdx0eSYQjB4aIiccwFWSB8pTugtAZHR6qKRbTDLnuI7DNtHILHQQIhRBHhU8XVjwb0hd9UzpWONna9+tAbgm+dZ3wEpJfngdSjrykpzj4HFJx+O6l1N91yOTKRISaXNy/bQE4CrJ/BYAV2s/lj1P9rO8/PxmYShAXQi9i6j+fAdDQ+s+DsDl/dC3hnbw6KioX6lQC/95G5111lkdi/TGUR8AV81GUJIv/r5R4wEbZimaoD+fn2sbYdcJSjoIz5XsIVyv6UPcCwoOBAeCA8GB2XMgAFw9eRwArp6MimTBgeBAcCA4EBwIDiyzHGCFOqAmAZ5YlQtohRVnbOkBEYll9dVXXwxw02Lsc4cDocYBleTk2wsOAXApHxx4RGDAmINhH5CRyJ0WXJOxmHNWec6fPz9F2uK3r6rmt1Pfd9QcBB6Jh4gveXQwfxfnOAPYRhEaxfcWQFXKdMS/ab/Pt+8prSBVUQQccOPgqLLwXM05ouf6ALh8G8pREWqI9MH7oOOPPz6Nmz7OylJZ3MhbMkbzjhzA5YbaUUAMOSMmBXDNmi/UsUbLC4BLDn22i8MZ4wT/MHjnYCgZUj1t3/O+coj8WmUdz+LoZvshHK4Cz3DdnWzLo6xDXjLf4NhjZTjGbNGoLZ2UZugRRxTADgj5RmSiEvWRByUQjrcxDgGAN6MIByVtOIokdxxEnKcvOepUvlERuBys/d3vfrcjSqUDMQBsEUWALZGJKMk8CGlumCWAS3WcRh8pOfcmkacqW+nY2nfcoYz8QgeE7wAiaHsHt+cALuQQdXQwvLZ0yWUh40wLAQDtnXzyyaVqzF2jDwN0HUeKNEk6dAa2B0WX5R1EUWWRAoAYKABc/xeBq+QELgG4WvtHH50oNUrlX9/xR9+j3MyvklWK5oMcqW3jymv7zqM1h2Wr3B3nPF1RAFzMRUQWyakkq1r6mUf7cyBw/r78d01nydPVfkum0L/oA8x10u9bAVzTyhPQvutqXgci9gLAH0fTjMDFoh30BIi+IMBCXgbNG65bej8RCCZ/ro+ccACXgMZ5PgAI0V0gj2ys8jo4fJI5x+dBvt+IlIT8Qo7xrc+3Qo361LX2LNdr/Gztuw6ccZvNJN/6owAreR1K38wtAK4W2SM+l3S8vJzqu741riIhKR8/Yh9hQSKEzoSccVBXae52u0DfCFyt/Zg+CqjfF9l5+Tn38rI4kr4HuBQaWvdxAC6PmjUpgIstt1nwCCnCevox4l8fAFfNRlCSL/6qceOBqKNEnGXxGJHOxhHbZqMXj9NByCcAXOO4GfeDA8GB4MDS4UAAuHryHaNbUHAgOBAcCA4EB4IDwYHlmQNu8KgZe2RwwHgkxxt1bjH2+ars008/PW1XlfPPjcZuDMzT6ffDH/7wue0Iv/zlL88BnXQf4AHvhdwozG83ZH7ta19LKwUxfOdOx9Z31BwE7qSubeWAIXfXXXdNziVWeGNwhkbxHcMuK7ihZTUC1/Oe97xus802S2U86aSTuiuvvDKd+z83bgMqxLANjao796cB4CIfgZ5qThjSuNMOAyhggj7OStXBwWTeD0vGaN4nIIX3YRntauAsjL8YgaFamnQz+6dVov4uksySL1kRFvm5LAC4MFTLqVICzBHdBVkJ5fKDa26Mpy0U6Qa5Cr+JLDSOWuUQ+XofGyXrcGYrShgACIAzTvABx7i2J8MxjtF4eZB1zAXrrLNOKi8ArZz23nvv7jGPeUy6XNqCNk8/9PesAVzugKptW4UjlehjELIXkOooUpnppzhgaOucBPhwHUHb6nGNPJAlOTmQ+atf/WrH/O3b7pTmfwc+zQLANYs+UnPutcrTnI/+28d5bS4RuM7lu+t5JXnk2/7kAC7ejzNx3oJtxSAcSDhQoTziAQAbAHiQRzhJF+wf2zg+5CEP6ZgncfSNIt8qjzoBFNDW13rO+1UAuIYDuFr7Rx+dSG3EcZLx587io446qtt///1T1h4hiQut82gNwNUqd8c5T1cUAFdtO3HX66RDt/YzydIaYJht0fieguTcr+ksKVGPf4Al0Of5Vrvgggs6ojpJv699O9S+UfS6WeSpvIce/Vv83HPP7WhHUW1O0/3SUW3k3z6ejvHA3AQ/fW7yOU0gmBY54QCuGoiMfGUvUFQ4ypgDLKYx56ivAIDBHpL3T/Gmpa56tnQs8ZN0Ks/QvlsDcE3yrT8OsOJ1KOk5LQCuVtkD72rjwcupvsuCGEAx0Pe+973u2GOPTef+jzEgQLJ/T/ocNw0A1yT9WAAuyl3b5loRBdHHmD8nqbsDuA4++ODFvin222+/bv31109slIznR0sELo8OWVvIp8jDvIP5i8UGrTaCXL6Qp9O48SAQHt9b7GCQ67/khfzjD2KuYpyP00FIGwAuuBAUHAgOBAeWPQ4EgKtnmwSAqyejIllwIDgQHAgOBAeCA8ssB3baaacOxxz0zW9+c7FoCG5scecs6VuMfQ5cKgFKcO5hwFWUmpIDl3c74XDH8Q7VVtXKMJcb93mPVr2yal9b7BB9gm1/RK3vcAeBbyPkK/UwzgGCgL9Ovt3h5Zdf3rGFCjSK78sDgMuNvUR5Y8V1Tm4w8ghFo+pOHjXniJ7LHQc1o6sATOR55JFHdkRuc8IgiJMYI6uiPXC/j7OyVBY38paM0eRdAnBptTr3S05pgSi4XxpvXC+R6u9OFNLpOufT5gt51sgdfX1kgvIp8XoSg6UcUHwww1snNx67wZ00vpoXAAzG5wMPPDBFfeJ+bRxwz6lVDpFHX1kHMAuABX371ltv7Q455BAvQjp3MATgHLbaWh5k3Yte9KIUmZFKYDDHiecEeFf1rRntPf3Qc4GheG4WEbgckO3AVy+nR8H86Ec/OhbA5WPPHZrK0510riM4GM7nLz1HP8PpAM8hZBnjAEcQ0Z6gUhvh5MQZD80CwDWLPjLteSZVvvKvz1xSAnApqlnJAYQsQGYpelwJwOXOcfQsgAgQDrzbbrttkdJKH+MivCFqjpNHZsvb2NPpnG2nFPGwJEtxUuJYA5wKleZK5eXHcc5S1cO3GtTzK9oWiq39o49OJJ5xnGT8+feK6/Of/vSnE5BQ72mdRwXgIh93nrfK3XG6yIoC4CKiEFvC+TeOA5V83mjtZ4xvbfFaip4sIAFtp7mmprOQpi+pT0gGSOccCoLx980iT8+/7/m0AVwCGfB+om0SddPJoxr7t4fPaQLBtMgJn6MATVGefOEEkcH5PofQ92+++eZ0ngMspjHnuO4kecVYYLEHv0UtddWzpWOJn6Rr7bv+Te/fZ369NC/zztq3/jjAiteh9M3cAuBqlT3Uo6bjeTnVd7Ez8T2PXgXQBp2Xb0anpzzlKR3bUEO+YGicToJcJW+otigz3Vz4b5J+7DK3tP3euuuuOwei1vfIJHV3eXTCCSekKKuqC/oWOio8hSYFcBGJD/2O/EpjEl8w9hDu802PDQ9qtRHk8iVlZv/GjQf/Js/BtmTD9z/9Ap2DPsd3IMdxOgjP+hgFHKbojZqnSOP6EL+DggPBgeBAcGD2HAgAV08eB4CrJ6MiWXAgOBAcCA4EB4IDyywH3JGBEYKIPmxXg1MVwxMGbhlEMGLwwc4RajX2OQgEBzpbB2CsBHyEUUHGU97hxkB+l8gNLayAI/KIoorgsCPaiCKO8D4ZSpSXh/nXNQwdvm1P6zvceAcICKM1zk3IV+8BlID3ukeIfQBcMh7BMxmaR/F9eQBwUXecOYpARFsRLQGjEE79nXfeeS6iGv0C46ZoVN1JMxTApfzo0xhgaXOMWu50pFz0KYG4AG9hrBPYj5WMrL6H/Dm/nm4u/Kd3OpjM+0nJGM2j9EmMs+7Y8NXilJOoQkQ8WW+99ToimBBpQjQNAJfXb9p8UTlLRweRYAhmvIwitv669tpr52SU83oSg6XLLsAstDHyCkcr4FSRA7gc7Md9ATkZq+QnsGqfbRpa5ZDK1UfWkdaBgWeeeWZ3/vnnK4u0ghdwDmPVDdfeh5dVWefb8+JUxtjOuIBoD4zkgAAgjxrixnNF2kiJBv6bNYCL4rijmnFAlEPkG/2Mdttwww1TqeVwHleFVVZZJclr6QEXX3xxR7QsHCYAqZBBIt6jKJ3wEwepnjv77LO7c845JyVFjuFwkHzybYncWfmrX/0qRXOirIwjAOdsjyzKZZrmds9PaUtHd4YBAGLctvaRUv66Jpk/rXlG+ZaOPg5rc0kJwOVzMoA7dC9k/JprrtntueeeHZHbRETmY9vgnJSvrufto+tbbLHFXJQ/5ngAVdpOlMgH++67b4esg0477bQOwNgooo+hP6mvCRxAn6e/7LHHHnP5kc9xxx3XXXPNNaOyTPe8f5QcVCsTgKu1f7jOUNOJvCEmHX9qE+XJNwHfFU6t86j3hzPOOCMtGCF/qEXujtNFVhQAF/wBhIA8IuoecwfjiTkC8sVDrf2MyCuA6CHkFlFtiEKMTEB+0a8gn/dcVuY6i4NP2G4K2VYiBwUx//Bdd+973ztFN9GWgf5c7RvF08wiT8+/77kDJnJQQG1OG5W3f6/wnYUuLDkMCJL5XTLcv3O8nQSCaZETzlfKieOLfgJIi++5Aw44oJs3b16qgn8zcCHXLaYx57DFJdFsnQR08WstdfXn8/MSP0mjNh0KPvSxkttsfDwP+dZ3nVuLRLweXoeSntMC4PKyDtWBxLtcx/Nyqu9SD9dz6YfYH9j6GUJnQVfXWECPBsgO+RxU0kmGArgm6ccO4KJsfBPzHcu3FXWgjnwnQoDjpOO11t37GXrj4Ycfnmw2XAf8qcUgvM8BXL4wB/lPWyFfkEGjCL3xiU98YkpCGyEDACIyb8F75jHIo9232AjII5cvXHMaNx6QJSwqoz3pg4xD6c0Am1kwxjcU5OUdp4OQvgbg8r6Y60M8FxQcCA4EB4IDs+VAALh68jcAXD0ZFcmCA8GB4EBwIDgQHFhmOYCDi4gjAqNQUBwSilTAbwwluo9hHCfMRRdd1GzsY1UeRm4Zp/J3+vtzYyBpS+RbM3IfYAFGDBw1eg+/McL/8pe/XCSL3KiLES03qvJAyzsw9ODYcdLWkRjaAEpgQBHhQIb3KjPXcwCFDIUlI6u/z400yt+jep166qndZZddplvV4yzeB0gC44/Xk76FsU/XaC/AB4AQRKPKQpqac0TP5UZ5N0DxvN/3SDXcw9hHmQS44Vq+qriPs7JUFjfylozRvKsE4OI6hkQ5Hfhdo5ozvZReRkh3oijdrPii/EtHB3CV7ufX5BAs8XoSg6VH/8nfyW/6B/3XAVzIEoyrUA4scSADz+J8duBoeij71yKHlEVfWefAXp5lbFIn5gGNT66ztRlzAeSyJ11Y8G9ZlHXudIDnzG+MaZfD9HuM4dQb8uhq1Hfclm7pocK/JQHgyudX6kjbCRBDsbiGQwVnYR/abrvtum233baalPzoFxwF4CLxjjvumMCNepD7yFGXoZQN2YYshYgIgIzxfkY76BnyQEdQe/E84xxgwDgniMqho8t/Abi419JHlGfp6O/h/qTzTOkdutZnLhHQyuU7gN/58+crm9SW/FA7wGfXTXDSwScnd8xx3UF7no5zj7TGb401tTPXcnnJtRq5w5Y05Of6BDwHWAHRhwD4EqlnFHm7lZylAgs5KET54dxjvEOui3me6nOtc5I/BwCFqJjQuD7gvCrVS1uier1a+0cfnSgV2v5NMv7y+TGPpqvX5On6fDM4X5WPwAUtctfbrxTBZEUCcIlfjD3JFK7hfPBopq39jLzc0cxv3gXpffxmjlBUpVE6C457HPgQOhlzVI0kBxiDRA+cFMDFe2aRZ638teujAFwux3je57Raflxn8cvqq68+l6TURrSXz00+7hwEM1RO5PqvCpH3SX4jAz0yZEm3cDlKXi1zjteBPPJogVyDPB3l66O33vXk4v9r/NT3Usm2QC6172sH1uQ2m9Zv/XGAFa9D6ZvZ22bWcxy8qY0HL6f3Xb6lsP+4Xk7/oe8L9ES+V1xxRQK+cg75e0r1GgrgIk/nFb/79mPvkzwnyscTIE1A86LWulM37GmuIypPjpRb9xzAxT31bc6hPhGWmZ8B9TkwjO8Xbx/GId/4HKEWGwHPleQL10XjxgPpPIIhv2kH/1biGotEmW/R56FxOghpfF71CFzet0kHSR+661f8Dw4EB4IDwYFZciAAXD25GwCunoyKZMGB4EBwIDgQHAgOLNMcwOCMUUjOLRUWY8hZZ53Vfe973+sOOuigOaPFhRde2LHaCuAXho0a4Klm7CP/Rz3qUWm1K8YDJ4wgRFLgfVBuDPS0fo7RZv/995+LnuL3OCdfwED5lg1Kp/D3/K45HVvf4ZG2yB+eskoRgn/wXpFI0sWF/3AoseqZP6dRfHeHRG40Iw8HcJ144omLgKP8HX4+q/fV+h3vpu44I3+8IGKa06iykK7W5/Rc7mTAuEy0DwEUea9C4ZOfOzD47VRyDLLqdJ999knJBF7xZzgvlYVINhj+oJoBvwbg4hnAFTi+ZAzGcEddici12267pchdfbaiIi9oFICL+7PgC/nWyLdjq6Xx6wJwlXgNCAHnDFRqQ67XDJbcy0EKXIPfgCFx2tOnMI7inCfy3/bbb0+SZFzGcIpjyskBcTkg0NPpvFUO6fk+so60O+ywQ+pTcn7qeY4YsYlEhSxzWh5kHauQcVgoAqCXn3PGCRH3fNs3ZIQiV/WJlJbnqd/Maw95yEPST1+Rrvs69pEHJRCOnke2Ek3CHQ+6x3yNw+v666/XpV5HIpwQyYQ5RoS8vPLKK7sHP/jBqd8zDhzARbrNN988yTZ3euh5eE0UNIAiTtR/l112mdM5dA+ZBuiGLRYlL7nHeAaMpu1L+jhneA6dAeAFJDAN5y19hOdqNO15pvYerk/Sd3JQi97DWMDBhNzfYIMN0mX0Q5w5Tquttlra/olr9AXmUjmKPJ3OPcKBrnHkWXQ15kJkTR9iDidCrMaXniEvdFiichHJQv0XID8giVHk/aPkLBXIIgehkKcDuFwX8zzV51rnJH/OAVzj+oA7bEv1EoArr1dL/+ijE+VtMMn4o/31/UC+OHxdlutdLfMoDmT6tADZ5MWcLgP6ULnr7VfSRRzAVdLXXXcotaPqqqO/b9b9xfsK31ToqPkcgL5D9N28ffxZlZ1jHznEvIFjOSfmGOY9gHJONZ1l0003TfMdaQHyoDfVSFvhIRP51mz9Lvb8Z5Gn59/nXGUgbf5dPG5Oq+WPPsk3EmCqnC655JI09xJB2AFcLs8cBDNUTjiAC2AMCxX07aeyoNPQT66++mpdSseSbjGNOcfB8fn3pxdgaF392fy8xk99Lw216YwCcPHulm99/85iS0lFJ1ddvA6lb+YlOcdRptp48HJ63+UZ5ASRkbRlONdE6C3Y2bRARtddfyjJ/BYAV2s/1naj6FLo8nwj5d+LtQUvLXWHB/CKbwzZOsQXFqixrSKRfaEcwPWMZzyjY6xpDrrpppu6ww47TI9Xj9goaaM11lhjsTREZkSvpO5O3nd1nfYs2Qh0vyRfdI+j51kaD0o76nuLeYzvJxZfiFwnKOkgpKvZQ8bpQ3pHHIMDwYHgQHBgNhzQ9yfAe75rBQJnruNcv/u+/W6bbbbZXUtf+j6xnKQLANdy0lBRzOBAcCA4EBwIDgQHxnIARW/jjTdO2+VgOGWLHI9Uhd6z1lprpdVaOKNyY9rYFxQSoGjiFCZfDN847LQyupC81yW2+cHwjpGHOrHVGo5dtpXDgFIjAVZIM87p2PIOjK9EFkHRpo65UxLHJ7wAyIUSDnCJNljRiTaat2DLCpzoODxpqxtuuKF3ZJhp8QeHAWXh/RgCndhCjPv0U5zRbAVDGXMgjj+ztM4xqAGmoB7qYwAiqRv9DiPhtGh54su06qx8MOrSZxmvbPNGf8hBKEo7q2OLHKIsQ2QdYAfmBQC3GPnZxhWjNc6tWn2XF1lHnXA8sQUmq5SJsnHjjTfObb87q3ZbUvlitGH7O+YVAAdEqCLqEFGNJiHmbfo9/NKq84MPPjjJnZoDkr7D1nj8ARTHeUJUIspUI/U93kWfYw7HsShijmdOpQw45SXvdH8ax2n3keVhnkEXwdGNfAHkgb5Hu4qQe/e9731TxDTacVJCtwQUBq8Zh8hTHF3+ziHvwJEKn3HwoVMC5pPuJ7lNH2SsK+rbkPxX9rRLsn+0jD/aHWAd8g/5wnw3ilrmUQAJbEuE49a/k3jPrOTuqDosi/cchIWjH72asUlENnRndCZ06RpN0s+Yo9ZZZ52kszNnIKd+8pOf1F6VQEOjvs+qD8aNJGtr306j2IOeKP2LrdWQ1a3fVH3lhAO4tEU2gE+2P2duoJ8AYhiqS0wy5zj4KY+2VOJf37qWnl2a15aVb/0+PJhE9ozS8Wrv5n08B0gInQS9C/0Eu9iSpEn6MeWkjQFF0kfRu9DxRwH4eaa17oxbvm2Yb+FVH9sh9hG+QdAtsbENsWMCOENP5btD34qj9EfpmkvDRoAOxPyHrk45KCcLbUZ9b9EWrTRKH2rNM54LDgQHggPBgfEcCADXeB6lFAHg6smoSBYcCA4EB4IDwYHgQHBgGeYA2zko6hJRSVgRHhQcWB44QGQqHE8YSYmakRt8MciyehK6/PLLu1NOOWV5qFaUcUYcCFk3I8auwNkS2Y9V3RBRYHB8OOEsYHsSnOYAGohyFBQcCA4EB5YGB4gc+fSnPz29um8E36VRzhX9nSUA14pe56jfss2BEoBraZfYo+h5NL+lXa54f3AgOBAcCA4EB4IDwYHgwLLLgQBw9WybAHD1ZFQkCw4EB4IDwYHgQHAgOLCMcYCw4azcI+ID4C1W1UFsU8fquqDgwPLAAd/+5Rvf+EZ32mmnzRWbFe5sJURfhwBZEN0kaOXiQMi6lau9p13bBdHUO7ZSgohoyRypCBWsuH/Tm96UIphw/5xzzklbLXEeFBwIDgQHlgQHiKyhreuI9gSotBYNcEmUJ97RdQHgil6wrHFgWQFwIa+I/uMLbIgQR7TkoOBAcCA4EBwIDgQHggPBgeDAOA4EgGschxbeDwBXT0ZFsuBAcCA4EBwIDgQHggPLGAdkyGVrHYBcENvOEcUoKDiwvHBg7bXXThG21IfZguSPf/xj2qaMkP+iCy+8sDvjjDP0M44rEQdC1q1EjT2DqrIFxyGHHJK24iB7QM+//e1v05aabKUn2fPzn/88zZ/arm4GRYksgwPBgeDAYhwALMRWv5JFJDjvvPO6s846a7G0cWHJcCAAXEuGz/GW/hyQLswT2kKx/9PTS/n2t7+9W3XVVReRVx/84Ac7tpIMCg4EB4IDwYHgQHAgOBAcCA6M40AAuMZxaOH9AHD1ZFQkCw4EB4IDwYHgQHAgOLCMccANuRQN0MsHPvCB7vbbb1/GShrFCQ6M5gDbmz33uc9NW5iVUn7961/vTj/99NKtuLYScCBk3UrQyDOu4mqrrda96lWvSsDQ0qsCvFXiSlwLDgQHlgQHHCzE+2IxxpLg+uh3eJsQFe3OO+8c/UDcDQ7MmAOuCy9tANf97ne/udrGN9ocK+IkOBAcCA4EB4IDwYHgQHCgBwcCwNWDSSQJAFdPRkWy4EBwIDgQHAgOBAeCA8sYB+5zn/t0W2+9dQK93Hzzzd0111zT/elPf1rGShnFCQ7048Dd73737qlPfWr3iEc8osMxwFZn1113XXfttdemrYT65RKpVkQOhKxbEVt16dTpcY97XLfeeut197///Tui/d1www1p7vzd7363dAoUbw0OBAdWeg6g92y66aZJ1/nhD3/YXX/99Ss9T5Y2Ax70oAd1RIiFrrzyyhS5cWmXKd6/cnPgnve8Z4rUBxd+8IMfdL/5zW+WCkPQoxgbd9xxR/pG+8UvfrFUyhEvDQ4EB4IDwYHgQHAgOBAcWD45EACunu0WAK6ejIpkwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQG8OBICrJ6sCwNWTUZEsOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAc6M2BAHD1ZFUAuHoyKpIFB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDvTkQAK6erAoAV09GRbLgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHCgNwcCwNWTVQHg6smoSBYcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA705kAAuHqyKgBcPRkVyYIDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHeHAgAV09WBYCrJ6MiWXAgOBAcCA4EB4IDKzUH0Jk23XTTbu211+5WX3317q/+6q+6P/3pT90dd9zR/eIXv+i++MUvdr///e+LPNp88827hzzkIcV7XPzf//3f7pZbbuluvPHG7je/+c1i6e52t7t18+fP7/7f//t/3a9//evukksuWSyNX7jf/e7XbbnllunS9ddf3/EXtPJy4AlPeELqt7fffnv3la98JTHiAQ94QPd3f/d36fzyyy9P/a+FQ3/xF3/Rbb/99qlvXnXVVd1PfvKTlmzimRlw4KlPfWr313/916lNaJsVjR75yEd2T3rSk1K1Tj755O7Pf/7zilbFqE9wYBEO7LTTTh36wFD6z//8z+7ss89OesjWW2+dHr/gggt6y/1pzRdDy90nPfx41rOe1TEXoUNdc801fR6LNMsIB6bZfswHD3rQg5JeftFFF6Uarrvuut3GG2+czj/zmc/0rnWtzz/qUY/qNtpoo5TPWWed1f33f/93GpNLuw+W6t67sks5IX1g9913T3rktdde21199dVLuUTx+uDA0uPANGXi0qtF+5sf+9jHdptsskmHDP7lL3/ZnXDCCe2ZLYUnl9f222GHHZIe9c1vfjPxfRqsm0We0yhXngd9bosttkh97r73vW+a13/3u98lHZl5vmSby/OI34ty4GEPe1i31VZbpe8OxsShhx66aIIBv7Chtny7DHhFJA0OBAeCA8GBpcCBAHD1ZHoAuHoyKpIFB4IDwYHgQHAgOLDScuBv//Zvuxe/+MXd3e9+9yoPAA/gED3zzDMXS/Pud7+7u8997rPY9dKF2267rTvuuOMWcaxiTHrXu96VkqPkvuUtbyk9OncNIweALwhnJvkFrbwcePOb39w98IEP7P7whz90Bx98cGIE4K1ddtklnX/+85/vLr744iYGARB629velp4977zzOgydKxKtttpqCawJ7wBZLk/0/ve/v7vHPe7R/fznP+8+8IEPLE9F71XW5z3ved1mm22W0r7uda/r/ud//qfXc5EoOLC8cuCf//mfm4v+6le/OoF2JfcBs1xxxRW98pvWfNHrZQMT/eVf/uWcfPvBD37QHXnkkQNziORLkwPTbL/3vOc93b3vfe/uj3/8Y3fQQQelar3gBS9IYAB+MAb6Uq3P8y2Asxd605ve1P37v/97N6oOS0qHKNW9b11L6ZZUuXm38+/73/9+9/GPf7xUpLgWHOjFgVVXXTUBMfguv+mmm3o9sywl8vGwss1pO+64Y/e0pz1trjn62DzmEi8jJ8tr+2kOYcwcdthhU+HmLPKcSsEWZsKCxwMOOGDkIkuS/uhHP+qOOOKItOBymu9fUfMCOP+yl71skQUnffSvmt7h+tiQb5cVlb9Rr+BAcCA4sKJwIABcPVsyAFw9GRXJggPBgeBAcCA4EBxYKTmw2267pVV5XnmiWfz2t7/t7nWve3XoUkTGEl155ZXdSSedpJ/pOATAxQMAEd761rcmxxC/A8AFF4JaORAArlbOdQkYgDH+zjvvnAOqtee2ZJ8MANeS5Xe8LTgwaw4EgGtxDi+vztLFa7JyXplm+8lZvCwBuABPLwkdolT3SXrUkio3ZfQ+EACuSVotnoUDr33ta7s11lgjRWV9zWtes9wxxcfDygbgeuc735nsKjTaf/zHf3Q//elPu49+9KPLVRsur+33nOc8p3vKU56Sxs0b3vCG7r/+678m5vss8py4UJbB29/+9g4QlwhANgu2AIITad/te9MEtul9K+rxJS95ScfiV4gopfAUu+Y4qukdAeAax7m4HxwIDgQHlk8OBICrZ7sFgKsnoyJZcCA4EBwIDgQHggMrHQfYeu75z3/+XL0J5c8KPAw8Igx1u+66a0da0ac//enuX//1X/WzcwAXRjEntv1hK7B11lknGc4IMw65oSgAXM6xOB/KgRKAi3D0z3jGM1JWF154Yfezn/1saLYp/YoegatmTGxi1hJ+KABcS5jh8brgwIw5gO0GnSMnonKiO7Ads6J15mnYQrfVCTKt+SIv0zR+U2+i8REhle2iv/Wtb00j28hjCXFgmu2HTkNfZbsjtgyFWiNw1fp8KQLXqDosKR2iVPdJmnBJlZsyOuAhAFyTtFo8CweWdwDXKHmyorfwhz/84aTLsH1dH8DHssiP5bX9iBSP/kj5v/KVr6S/Sfk7izwnLZOe32677bptt902/QRgBFCQiNVO2PfYnlh08sknd2wxGTSaA0Rmxz5EFMR//Md/7P70pz+NfmDh3Zre0frt0uulkSg4EBwIDgQHlhoHAsDVk/UB4OrJqEgWHAgOBAeCA8GB4MBKx4H3vve9KcoWFf/2t7/dnXDCCVUeAPQSiAvnEatIRQ7gGhVC/G/+5m+6v//7v0+PsfIRowcUAK7EhvjXyIESgKsxq8UeCwDXYixZZi4EgGuZaYooSHBgphyQ05PonWwnWqNwgtQ4E9dXVA60Arhq/CgBuGppuV5zSI56Zlm4tyTLHQCuZaHFV5wyLO8ArhWnJYbXRFFGf/KTn3Qf+tCHhmcQT0zEgX/4h3/oHvawh3XTBNDNIs+JKrnwYWxsq6++evqFve+WW24pZutz/ne/+93uE5/4RDFdXPw/DvzTP/1Tt8oqqyTgFu3fl2p6R3y79OVgpAsOBAeCA8sXBwLA1bO9AsDVk1GRLDgQHAgOBAeCA8GBlYoD22yzTbf99tunOrNyjMhZRLeoEWHWDz300BRunRVnGCwIGw71BXCR9oMf/GCKJMH561//+pTH0gZwPeIRj0jRmqjX5z//+Y6oYRhTNtpoo8ST73znO93Xv/71ZPDj3jOf+czu0Y9+dHf/+9+/I+rHv/3bv3Vf+tKXqvybN29e2qaSLS8IWf+rX/2q+973vpdWObKFgtOsy/K0pz0thX1/8IMfnOqJERND8mc/+9m0nYOXZbXVVpvrI1/84hdTeiKBPOhBD+p++MMfplWspL/mmms6ttYsEemp8+9///vutNNOKyXpdY3+9+QnP7lbd9110/vvuOOOjnYBeEg/euADH5hC2B988MEpP4yWbJUAsdKWLUGdaN/NNtssbS1A3txnZeq//Mu/LBKBbhyAa9NNN50Lo3/rrbd2Z511lr9m0LnaHpDCKaecksq21VZbdWuuuWbi4W233ZYisHz1q1+t5tu3r2255Zbdox71qFR2ViMz9umTGHjZnol3Qqeeeuoi/CBqx7Oe9ax07xe/+MViK5gf+9jHdhtvvHHK7zOf+cwiW1RMq++xOpj+OArA10QJmwAAQABJREFUhYFeK4/h54knnpi2bk0FH/GPKDdE+njMYx6TjLPwhpXLRCdkjOcGcPoOxm/osssu62688cb03rXXXjv1Sfrpj3/84xQtxSMbpgcW/mMbBvj28Ic/vGP72muvvba74oorOmQ0fRQCtEI9RtEDHvCARaLOIddZWb3hhht2rBJH7jBOKWdOL3rRi5JcZjwR5WfPPffs6Eu//vWvu4985CNzyRn7tD/8lfGafnnJJZeksTiXMDshPfMNR+QBAF6AwF/4whe6H/3oR1nqu36uuuqq3dZbb53KwRYgyCrKdvnll6dylR4iHSvKH/rQh6Y6I18Zl0Qtqq0q7ysLSu/za2ylwvtrcoD55PGPf3ya8xgbmj/Jg3nl2c9+dgfIGfsF4xHeI2eRKcxNOQ3tq/48snGDDTZIcxh9G97Q7x73uMclhw9zAo6cnOgTW2yxRdq+adxclj87ye9WABcyffPNN08RQFVe6nneeectUpxpzReLZDrix/rrr58A8byXfk1EU8Ym/RZ5wHxJ5EgIGcNW19DVV1+dxsALX/jC1GcY4zXgPfJ9ywVyHvrGN76Rnks/Fvwjz6c//empvzGmyQc9hnFSG48tfVTv05E5gD4EnXPOOYtFg+A6/Z8xDCFPL7roonTOP4AwO++8c8oD/QtCjlx33XVJPqcL9g/5R/RXxs8nP/lJu3PXKTJzp512Sj/QE5jTRlEL39kiLG8/fwdzBTKVstzznvfsMPQyz9BHmTucmJtIRznRSaEcwIU8Yw5mrqZdkfsXXHDBIu3Pc7U+787cN73pTWn+L/VB+lZNh0DuTtLOlC+nUt1bdKZR5f7yl7+8yGuH6CyLPJj9yAFc6FXIJca6viNoa2R9/k2grJAZyG1kBNtfoZfQT84888xFxhE6MjIaYl6m/zmN6sPIgvnz56fk9D/mgRKtt956qfzca9X/h/AWfRRdGKqNU8YBfGbeUpTCUd8w0iNTpiP+9ZV7k8i2vu9Q2/L9ea973SuVmvoyztEpXEccoj+5HstcQV9E90RfQR6jd1188cVzkYzpX+is9EX6IemJCJh/Z9XYWpInnnZaOtkuu+yS+gTzww033JDqhI5DP0c35xrjp6RfUZ6h8yQRj+AJegfz2w477JBkMXId/qEfSqdnnDOfM8bguWjoHEd9+GPe4Duec759aDd0A8a/vlGm1ba19nviE5+Y9ErsImeccUaHnKC+yGn0VSJhMzY9grvqzZE+Lb2Ec+Z/ysxzXOcaz+ZRpDyPcefwh+8dCJtWHp27pQ6zyHNcPfrcf9/73pd0Cr4nAH3WiO8ygZD47ihFuR0irzUvLws2tVqd8+t967fJJpskvZkjY5U6MpcgB7FL1Gic3oHMQ15ByPK+3y7+viEy35+L8+BAcCA4EByYHQcCwNWTtyiuQcGB4EBwIDgQHAgOBAeCA4tywFfm9Q0lj1MIpw9GYkAkMhYPAXBhMMNYDQmUsLQBXG44AcBAPQFuOGGkOfzww5MhFOBCThhbqFtuCMYp/9SnPjVPnn4DYgAc4QbEWZUF5/WBBx6YjMulwtCWOJUAjogw0MvwSx/BKCy+YLCXgw5nM1GwcgKQArgKuvPOOztCzrcQRlv6qxy2ngeAF5wX9CEMaAJwOR8B5WE8hzAi45Qs5cV9QBXHHnvsnLNzFICLdqV9Idr9pJNOmnMcpYsD/3mZ4TcAHgzlOeFgd1CN7g/pawA2cTTkRJ/EqYFjEQLQ5g50DIyUE6LP5NFwDjrooOQ4hh/cw3A87b4HAOOYY46pArgwHhMJkLaG6NMYRMcR/YxVtfe4xz2qSXGEu5HWHbM4FgALUN+cAG8hJ+mjTvvss08Civo1zuEfzlm1kWRlns5/02YveclL0iWcqgC3NF49Hf3nyCOPXARApKgAgCJxEuJwgig34wXCaQewopQn92+++ebuYx/72GIOaByvO+64Y/U5AFkAFp2QPThk1YZ+D9587nOf6y699FK/PLZ8OLKOOuqo1Cd5cKgsWORlhR8CFNbkoUexpJ/h5IJwMtO+9KUSITsBMLlztKWvkjd98zWveU1yYObvom8y/nEyluT1EPmS5z3p7xYAF05n+nKJcOgeffTRc7dc9k4yX8xlOOIE5yEOv5zo18yrgBU4p50glzH0YcbuIYccMidn6NPUNSdFaeH68ccf31111VUpCc7rV73qVWnOzJ/hN3KMbbKdWvqoP69z364HwDBzbU7ICpxpkJcFuU4EV/p+iRh38ML1KXjIFt5QKTosDuaXvexl6X4fPbiF78hioi5Aar/0Y8G/Wl/gPn0AEADlEr3nPe9J4B1A1sy1kAO4AOEBAi0RTka2RxLV+jw6H/IXEoCr1AdH6RCAEbUt09B2VvnyY6nuXoe+OtOocisqcIvOkpfXfzv/+FZgPJXmNoz8OM/Ra50AVLNYoET0E0DZWiAB6GavvfZKSekPbJslYixIrnCNBTU//elPdTs5kKXfoWOha5UI4N4rX/nKdKs239X0/xbe8u0BOAZCl/PvlHRxwT/NEQBiFNlo1DeM9Eg9XzoOkXutsm3IO1w25uVlm2E5iYbqT94/b7rpprSYgGs5AcpBx0Vu5kSfRccFkDyO/H0uE6etk6lPMCewaKmk28Ozww47bLFFAS3zpKLyAA6DD5p74Af8kV7t/OH9tB3UMsdJXiMHsMto4Qr5Ie+Z9zX/TKtta+338pe/PC1u49sQ8GjN9sFimHPPPZcizhEL3F7xilcU24hvC3jD97DrBHMPDzyRrp7rgWTTWodZ5DmwWoslV3/kxnHHHZfAtoslWniBBSaMDwDfPie0yGufl5e2Ta1WX10fWj90Zy1wUx46lnRM3RundzjPhny7KP+hMl/PxTE4EBwIDgQHZssB6eZ8L6HnYkdFn+GPc/3uW4q7LUDG/7lv4uUpXQC4lqfWirIGB4IDwYHgQHAgOLCkOOCgK7ZDJIpBK3leowwYy+oWim44EQ9wZuMwV6QqXeeIoRTnO8ZYgEMiosl87Wtf088UxUZOF54hMguONyIjsIIe4jrAMEW9mFVZ3v72tyfHPO8E8ISBDoMyThbpy5QFIyQRWSB3fqQLC/+RjpXDGPHlTC31IXcsYqzFaNtC5O1lhPeUgUgUDibpA+Dad999E7CFclB/nD08h3FYoC6MzzhHATPUAFyAUhS5g7LgIMewPAmV2l4RNKg/qytFRGrwKDLu4KM84/oaQCxW8dO/4SFAKyIp4XgA7PbSl740vcqdK1wQQEvlyB2AOM74IOX9OLuhWfQ9AAYymOMQlYMCJwAGVjlHWb0tx6bKXDtqGwzuA1xiTDLGAa36OPetKNyRoXzhP+ArgKpEhlMfxXmCo0gE2EpAOa7h8OG9tInKr7RDAVx6DkAi0VoA5TjwFKCWR6QRgEvP6chYwwmAcR+wmYhV2uSLkwt5RptDeR19ZTr3MV7gTGNc8Zzo4x//ePf9738//ST62d57761bSQ5TDnjpdXDABeODcsJr+M98hoyj3dZaa6258gH6IkoBNFQWzBWocqL+WHNo1wBcOOzVv+gDyCTaHye5HH6MTeZZUUtf5Vl36PCb9zH2kS3qp1zPAVxD5Qt5TJPkiC2BRv09JRmKnGc8wmPmDJGDU/05B3BNu4/I2aoyoA8w5pl7NIa4Rx8W0MJljOSxgwVKTkj6D4By2hQZICA1kfiYT/UuooAw3zN+fD7Nx3FLH1Ud/Ug/e8c73pEuMb8KMONpfL6QrM3LzRhjfKN/ED1CjnnamjmKPg1NG8DVwvdS+1E2tiRHJkD0a9qBeRP5j14mAoADEAcqgZhcz9IzjF9kYC4zkX0Cvtb6vPfRUQCuUToEwIGWdlb5S8dS3b0OemaczjSq3AAAIe+DffVlvb909D6g+2pzxiogHsnfHHTl7YtcYN6lben3yA09RySh888/P80dGvsO9OO9RIlUZFp+E8Xt9NNP5zQR7c2czvgZFbGFxIzNFv2/hbfTAHDdVcO7/sNHvmFyoKqn4XyI3GuVbUPeAThPkei0EArABcQciTxv0Z9K/ZN5A3mE7gjIISccUowNdDn1QSKGHnHEEXnSxX77+zSnkWja8630BhWAdqdOzH+um6MHvfWtb1WyFL21ZZ7M9StlyHiinq73Mv/xvYXc/9SnPrXYO/vOcS6vqZ/agnMiDxOhTd9HKs+kbVtrP4Gf9B6O1ANbCvyWvOC62wzQzfjtZUfnJz1zv9M0AFySqchgdGnpC7yntQ6zyNPr3XK+//77zy1ioD/QB1mQhc7I7z7UIq9L8/LSsqmNq+PQ+hFVj+925knkCHxEptCHiHhWo3F6R4lnfb5deF+LzK+VM64HB4IDwYHgwHQ5EACunvyUs6dn8kgWHAgOBAeCA8GB4EBwYKXggIAWVHYU6KoPM0YBuHCs4cTnDwCXgAlEJWBFIITxDiM25KtR04XCP7bV0jYfnk8haa9LueGErYtYaQxRNoyycnhi8IN3gEYgj3oDAAEgAoQzlKhUGCQxlgLc8FWNbGNCpC/IV4zPoixsCbDHHnukd8Ff2su3aZHhkQQCbHCeA7gwcrMqXxEuHJDgwAieheRww8CFU9XfeVeK8f+J4oDTFMIACGBIYEOcXjhoBXLoA+DCkI3xmXZ84xvfmBwQKoUbwonwwwriEoDL+x91+8QnPjFyZavyH3fM255IWEQowUkCYQAkDeSgpda+Rj7ihwM26LPwmT7vPCU91zWG+e0OQLaCYgU1BLgMkNms+h7vEGBGvJg3b16KCqGxChANMEYfAhjEOIcAJzFG3MDtkQ8cOOSODJ6Fj0RHUx/1CC8OpMDZh+EYXvMewFTasg6ZCWDLAU4tAK4chMEWTPvttx/FTO9E5qqcDuBibBBNDkeJeOAyPgeq4phhfKtf+Epvj1ijPpEKsOAfhnC2RYHc8efPOOCAdPQntmWFbw4AYVtb/qActOfOIe/P6vt9ZUHKfMQ/9cchAC54p+iFOUiLfkzUQvoKxApuDPqtfdVlKbKY/ieZDPCRCI30Z8jlwSTyJWU2hX9yxNJWjIUa5TI0j+7m/YR5DKAD5M85gGuafQSwBZE2NOYZc0SWgHBSUi+BiBl3owBc3gd8HKTMFvzztnawpjsn/TrPMZ8C9BIwgPkbcEBLH1U5Skcc5aonehFySoTtDkcu5OMIuYX8gohOIKANv5E7AE+UJ9GfkF/QtAFcLXz3OcLBCh4hzUFalNt1L39GOpUDc1x/41kAy8hAkc9dyA/0HvpXrc+7HjQKwKX8NUZcZnCvpZ2VZ+lYqrvXgWf66kykrZW7VWchzxp5HyAN44rvCNoDQv7SV5EN6Hvazsr1BGQf44XvBRELRADXQjiQkS+KmomDGSIqLfMelAPwKQd8FUnOSqfS9dKxRf9v5e20AFz5N0ypXrrWIveG9vmWd1A+yQ6fJ1TuFv0p75+0P/OTvj3QFQByiYiYytwKIbMBa6KvuMxW2tLR3+fyTWNyWjqZ+jNlYKwhz9DDIeZcyq35DiCjtqVtmSfJ0wFctA0LuvheYI4WSddGVhGxVtQ6x7m8Ji8AoAATaQvIec3vabSt5+nt53yj/izc8m1pXf7498ABBxyQIrtRPqKQIud4Hnryk5+cvn2RjdA0AFzoCgLsebuTf2sdZpEn5ZmEkC/+baa84C3gQfRPImSiC2pcKA3HVnmdz8tL06bm9cnPW+tHPhrr/k2Z51/6LRmX60s5z/p+u/COFplfKltcCw4EB4IDwYHpcyAAXD15GgCunoyKZMGB4EBwIDgQHAgOrDQccOObOwtaGeDO/T55YCjCkCrnxbIE4MqNKtTHjdcO3lBdBYYDDCHnI1v9aCsbVsF+61vfUvK5o4wuGNNwqsEPN+JMqyyUSTqxIlrMFWLhibehnHbuRKTNMATKoM9jOPa19UO+gpmoRYquQfQT3ttCXnaM8XJ4Ky9feeiGNOejO+Rl0MegTj29PjgotJ0SW+XhCM4BXLTRs571rPR62g3AXm2bGZWx79HLXAIyAuwBJAJ532jta+RTMyZ6uH/6JkAPopQJPEHdMai7AxCQIAZRCMcIBmJvv2n2Pd4hwAwOCdoY8JjAWzhOABr1JcBnrFaGAO/JOaXnfTsgBwi5LIUnOAX0oa5nfWwJLOvRt9y4rWeIeMBzcloMBXDlMlb5urPfwXdyKpEOAJoiAvJ7k002Sdt0ce6gF36LFkTzTsAqfvMsebhzOQcnkY62ov8BwGBc0ed8W9IaONdXlRO5gEgaDogtyWi2/yPiHM5tRYgYKgso8yhSf6w5Md3hjfEfsKwDgB3Iq/cA8iXiBgSYkzmmta9qLNJPmbMcUEz+m2++ebf77rtzOjX5kjKbwj+11RAAVz4nUQzAvlol7zLUZe8k88WoqvqYL/VRB2vQRqMAXLwH4B8OOog+DQhS5BHaJHfJX1GRSvMLzzrwi3HF+GrpoypH6Uh/1hwKaBUAtMjBRkSKIEoaMpaxlYNb9AxHInVIx5As4fq0AVzkOZTvPke4s5u2EDhTbUT+IuZ1dGPalT4JlUBMLtN9EYHy4ejvQtbjJK/1eQcESBes1YG8azrE0HYmr1FUqrvXodSnazoT76mVW3KSNKV24brP6eIR12vk/GNsozcLYKFn/HtAY99lBtujORhCz6Gzaks7tndGd/Vx5CBoAfApA+PJ5SnzI+AFqLTFmd6nY4v+38rbaQC4St8wqkvp2CL3hvb5lndQ1hqAq1V/yvsnMg5dSeQypqSnMAYADPq8pWdLR3+fy0TN832/z0p5+zXlx7Vcr+Wa81/fMq3zJPkJ1MF5Sa/nunRtB3DBj9Y5zuU10byk3/AuyHlN+0yjbT1Pbz8HP5X0d9cxBcTyvJi7KZ9/l1MHl3F6juuTEDIYGeZ2G/JrqYPKMYs8lXfrke92IttJ1yjlQ7/AjgGYTYt6SNcqr31edj1b716SNjW9s3RsrR95aay73an0jvxaTe9wng35dmmV+Xm54ndwIDgQHAgOzIYDsgvz3YfNk+8vbKD8ca7ffd8eWyj25VSkCw4EB4IDwYHgQHAgOLACcECGzSUN4CK6DVsGYtQRLUsArm9+85sd2784sSXbmmuumS5hZGULEyeieLCKVyvfuSfDEKvia1uRsE3dBhtskLJS1Bo34kyrLGrrkiFN9QD8AjAAUlkcwFUzmrpjkHNAO5Ab/BXNKt0Y+K9P2QWccEOa89Ed8u54w1gMsI5ITYDMSuQALgAXivJBWvGp9FzLNS+znNd5PqW6tvY18q4ZEz1SjSJ6aPsd+vSNN96YAIruAFT0AW+HPu3X2vecF2xJygcwVOur6eaAf3xYP/ShD03RMbbffvs5EGQNwFUDKsrZxqs5h38aNxjOiXoDH3MC7DVv3rx0eSiAC6AjvM/Jt7F1x4ucSiUZsdtuu3VbbLFFygqwhSKFed5EM2BsQcrDnytF6CMt2wQSaYxncCC4w1pAA9I5+dZj3/72t7sTTjhhEfARaQEnAVADhJI7g5TXUFmg52pH9cchAC4HnpAv8pP+BYBQBp/a+/x6n74qoLGPT8+Dczn41YZcm0S+8Pw0SHLE5U0pX5ehV1xxRfeZz3xmsWSar50P/twk88ViL7MLcnJzCXCzR+VQMgFn3RHuDk4fs+64IYoCAD+I9PRF5KE7gth+a6+99kppiC7JvJyTA9zY9ghH27T6qN4FqAYnM+VzsBX3eZ8iveDIZSwhI175ylemx0fJdvKk/M67WQC4hvK91n7ulGZOYFsj5n3auEYlEJPrWvAAJ35O2223XQcIBlL/rvV5BwQInFSrA/nVdIih7Uxeo6hUd6/DEJ2J99TKLVnjMjAvV0lnydP4b+efR7n1NA66lJ4gmUGfRmaU5rItt9yy23nnnVNWV111VdrO28FViljnAHzmF0DXEDKf+dIBxmr3lGDEP+kxJOmj/7fydhoArlGyo1TFFrk3tM+3vIOySqd0Wcf1Vv3J+2cOaCFfIqUSMRUCIAhQ0AkdVtu+apGC38/P/X0+p01bJ1N/Y67VgqK8LEojfaB1niRfgTpoF9qIY07StR3ANckc5/KaSL5EU3JyXk+rbT1Pbz8HP0mueFk8SpXkktfdeeLPsV2sohIOHceej5/73IGOK+BSSx2U77TzBIiLDSInANjMd0OICKsA9AH7kqciJnse6CHojrKHaGwMnQudD9OyY83iO6S1fvBMY11yw/k46rymdzjPhny7tMr8UWWMe8GB4EBwIDgwPQ7InhcArjE8VbSBMcnidnAgOBAcCA4EB4IDwYGVigNyhlDpPgbXUcxxoyvG3ZwwCuFUIrQ/DsWcfJV8aRV9nt6NyXLg52mG/HbDiVaw+/M4DzEyQjgEc6OsHMIO4JIjnGfy9FyDBDjhnEgTGOSmXRYHVsB/tuopERExFOlFWxs4gKu2It+3QfOIPoomQNtj3Oc4lNwJpag+pTxKwCHnoxyWPEtUIBxVznuuszqfleU4tjCeiRzApWs61qJd6P7Qo5eZLTxZwZ0TESFw+rjRsLWvkXfNmMg3FAZTCMfy0UcfnaLEYQRmpTptra0teT9RsDjCV628nnXfE2AmFdL+1QA0lqR6ioGbLTJZrVwycPNgDcDl2wD6CzxCmoBY4nsOYPDn2BqJ/grpOb+fn3s0gZoBGKCpto0DTKuta+VUwjGgc+XvzmpFY9M9P0oOMpYY8/4cW7rgeBlHvsVKTW6Sh8avZBptxVZRyIyccD4AcGFukpOGNENlQZ5v/lv9sdb/3EGO8R9HPgSoBqdhTmyTRv0Yax5dSemG9FV3FJeiNChP+gOAanfYTCJflO+kRzlahgC4SnM55VA7uQx12TvJfDGqnpLdo+ogUDd9X1F4as5S+jxjjrHgDmoHdJxzzjnd2WefnYolAK7KWBtfGltEXWS8Q619VO/Kj+7wB1jI3OtAfpdNW221VbfTTjulLKQn5fnx2/NU5KRZALiG8r3WfkSsAVTAfSfahTkWMA5jX9uckkZ6e20LxZJ+ynM+N2j+qvV5BwQIyFOrA3lrLnOZwXXI22RcO9/1RP1/qe5ehyE6E28plXsSnaVe8kWj4NR0WV8oovleoEQfi/l76Ee0E4QeRr0gPavvEqJQAj5E/hApBh0dkt4u8GSfb7D04IJ/Q/T/SXg7DQBX7RtGdSkdW+Te0D7f8o4agKtVf/LxXeqfT3va01JUN3ikBR3OL69zH3uCv88BQNPWyaQ31ECT1EH2C825k8yTAnX4fOx84lz6tYOVJpnjXF7zLZzbWJzX02pbz9Pbz8FPpW8F10NVfwdE1yIns0CB+QOaFoCLPJGVzOduR2qpQyrYgn/TzlPfVMpfx1E6vNKMOxIlHRDvRhttNAea5xnNF5PIa5+XS3r4krSp1fgwSf3IU2PdvyNq7/LrJb2D++N4RprSt0urzCe/oOBAcCA4EByYPQcCwNWTxwHg6smoSBYcCA4EB4IDwYHgwErFAUV6oNI4RnAYjSMc8muttVZKdvzxx8855GUA5UYf423pPTK0jnJu6jmPOFAz+Cltn6MbTojY4QAenndjU6l+MrLJ8IUDlPoMIbZ7oy7TLgtRtWg3SKvzS+Xy955//vndmWee2TmAq2SEIx/f6k3RPohaRP+CSgbjdKPHP6KT4dCGiM5zzDHHFJ+SAcsNaV4fd8iTAVGIMNIDRpKz2jMGgAF4CXBNDuACiMYzeg5AHCCLaZCXudQPeYdAAKrrJH2N/GrGRO5pXKtfC8iBw4+tfCgLJIDJPvvsk36zrSTgtln3PRkz00sX/EN2YIyHiKpGu/clQKQ4QUvbTABIok8oMowc4OTtjowaoK8E4JK8E29L5XRHkhy6pXS65k762nglrd7tTi05lUrjlUg4bNUmB5felx/lYJcMh584lyFF08mfyX+rz+XXa78dNAc4DSc1YCgi8ZQIuYZ8Ew2RBXqmdlR/rAG49t57744tXyEHcPH7yU9+crfNNtt0q6yyCj8XI3eStfTVNdZYYy4SZK2f8lLxX2CMSeXLYhVpvKA+q75Vy6aPDFU7SYaSlz83yXxRKxfXJT9r/YM0isrkY81ljPcD0rtuwhgGgKn5kDxwojKPQQKHpR89/uXg0iF9dFz2HkVPWzU6GMTll0fyGxX10h2vfQFcj3vc47oXvehFqbilbS1r9RjC91Hth37BtqXIIc1d/k76+2GHHZYAblyXjB0K4PLt8TR/1fq8AwImBXANaWevd+m8VHevQ1+dSXmXdJ9JdBblWzp6H6jJ3xKAS3JP8nhc3g7gkizhGYCCgPvQzbVVnGQ9v5GJAoM6mKH0Pr82RP+fhLd9AFzSYXyLvz7fMF6f0vlQudfS54e+owbgUpuW6lG6Jv1pXP90ANcpp5yy2Bbj0wJwUcZp6mQaPyW9VvzQtneacyeZJwXqyOdOvYuj+qkATFybZI4ryWvyFM2ibT1P10l8Di7ZSkoALv/OqcnwWQC44M9+++3Xrb/++imyoSJ8tdRBvJ52nrItef6c9wFwIZu1oIWIWqXojcqXb0e+32XXYN5nrmi1HY2bl11/KvUT1Vvfx7P4DplkPoJvGuv+HSF+jjqW9A7Sj+MZaUrfLq0yn/yCggPBgeBAcGD2HAgAV08eB4CrJ6MiWXAgOBAcCA4EB4IDKxUH3JmsiDmjGIBDHKeYDDw4BLQFkBsQSsaYUfnqnpyb/K5tA6O0clDyW8An3Ws5jjOcDDU2UQZtVwWPPvKRj4wtFtsyAg6adlk8kpKi1ZQKo9X53AOcB9irr/PD24MVwPPnz5+LHCQwT+md464BGgH4AeGYluE7f06h9d2Q5nzMHfJ6HsAHq09x4hJhjRWZIhmlHcAFkAeHwBOf+MTuKU95SkqKIxVjZ0uEMb1LRy9zzZCdA7h4trWv8WzNmMg9dwzgRMbBCGFc/dnPfjYH9sABiEGZ/gIfcC5Bs+57MmbieGE7MMohgzPX4BXX+pA7bTB0sw0J22vedNNNCQDhoEQ5wMnXHRk1x2wJwCW+j1qp71sjDAVw1bZI82iH7gTQ2Co5utyxXFpVL/5KhgvIyVwwb+EWkAJUKG3t6LKEMjHmRhEgNMZ9TmzLSDQHnDO0neYt0uGwkzFFz/WRBUpbO6o/1gA69E2cBhAyy6OBKU9kEAA0HCqPfOQjF4nOo61oW/qqO87ktNU7/SiHpwMGJpEvnvck5yrX0gBwqdyT9hHJ7lHO3RJY0mWM5iWVif6NExJCZjFv8B76uwMZuL/LLrsk/YJzQLc5UJ3rTvRjtlHMqU8fzZ/Jf1M+5hFAS+KHImkiu9Ev5XD0iKujInAhmwBlQ4pENS4Cl0crGwLgGsL3Ue3nfAG4sMkmm3SArXyrZtdrSiAmX9DAvAKIJycAIs997nPT5dNPPz21v+sbriP5vD8pgGtIO+dlzn+X6u51GKIzkbfmYJd1k+gseXn9t/eBmp7g86zme80poyJw+daIbG/MNseQ6+/wBqAg443IrkQr0zyCTGUbYtod4nsFPaAv+Zw9Sv+fhLfjAFw+v7nccx44KLRv3TxdX7k3SZ/v+44agMvbYoj+NK5/LkkAl3g+6XxLPtIbbr311rmIc8pfR8kV6RaTzJMCdWhO0zv8KF3bAVyTzHGMW/o5JHnt75tF23qerpO0gJ88AlctiiJzI99R0LQicJEX234CPoQUJbilDimDhf+mmSf6DOMgJ4BNbLc+ith+kXELKar6qPT+vXbiiSd29E9F4R5qOxo3Ly9Jm1qtzpPMR+Spse76We1dfr2kd3B/HM9II33A39kq88kvKDgQHAgOBAdmzwHZHLGr8B3GdwLAdP441+++JbnbgvCZi2/Q3ffpZTgdE3NQcCA4EBwIDgQHggPBgeDAohzwyAM4zHAcYhSqkW+xhyKKE1w0DQDX2972thTtiDzHOdJkACHtOLAXacbROMNJi7FJW1HBW5wxJYAPTnr+IJwLKPazKIuM2DVgAe93IxBlZwujvs4PLzNRxAA4sfoTB6xWtfKOFlLZPeKE54MhGaMWHz9u1PIyyTmJYVXAKyJI5c5pnKc4QyHl5QAuRSbjvraI5Ly2XR33hpCXeYgzsrWvUTaNJXdiqszrrrtut//++6efALbgn5wcXBTAiGvIDqIHAUSkPURqv1n0PRkzPeoE5aXckEeYUnlqR203xHjFMJsbx32Lk2kAuLRdEeUpbXui63Lky6FbKz/XkSW0CVQDPHofv/TSS7vPfvazKb2cSiUA1/Oe97y01QYJTzrppO7KK69Mz/g/wEaAJSD6CuCMPfbYI8kCrtWAF+q7cnjJqcwzbNvJ9p05AewkWhVEWXAubLfddmn+oC+wFa0TxhJ4rOhqbAGEk3eoLPA8S+fqj6pLnsbnSQG4aI911lkngaEBaOXkQOvrr7++O+qoo+a2xhraVwXEKo113ouzCBAM5GnURkPnspTRlP5Jjrj8KWXdR4aqnSTjycefm2S+KJVJ13CuEpEOPhKhUgB43eeoyAOk0XiqOUv1nOpDv2OO2n777dMtgBwAOkQeGaY2ZzGPsjUuTkPGCEDQlj6qd446uqOUfq25xgEYPO/OW7ZCPfbYYxfLlnILEOZ6hwO42GKVNndSBA6ujdM7/TnO+/K91H7wl6grEGB5xrYT0R6RWRh3IS2YENjA9SEHcKnvel6cezQmwNgAk0t9nrQlQECpDqSFRukQ3O/bzqQdRaW6ex2G6Ey8p1ZuyZqhOsuosjv/hgC4PDoQ8iPvv7zTASdf/epXU3RUrtN3AFUzNqS/cZ16M0+6LsDvhz3sYYvod6TtQ94G4/T/Vt4y30uulUAefHOgb0AuP/p+w5TqOYnc69vnW99RA3C16k/j+ueSAHC1fJ+V2s2vqb/VFkp4BLnbbrstLUppnSd5r0AdNR2QNNK1HcA1yRxXkte8RzSLtvU8JwVwrbnmmnOLg2rgLI/OWUuj+g49qs307epjt7Qg0cGi3ob+3lnk6fn3OWdxmqJijwIwKi/XlfTtpfEzdC70OaE0Ly9Jm5rqVzq21o+81Mb+HVF6R36tpneM4xn5SOf0d7bK/Lxc8Ts4EBwIDgQHZsOBAHD15GsAuHoyKpIFB4IDwYHgQHAgOLDScUBGYCqOcomDNwcucA+w1wtf+MLkCOA3Tn+c/yJ3TJcMXko36ugRoDC24qwplcUdUaOMpKPeld8bZzhpMTZ5tJVzzz23+9KXvrTIawEVECkDQyjgLlaBcpxFWWRoogBHHnlkh8HViTD7APhw9DhP+zo/qAvOUz2v7cuICPLJT37SXzX4XOABHswd0lzzfuNGLeejnJpsXwYgAqpFKBKQRxEPHMDlq1jd4I6znbHz05/+NOXd+s/LXDJ6kq+iuHhdW/sa+dWMidyDFFXprl+Lbt3gIFDdBxiHE1E0y74nY6YDuDCuI4/ok1Bfp7zqWQKp0K9x/ms7wGkAuNzoWnJIOCCKegwFcAF0YezkMtSBshivb775ZrKfcyqVAFzezkSN0qrs9ODCfx5lTFERfQtUwJKA1pxKTtctt9yy23nnnVMyAcH8Gc59ux5tYfqOd7wjAbSQofCKdnRyAB5gMsozVBZ4fqVzABc1wN0WW2wxB9jgWQG42L4N/kI49gDeOeFgBCwKaRV8a1/lnbKNHHHEER2RuJwEFuCaA7gmkS+e/yTncrQsSQBXy3wxqo4CvJLGwZN6xoGSQwBcDuBh3kIGlhzWPpcBcgIUlI8TB+pffvnlHdtltfRR1WnU0edQ9A7pDYqEoWc9+ivjG/CbonMpDWBMtmKCPMLc85///A6HPESUIbaHEyHPkevId6jvXKHn+/K95Ox2cE3NuerjTpFVSiAmLwd6PDIePonoD8gQ3unt7vqGdCSeeXEhokupDsp/nA7Rt52VX+1YqrvXYYjOxDtq5W7VWWrl5rrzbwiAywG8Go/+HtqUeZ55AtK8ojS+MINrLj8BEcID9X/uow8ga4fQEP2/lbe+mCCPGA0PyPe+971vKva0AFyTyL2+fb71Hf7t7t/crfrTuP65JABc055v6QzSGzjP5xWu7bXXXiniKefadr11niQP9W//jua6UwnANckcV5LX/r5ZtK3nOSmAy/NCPrGIwOd3Fl6gmyoSVel7yes79PyZz3xmxx+6EHYQxiRRMCEfW8q3D4BrFnnq/X2PXk7qRjT0a6+9tvg4spO+iyyFBBhXf+baENvRuHl5SdrUKHuNWutHfnrWbTG19/j1mt4xjmfkIZuHv7NV5nuZ4jw4EBwIDgQHZseBAHD15K2MlD2TR7LgQHAgOBAcCA4EB4IDKw0H2K4BI4TADhjPcBKwqhBHMkZoVvFhWBWVnOrTAHChs+GElwEJBySAGSJIYMwjugxbYrFKXHTBBRekyFX6zdHLAqiI8o6jcYaTFmMTjnyMjjhHMJ4BegP4ARFd4cADD+wATkFEmWHFIzSLsjgQizbmXQJxUQaMdXKe+jYj/pxfTwXN/vn2L7oF2OiWW27Rz6a28SgBOCUBhOHAgTBc7bTTTnMOKDdqOR/lnGT7OABatAlbs8EHQBEQxmFW+SuyD/0fQ7sb8x3AxTO+UlfbxnEdmkU/JN8SgKu1r5GfnKL0UQyLtJc7fx0sQ3ofc+644x6EAZzVuiLvQ9PuezJmOoCL93rbUy/kSh5tTeXTUQAgfiMDGa+Ul9Xhe+65Z0fUJxERU9hmFHLnQ80x6+AmAbGQd8heOU/h61lnnZV4P2/evLQVpBwWvEfPcV4jj8BFGgwGRKvBKcv4PuCAAzryhjyCC7/lVCoBuLjv/GHMEDEH/iCvAVwpmlXuuHJQEM8dd9xxHSCTNdZYI9WRMQmdeuqp3WWXXZbOfeywjSVRv2hH+hsO7Q033DClI+obwAbIwTHIU5wV6sdsp4i8lYMXIBmyYqgsSC8a8c+d7fTJY445JvEIoBoRBNTWZCFHO9sl4kSE6KM4HAFPQczPOOeYhyFAwICBvS2G9FWXpbQdEb9whq233nrd1ltvnSLspRct+OcArknki5e173ysMvhRjljKzViokY/9GqBDcmMW80WtXFwnQiH8UD/AYQzYFSDRk570pBQRR8/T3/tG4OJ5jQM97zJK1zg6AB7gEMBjxiPENk4AuCgf70c+AQBt6aMpwx7/BJhWUuZlbWmkaxwd6IBcY65izoXQURl74iuAFiKIQg4+RTYdfvjhaY7jOuBvAV9IOxTA1ZfvPke4s9sBlWeeeWaKnkY5IGQ5dUK+OhhP87XLbwdw8SwAFsY2R6LqIfuQJZBHEfWxIh2JNMgc5m1IwLFaHUijMtFnSjoEafq2M2lrpPd43b0OtfFe0pl4h/LLy92qs9TKzXXnX01PcB1a8z3tRn9W3z777LO7c845J70KBz0gPyIXQaW52yPXkEZRZjiHXD7zW9trcj6EvOx6Ltf/ud7KW+Z+gMsQ7UVbA8ZkS2K+Afy7cFoArknlXp8+3/oO//YgIhkLUrTddIv+NK5/LgkAV8v3WeoQI/5JbyAJ+iB6OzYF+tD8+fPntpjjNyBXonBBLfMkzwnUkevB3BNJ186jN7XOcSV5rXdxnEXbep4+p3m/HAJ+IlIU8ziEXv+pT30qySrGB9/kmr+4nwO41MY13YFnRpH3O3QyvhcmBXDNIs9Rdajd8+8i0rBwgO8s2cWw/2BXg8eyvTmIt1Vej5uXl6RNrcYbrrfWj2c11v07guvjqKZ3jOMZ+Za+XbjeIvN5Lig4EBwIDgQHZs+BAHD15HEAuHoyKpIFB4IDwYHgQHAgOLBScgDnMpEf5CAYxQSc0kQ8wWDh5MaDktHO044633zzzbvdd999VJK5ewBNcBDk5EZzDP59oiKNM5y0GJsol0eH4jdGYoyMGD9FOE8BOWjF6azKkgNxMGZTHoH3KE8eXceNW+MAXDl4JAc0kX9L2/AcALPVV1+d00SUG1Kf5TfnbkhzPrpzMnd24hzleQyunh+Gsl/+8pcjAVw40Oj74iHbxxB9CGqpq5d5qDOypa9RTje489udo/zGqY8TUJQ75QDDAEiEatsszKrvyZiZA7goC0Ayga5oR9pjFAFgwaEjyvsY4xNAlfqIIkq5I6PmmC0BuHgPjkciQ4l4J+NS/Un9mvty6Cpt6ZiPQaXxfLjGb8A08E0kp1LJCUwaQET0FdWfawJw6Rr5ArYCdCUCMIb8lHOA69TRf7vDgPtE28CB5vnCf8aoiHch3+WIyFeQc5+xDS8dCKcoVuQzVBbo3bXj+uuv37ElW40ok+okABdpfQyRBucf5fZ5AtnGPAHPW/sq72J+pk3GkQO4SNsqX1w36Dsfl8omJ92SBHBRjmn3Ebb63HbbbUtVTNfURzj2BXDxYA7EcBCTv4w5i77nfSuXbaTPAUVD+6i/c9R5zt9a1E4AqJTbZQB9gfHksiTfGjKfo/OykIfk7VAAF3n14bvPEe7s9ogz5EVZaAvqKjnBdd9+Vs4/n6dzHvIMpL50168uAd4oL++BXN9wHakECKjVgXzG6RCkyctYa2fS1qhUd6/DUJ1pVLlbdJZaubnu/KvpCQ6C8vl+xx13nAOakBft6noC1+g36Gbo8E6+AIHrDuDjd94uRLdj/hlKue5R0v+VZytvARMCmiyR9/VpAbh4zyRyL+dtrc+3vMP7vfghsGWL/jSufy4JABf1yHk27vtMda8dpTf4ffoK5DI2/75tnScF6mgBcLXOcSV57fWdRdt6nj6nuUwt2YLgq+w2DmBjXmd+hwcl8nk6B3Dp24U0o8D9pXx1TbKXeRWb0aQALvKdRZ4qb98juhF8dQAcz5bGANeR264jcK1FXrt8Ks3LS9KmRh1GUUv9yE9j3e1Oo96jez5GuCZdbhzPSCubR/7OFplPfkHBgeBAcCA4MHsOBICrJ48DwNWTUZEsOBAcCA4EB4IDwYGVlgMY+Vmpp5XcOSMwjBH9gygxJcJZqOgmJaNd6ZnaNQwRGHFzg5PS47gAIIBRqEQOnOkb8YPoF7vuumvKrrTNQl9jEwo6TnYnQGnk7U5G3QdAQaQYjw40y7L4dkIqg44l5wIrYlkZC/VZma/Q8KT3SAH8hlrahucwtFMOnEQ5sfKcVaREkXGjlvPRnZM4a/fff/+5qDZ5fhjeAaGwShti+wYMmlAegYtrWy6IAqYt3zCKsmUNoLyWunqZS/2Q92H8xgjudeU6NLSv8QzAnH333XfOcO7RPrjPOASEAZVWOHufqkV94VlPx2+n1r4nY2YJwMUqarbikJOmxk8vR+480j1W5eMkoA5sCwjJWQA4CDkDlerB9RqAi3sAuHDQqpxcgwCd0bcll9yhe1eKxf+7ExUgAwCB3CFC+yI7r7766kUykKPLAU6LJFjw4xGPeERy1t/73vfObyWwFFts/HhB5LqcaAtkaC7TGS9E02NrMxzQTryLiGEeIUf3cTJQh+uvv16X0pH6E8XAAVuegPREJFOkiqGywPOqnW+22WYJhJy3J3zBcU+EIwinipztyC/4g6wpEdGEiBao6BCkaemryhsAESAwgWFoB5wIRO3ZbbfdknzhnczrTi3yxXWDHPzpeY87B/xFe2nc1dL3kaGSGy5D/blJ5otaufw6QD+i+vl4YFxeeeWV3YMf/OAkk2kTAbhcxriz1PN0YJjApX7fzxlTOJFK+h7lIMobf04tfdSfr50jG5gzRcw13s91nSPlJprUQx/6UL+czuEXkWguuuiixe6RHl6qvysBIEVkD1GMoBYAVx++j2o/wNGMxVxeUB50XUVm5DdElBj4IKcf17QNGGODKI7kKVAa9yFALYcddtgceItrtT7vkWCYQ9FnRtVhnA7Bu4a0M+lLVKq716E2x9d0pnHlHqqzlMqsa86/mp4ghz/P5PP9KNmLrGb+JnJNicQ37uUymO8t9GEoB+2miwP+jdP/PasW3jJ+WcwhYL7yQ3cgYipbqKIbO4Br6DeM8tRxErnXt8+3vIN6wgtt2Ux5+faUk2io/jSufzqA68QTT1wEpM+7HQTRxwbg7/M5bdo6mfRagP7owjkAkHmDhTcAuHJqmScF6HX5nOcrwJEDmJSmZY4ryWvlx9F5XZI9LW3reXr7IUuQKVCpH9QAXKQnT6JO8i0vewntgz7ziU98Yg6c5VvY85z4Wfo+5X4fArCFTgShq2uRw9A6+Ltmkafn3/ecMcV3JIs1S3oG+aA7sL0iW2aX5pGh8nrcvLwkbWp9+DS0fuTZCuCq6R3jeMY7S98uXIeGyvy7nor/wYHgQHAgODBrDkg3Z6EzczJzMXoOf5zrd99y3G2BsfGupQh9n1hO0gWAazlpqChmcCA4EBwIDgQHggNLnQM4FDEusG0YCiZRrnB6azuaJVlAnItslYWBGsM9RjyAGmybgWNrFOH4x3iGURnD1NImyr/OOuskwyYrV3Hc46y9/fbbl3jR2MYJA+laa62VQBM//OEP05aZOLMnJRm0MLriXMhBGeQ/Sdvg6GDrIwCHrJIFZNVabpxAbB2Ac5cPKLaTwsjP6l7KPw2apK6t72/tazKa/3/2zgRu06n847dSoj6hkkEyNGWZIiMMKkurrEVpUWlDkVFSIU2rFm2UPZElSyUxWcLYksmWLYNMQ4Qo26dC63++h9/zv+bMOfdz3+d5nvd93neu6/N53+d57uXc5/zOOde5znX97nPAgEDeIGSQba9f+WXLKYhAtA8CExCMIDVICE5AVqUPz507V4d7+iRoQ6ARByw6AXKVyD1tErYELm23RxBxvfXWCyuAkF8CON30Z90z6SsEN8CBYBjthUAUn92E/otOZ35O/0X3pIIFSgeHBmMRhBfGAbAhyMAqYTlh3AIH7qGecJYwjkGe+vOf/5y8rd+6gHxPnIcRYxDPZwxtUp+rrLJKR78RjCLfc+bM6WzzGme+17ZKMA3CEHWnNsGWetRxvCqanl2qX3T/WP3sdxsBB9oqdg71rFVv9t1331An6BzG0EEK7Yd+Qh5opwQuIeHWSds2WpcW52hPkJ3pM/RvbIhuQr4Zr9iGlX6FnqafdLP10IWUl2dxPW18GAS7e8qUKRXYQjJg20q2umMcqNOPdXknLVZQpW2h80vTqXtGfK7Ohiip5zj9Qf2uy/cw2Sy0DcYU/iBRM56zXfBozCNSddHE/rf3lWLLuA6RnnkAZC3sj2593z635HuJ3mvb5kuegc3ICrjM0SH9W0HPtbWf7P2j+b1f460IXIxtEH3QtdjDjAXYoE10bMk42St2pWNcr88dlvvl92H8QibOs6dFpkq9HDYs+R7mfDDPZOxAZ9C+0JmQ5fGr0Q80B8iVoVRf59Lr5fgg5iEjXb46u6MUm7Gs80vL7Pc5Ao6AIzDsCDiBq2ENOYGrIVB+mSPgCDgCjoAj4Ag4AuMAAQhSvHFOQJRt1FxGBgG2OFTAGcLC4YcfvsCDF6a6WZjKukBF+4FRQSBF4BqVjPhDhxIBVjdYYYUVArGWVePioDfkbVaLQ2bNmhXexh/KgozRTLHSEivpIKxiQlDZCkEptomD2JVaAc1eO16+s1oU2/QirKJz2WWXjZeieTkMAl7PBoxx+LWJ/T8Oi11bJG/ztfCMyMmYwDUiD/WHtEaAVTixeyATHXPMMQvc//a3v71idVvkkEMOyb7UsMCNfsARcAQcAUfAEXAEHIFRRMAJXA3BdwJXQ6D8MkfAEXAEHAFHwBFwBMYBAmxNhf3HFkCsQuMyOATY8oA3/lgtBvKWtjaKt2lRDhamulmYyqr69c/RRcAJXKOL/7A/fZdddgkrF5HPeBsaVkcjiIZORyAS5VYsCxf4v9YIEIAkEImw4iPjpFY9YNWz/fbbL2wHzPlzzz03bEPM9/EmrPwGwZ4VIaZPnx5W4RqJFcfGG47DXh6v52Gvod7y19b+7+1pY+Nub/PDVU9O4Bqu+sjlhq2TtaU0L3/xEpiEVbfZJhJhVe+9995bp/zTEXAEHAFHwBFwBByBoUbACVwNq8cJXA2B8sscAUfAEXAEHAFHwBEYBwjsvvvuYZutmTNnjoPSDHcRRBhhy0GIXAjbcbG6S0oWprpZmMqaqms/NvIIqD/yZG2hOPK58CcOKwKTJk0KK2xJV7P97KOPPhq244JMI7nooouq008/XT/9s08IaFVGPhGIz2wNzdZobAumemEVCsbQfm3l26fs9y2Zd7/73WHLQJWXhM8///xqxowZfXuGJzT6CHg9j34dDDIHsjea2v+DzMuwpO1tflhq4ol8OIFruOojl5utttqqYoVSycMPPxxWiMUuYmVSBD1z1FFHBf+OrvNPR8ARcAQcAUfAEXAEhhkBJ3A1rB0ncDUEyi9zBBwBR8ARcAQcAUfAEXAEWiCgAI5ugQxw4IEHVg888IAO+acj4AiMEAK2PzqBa4RAH2OPYQu/7bffPmxXk8r6JZdcUp122mmpU36sDwhMmDChmjZtWiDNpZIb7+QtygzJYZ111ukUv4703bnIv4w5BLyex1yVtcqwtTe40e1/122tGtAIXOwErhEAuU+P2HHHHYNdYIndShry1g9+8IPqhhtu0CH/dAQcAUfAEXAEHAFHYOgRcAJXwypyAldDoPwyR8ARcAQcAUfAEXAEHAFHoAUCbHnAW7NPfepTqzvuuCM4V9niwMURcARGHgFW8pkyZUp48K233lr99a9/HflM+BOHHgH09cYbb1ytuOKK1VJLLRW285s9e3ZY2YCt7FwGjwDbAq2++urVc57znIqV0G655ZYwfj7yyCODf/goP4F2t+6661a0tdtuu22+7ZJGOWv++D4i4PXcRzCHMCm3/xesFG/zC2IymkcYZ9nW8t57763mzp07mlnxZzdAgNjdJptsUkF0X3zxxYNf4aabbgp2wnhdkbQBLH6JI+AIOAKOgCPgCIxRBJzA1bDinMDVECi/zBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEGiPgBK6GUDmBqyFQfpkj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDRGwAlcDaFyAldDoPwyR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFojIATuBpC5QSuhkD5ZY6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj0BgBJ3A1hMoJXA2B8sscAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR6AxAk7gagiVE7gaAuWXOQKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao5AYwScwNUQKidwNQTKL3MEHAFHwBFwBBwBR8AR6IrASiutVG200UbhupNOOqn63//+1/WekbxgkUUWqbbYYotq0UUXrebMmVPdcMMN4fGrrLJKteaaa4bvM2bMqP7973/3PVvPfe5zq1e96lUh3VmzZlX33ntv359RlyBl32GHHaqnPOUp1U033VRde+21dZeP23OjXQ8Au/nmm1eLLbZY9Yc//KG6/vrrB471lltuWT372c+ubr755uqaa64Z+PP8AY5AWwSWX375asqUKdXEiROrCRMmVE9/+tOrRx99tHrggQeCrj7rrLOq//73v8lkad/o9Jz861//qu68886Qzt///vcFLltmmWU64xZ9hL86efGLX1xNnjw5XHLppZdWf/3rX+su93OOgCPgCDgCQ4BAbg4wBFlbaLNgx9O2IGA/z507t2huM1Jt4XWve12FjXH//fdX5513XgH4P0YAAEAASURBVNsitr5+vfXWq7CnHnnkkWrmzJmt7x/kDauttlrFH3LGGWdkbbo4D894xjOqN7/5zRV1dtFFF1V33313fMmo/8b3QT0/+OCD1cUXXzxfflZYYYVq0003rZZddtlQhm984xvznfcfjsDCjgB94zWveU2AAb010j6yhR1/L78j4Ag4AqOBgBO4GqLuBK6GQPlljoAj4Ag4Ao6AI+AIOAJdEXj7299eTZ06NVy31157Vf/5z3+63lN3wZJLLllBuIEIhpO+V3na055WHXjggSGZW2+9tTr00EPD95122ql6+ctfHr7vt99+VSrI3+uzIW9tt912IZmf/vSnFYF/BCIPzl3kT3/6U/X444+H7/3+Z8v+u9/9rjrqqKP6/YgxkV6uHkYy89/5znfC4yBwHXzwwQN/9Le//e0QNLjllluqww47bODP8wc4Am0Q2HrrravNNtus9hb04sknn1z99re/XeA69acFTiQOzJ49u/rhD39YPfbYY52zBA222mqr8BtS79FHH905l/oybdq0auWVVw6njj/++Orqq69OXebHHAFHwBEYGAKQXSHkQ3J96KGHBvYcmzDk2iWWWKL6xz/+MdQB1tzcwdrBdg5gy7gwfh+NtiScP/ShD3UI0TrW9BMCF2Ow5nVt5jYj1Ra+8pWvVIsvvnj1t7/9rfrMZz7TtGjF133hC18IL2xgM33qU58qTmcQN37kIx+pXvKSl4Sk99577wpyfROB5LfbbruFS3/84x9Xl112WZPb+n5N3Xxd9cyLB/vss0/n2RDWdtlllzAH08E999xTX/3TEVioEMjZENY386Mf/ai64oorBo5Lzk4Y+IP9AY6AI+AIOAIBASdwNWwITuBqCJRf5gg4Ao6AI+AIOAKOgCPQFYF+E7g+/vGPVy984QsDgetjH/tY1+d3uyDnsB9NAtcb3/jGij/kmGOOqa677rpuxSg6b8veJshR9LAhvsk6CS2RbiSzLMKJE7hGEnV/1rAhwCpbe+yxR/WCF7ygkzXIupADCHY+85nPDH+suiA59thjF1g9UP1J13T7ZMWsL37xi53LnMDVgcK/OAKOwBhAgBcb9t9//5BTSK2QUkdCIMpgSz788MPV9OnTR+KRRc/IzR2sHewEriegHa22pIp1ApeQ6M+nE7j6g2Mqlbr5eo7A9YEPfKB62cteFpJjdW/s289+9rOp5P2YIzDuEcjZENY3M1IErpydMO4rwQvoCDgCjsCQIOAEroYV4QSuhkD5ZY6AI+AIOAKOgCPgCDgCXREYdgIXRADy+NSnPjVsk3XVVVeFMo0EgYvl4dlKA2ELiLvuuit8tw7hH/zgBwPbUs8GrhZmAleuHkJljNA/EU6cwDVCgPtjhhKBOHB75ZVXVjju7da7z3nOcyoCYFqlkFUdCVBCIJCoPxEY+/znP6/D4ZOVLyZNmlSttdZanSAaJ9gq9/zzzw/XOIErwOD/HAFHYIwgMFqkm1zwddhgywVmc3OAYcv/SOZntNqSysj2eKzqFss222wTxm2OsxrLOeecE18SyDDYBCUrcI1UWxCxx1fgquazw7D1cttixxU9LCtw1c3Xmd8zv+QFgbPPPrtTBIiuSy+9dLBrP/nJTzZedayTgH9xBMYRAjkbwglc46iSvSiOgCPgCDREwAlcDYFyAldDoPwyR8ARcAQcAUfAEXAEHIGuCAw7gStXgJEgcOWeXecQzt1TctwJXCWoDeYeEU6cwDUYfD3V4UeAoDHbCWl1LVaQSW2PSEm4hmu5B5k1a1bYTjH8mPdP/Ymtb9kCNyd2fPrjH/9Yfetb3wqXOoErh5gfdwQcgWFEYLRIN7ng67BhlCNwDVs+hyE/o9WWupX9bW97W7XhhhuGyy655JLqtNNOS94y7HMbJ3Alq63VwbFA4MoVSCuisV0k20a6OAILMwI5G8IJXAtzq/CyOwKOwMKKgBO4Gta8E7gaAuWXOQKOgCPgCDgCjsBQILDGGmtUa665ZvXoo49WP//5z6uVV1652mCDDapVV1015O/uu++uzjvvvApihJUJEyZUb3rTm8KhM844o1p00UXDSkzLLLNM9Zvf/KbimIRnvPrVr66WW265itU7WNXj3nvvrc4888zqT3/6ky5b4HPttdeuNtpoo3Af6T/22GNhlaUf//jH1UMPPbTA9RyYOHFicFCzTSBvIP/5z3+ubrzxxpAn7k8Jq4lsscUWIZi92GKLVRi+5I/VRG6//fYFbiEvb37zmyscoNh+vPH6l7/8pbrtttvCKiR2tZMFbs4cYDuAl7/85WHrq8cff7y66aabwhvSr3/966upU6eGu/baa6+KN6OtLLnkkhXBcsq91FJLVY888khYCYuAPHmSrLbaagGXl7zkJRVvZyPXX399eHOVt3ZtulOmTKk23njj8IYrjnzaBnhfcMEFFStNWXnKU55SERRArr322vBsvtcRuPqF3/LLLx/aFc/jTXK2UiAvtDPaIXLPPfdU999/f3XWWWeFOg0Ha/61yVsc5DjllFNC36EuWeXmgQceCO2HlWlyba9p31D9kfVf/epXFVvVWHnPe94T+iAO7eOPP96eClhstdVW4RhtGqIF0qas4YbMv7ge1Dd5259AFn1o5syZFc5EsEF30MbJByun3XnnncmUaadvectbQtumndNG6ZsEni677LL57hHhxBK40B3oMfojW2nGQt7II0L7QddZoW+TBn2G76zwBiHmhhtuCEQVSDC33HJLddhhh9nbwvdXvOIV4c14rXJEGelv11xzzXyrIS1wY3RgxRVXDKvMUfaTTz459PFNN9006Gn0G2375ptvrn75y19Gdz7xk5XxeIud1ZKe/exnB+IO+pd+8Ytf/GKBPqHngRnbYdJGqDfGCPQc25GCP3qGc5AlwUft/fe//31IN7cKAHqKAGIb/ZwsWM1BxjAwQlL1yvF3v/vdYdsq6kSrBnIcaTsecA968LWvfW0YE9A99EOwIO147OT6zTffPOgpxr9zzz232nLLLSv0LuNPHWGKe3PClrgrrbRSOE1b/cY3vpG7NBxfZZVVwnaL/Ii371J/6kbgoi9/+tOfDunZa0eCwNVr/6ZdM7YyblJ/6C3q42c/+1lFWVJC/8BWeP7znx9smX/+859BJ1199dXVxRdfnF39ApywgWib9GVskssvvzyMWbRV+hvjazxOtG1XqTw3OSZdPXfu3KCTsUXABt1OW/r1r38d9B5pYfe88pWvDG2Nfs55dAL2T0ralmGQeaFvUn/oZfQh/RQdyphKHcZS108ZB9DBYABZMmX3dRu34+cx3tEe77vvvmBLxufRw4wt2DrYbXxKSsbztuODnsUn7Xny5MlB9zOeYPdjt2K3024Y39GvsYzEGKBnNtXlEFFZ0UXzHvr/nDlzgk3LmC3BDsCWoh+zHS11z7XoelaIifUGWPCH3cK8he9veMMbgj3BWPqsZz0r2ETYEqSFXmDugb1aOo5h19iVaWnj6MqXvvSlIc/MiViZEf3TRNSGc3MH2n1qDrD++uuH9oG9ge1CvaNz0R/oWlaAos2QP+WZMjNHBANsrbo89svGaoIB1zR9XtO21Ma+bZrHbteVEriazm3Q9am2oHw17Y+6PvdpCVysELreeut17G3aG7qH8TS26ZVe2zFJhCHmLZ/61KdCMmqz/GAe07afoeff+ta3BjsQXcLYj97HZqRfNBV0CnqLvGG7xMJ22vRF7D3KjT1K32O77d122y1cjm6K51RtMdK43XS+h+7rNl9Hj4Ez9YiNsc466wT9wSfzb3QPWFHnrMaGjkGwqVO+JfQ3mCPod2y2JtJW70v34QM4/fTTq9VXXz3YU9iPjLnYTNSzfcECvdFPe6KX9vne97435BObiPnlu971roAtPqWDDz64Axl1sO2224ZzzAERVkubPXt2mAd2Loy+lOi+pr4u+6iSPlaio9r2FfJYYq/ZsvF9k002Cf0av0rKhmC83W677cJt2Iv0Cfl35R/FZtPKyXH6/G7qD+xmJ1gfY+o5fswRcAQcAUegPwg4gashjhh3Lo6AI+AIOAKOgCPgCIwVBAhk4wxDcOrhCI0FJ9mFF144HymLAB8kHYQgOUEJHAgIDoEjjzwyfMfxs+6664bv8T/SxUF/6qmnzneKdN7//vfPtz2TvYD7jj322EAmsMchVUE8SgmBVhxPOM6s4KjCAZoSnoMjkPJJCMRCpMJxlRIC4d/+9rezBLPUPZSVwFwsPJ9ABoQkJCZwUQfUHw7BWLj3Jz/5Sccpu/XWW1ebbbZZfFn4vf/++4cAEz94m1Wkk9TF4GeJAZbEBKno0EMPDbfRNsgfAiFBga1+4medU5BNcMiynUJK7PZeqfMca5s3W3YcY9yfqgsmUl/60peCg90+u03fIHi04447htsJFh5yyCGdpCBtQN6QfPOb35yPFIUDD6wQ+iX9s21ZlXbqM66HSy+9NFymIAtBCYITkHZiIWj5ve99bwGSC8GGXXfdNQQZ4nv4TdD9y1/+cueUCCeWwGUJLXvuuWfnWn3Bqb7LLruEn/Rx289xtE+bNi04WXW9PglWgx96KiZwERT5yEc+Ehzaut5+4gCn/8REDXuN/W6xJX8QOnEWx2LLrXM4yQk8kaecQKyzZFv7PJz3OG+l15UGuuW73/1u0P+QIGKhL1BGrrNSop/t/U2/MxZBvEBwWjOuxYKOplwE+rRqFNe0HQ+4B1IIbYWAVEoI0kDwsKKA4IMPPhjIcCJecU2qrdp7c9/p99I/X/va1wJJL3etjhMggzRGPmwAT/0JvV1HKCNIxPiB2O2MRoLAVdq/wYgyKeAkLPQJKeb73/9+h4ys4wQoRQzUMftJABU9L8eZztGnIObE/YjzkIAIviDxdr8l7SokVPBPupq8U47nPe95C6TCuMPYAQ6x0NcZWwjcWSkpw6DyAnGU4G2qHsjzHXfcEci4VjfX9VPGIMqHnHjiiYEUE36YfwT7Zb9BIoa0Uydf//rXg762fcle/853vrNjo5M3AsRIyXheMj7wLAJ/9D2VnWMSAunY2gROY1Io14zUGMCz2uhyqzu5VwKh8aCDDgo/IR4xtufaD33giCOOmE9vyA7mHCRrxiYJJCZwigX8sGNLxzH0CVvkIugXiFupPGMzYK9bEmCcF353mzvQX7Tdnp0DgBWkLyQ3t6S9oPc+/OEPd8aucMOT/5h38mKRlX7bWDbt1Pe2z2vSltrat6l8lRwrIXC1mdvYOZFtC+S1TX/sVjaNEbQf2m/KDoUwgB0TvyBSMiZpHLAErl76WbexCOwOP/zwLCHc4mP7GfN3iGQSiNbM/3L9n3aIxASuEoxUJ03ne5B8u83XlSYvke2zzz7BzobkmRLsV4iqCERYbLhYrC5L2eXx9fwu0fuqE9og5PCcX4qXaHhBEsEm1ZjaD3uil/Yp+x97BWIOcwTEzgeYI0MAxI5ICTYMbTj2uZXovja+LuWlpI+V6KiSvlJir6lc9tPal/a4bAg7n4eIR12mBLsd2yWWNv5A27fidKyPMT7nvx0BR8ARcAT6i4D8UIzD+LywAfEd88d3/W761EXmvdE3v0e36Z1Dfp0TuIa8gjx7joAj4Ag4Ao6AIzAfApbApRM4RQlO4WSQ44ZzdksmS+DSfXwSqGAlJq61aXOctyh5Ow+CEMFTORVZiYu3ZSXve9/7wtu0So8gFfnhTVIFyXGMffazn+0QgywZhmfhRMTpt+yyy4Y3upUWxAMCFwhkNQJiCOlBzOA+VtfgWRLIMgQvEQK0ygOr0EAAwDjGKSWsYnKJ0kl9EmhRAJfzpImTjDwoGK/7LIGLFXXASUIwCJwIvFpntogpEID0Nitv/iG8iY9AZiAIwyowrCKDgAcrauAIor5YnUD1RSCKFQKQnMNegSuusQSufuJnnVMQuHDaQvqhfnijGSGISNkgcBHMqpO2ebNlV7pqR9SdSD6ci0lXbfsG6UGKoQ7kzNYzIQewGoaEN5Xt1ihyDEOWYhscpG1ZlXbqM66HmMBl74HARJuiX6p9y9mo66i/L37xi532xnn6Ge0WoovaIW9en3TSSeE2OZwtkamU4IFuwuGo51Cn5JuVIWzf4sExgQudJGII96HzKCf9Wf2O9sh1lKubWGx1LQEa+i7zTt5KlsQkRet8RaeADTqKwIl0GPd+9atf7azElXoe4wH6xZZBz0TXondI16ZJ0JXgq6REP+vetp+lge+S8QA9Q1sVqY66ZRyhbiA3qQ1ZIgDlUUAwLpvto/G5ut+0LRFraR/UfS+i/mQDNqn0IMQoaHb77bd3tl4cZgLXBz/4wUBqoDwEZNEttHECUrbvEjBUH2V8/+hHPxrqkzYP2QfSG32JMVe6LCY4Q4BkdUQJwWbuo9/GxEpL4CptV3pO208FSnWfbCjaFe04FnBhbAUvld0Gt7m+tAyDyAsEeYjyEvQ5upngG2OR+m+bfoqOg5CE2LavZ4AdxBZ0APX+iU98Qqeyn6UErpLxvGR8IOOx7sJmRW/RpqXvuC4mcI3kGNBWl++xxx5hbBdxkbbMmIfNyLjK+IaeV59lngCxhfZv50mx7rV2MH1K+PAdkgYELsZVjoMhcwf0A4Hv0nHMBu6pB4T2R3vnedaGIUCfWp30ibue+N9t7oCd043ApfSwFcBIZdZxPik/88NYN7LVL0EISb9tLKWb+2z7vG5tqcS+zeWt7fESApee0WRuY+dElsDVtj/qmbnP1BhB38H+gjBLPhB+a4VQfpeOSdJ5dowr7WfYhqRHn0cP0OYhmdEuXvSiF3XGIghJEKu6ichCXIdOp38hrLSzww47hO/800razOVj0o0lcJViFNcJz6yb75GPbvN1pak5Lz4KcGfcZsyWnYLuQGeyGhuCfZIih33uc5/rkGbtvCfclPhXqvdtnShZdBhjCmOMxZ9xhTYAyauf9kRp+yS/sv+Vd32iv2m7cRuhbLRhyoW/SOMk/QU7mvpBSnRfW18XzynpYyU6Ksah6fyvxF6jXLFAzoQkrfE0tiFS83nqhBdDqQtr27N6n12Rrq0/sJudADYujoAj4Ag4AoNHwAlcDTHGWHBxBBwBR8ARcAQcAUdgrCBgiSTk2b4RyG8boJUTjeMxgYsgDisw6G07nPA4y3BS4vjlLXYCpRJW5SKgg+B0gDBBsBinHg5zOTchT9mteQiCiVwlkgBOCJz83EOggmfZt24JoBJIRciDVlyBzKJVgSxJi+ts+eSIxvHGc5CYpIUzcfr06R1CBW/G4SipE4sRjkgCKdpyBgcYhC2CixJL4DrggAPCSgics85XfrN8P8vxg0fsyFSZeR4EFysKHlJf1AH1IQEnkX8shjmHvQ1cicDVb/yscwoCl4hDbOvGH2KD4ipL6rMkb7bspAmphralegczMKYebGDP1nubviEiFs/ad999A+mA7zhIbTshHzi+JVptiIAjgbaSsiqt1GeuHuR85x76Je1bW3DiPKS/KNBinek49fWmqPKs50KyoB8g4KztTORw7geByz4fPYI+If8IqxWyEhp1ilgCF2+7b7/99uE4jm5WYlAfopzUkwgiIlaGi2v+WWy5jOfxZrmCNHZ1NYsVQVoc7QhBFFYro89L7NuyNi/x8y6atzUMW4Ag1BlpiuxA26W981zEBgyo56OOOiocL9XP4eaCf6WBb+lGHtlkPOA6G6iJg+EE9hmvRNyjP4g0C44KpFMvkN2oBxGGSLuNQDAicIwwFqO/exH1pxSBi3wT7KC+7cphJ5xwQmc7ymEmcKED6Y+0X4K76kvgZcctVrBk9QSEVdEmTpwYvrNqDTaBhKAV9cmYHY+3dpyG9Gy3OrVEddKyY1Vpu1Ke2n5aXY1exRYgqIjENiKklqOPPrrzCBsUpb1LV5aWYRB5Qf8RbENkN6oAjIfoZhHRKJvI3nX91BK0sGEZl6yOhVQNuRrJrQiiPOhTNhgBUdmaOsdnagWukvG8dHyArKktqAjIUfcKzGHvQHLUmG4JXCM9BpTocvKo1QTj1Vl4+YGtNxG2J2dLQCuQBSBzIrQZxn/E6hN+Q+Tn5RZLSJI+snhxbek4Zsdh0olJiWzbvfPOO3MqtFeCyerr4WDmnzCN5w7UdzcCF/dgD7CCHxLnEdIaq0aqLVmdY1fRHISNlSluOFz6vLq2ZO1La7PxwJx9W5fHNudKCVxN5za5tqC2Q16b2lZ15bJjBGMudiikBITxGCIP5BsE3c98HSkdkzQO2DlH3Iab9jM7P2VrYrsCuSW4QCpnntdNbJksgcvaHtaOJz3bDvhtfQg2vTY2ra2TNvM9i4e1gciX0rS+J46rPmKMLNmSeRt1IiFOB1kKyY2vulafpXrfYojuY5UtvfRG2nbOzvZ1kIT7bU+Utk/yJ/uf79jJrAjGmCj7hvGDcQRhZSfIcxLsKHwVmuuyXS/3IyW6z7Zj205JL+frsm2qaR8r0VG2npv2lRJ7jbLWSc6GiOfzse1icYpfPpEtSv039QeSR+EY2wl1+fdzjoAj4Ag4Av1DwAlcDbF0AldDoPwyR8ARcAQcAUfAERgKBKyjHAcoRIpYbIAOUgKkBktwwlmHQ8oGQu3KUnbFJps225exjRnCNl5s52VXx0i9gWq3i4PYxfZrLOPOMveIDSKHA0/+kxMIpwJBWxyxOHkh0yCWQPLkLSFdHKo8B4KQdYhZEpOuhySGww+BZNEtKGIxih2spMFWNTifRRYRgcu+qRkHU7kPsY6yY489NqyKxvE65woBFJxvBBu5JxZtDQIhhKAPknPY28CVCFz9xs86p3olcJXkzZaddkXwzwbmwMe2OxHmbL236RuWdGMdmaoX8kBbweEmkhNvZ+JkRETOLClrSCDzL1cPcr5zW6pf2v6v7aUgQeA4RCgHAQz6qhX6L+QYRKszyOHcK4HL9jmeT2CWTyt2mwVL4EKHEDyiHtCZBGStkDbtAWlKsrHYMiFXgFnpWrxsAHjSpElBB3AdRBQct1YgwWpFGOvgts+z6eleOxZY4pfOE0iD4IXuU7CkVD8rzbafpYHvtuMBY4fe+E/VDfm2hAdWppReVQCKa1K6n+NtxAaaY0d8m3R0rfqTfnf7jEmjw0zgEqGVwC9jk7VbWKFG26pijxB8Qhg3IWjQJ9SuLSbayoS+Lz1vt71N2Qvcb4lFCl720q5sntp8t7pa+dD9drwAM8pKOSWW5E8gj4BeL2Xod14g3WLnIrm+MW93gEB65xo7hnTrp5bYd9xxx1XXXHMNSQSxpB7ZzTqX+1TQLBdgThG4bP2k2lnKLi0dH2j7+Dupf3S9fVGCMtlVX+z4MdJjQFtdTt7rSDfbbrtthY6l3KQd23mc32STTUgmzEn00om1g1mZERs7llzwtXQcs+2BuRk6Lrah7LwvXrE1zp9+5+YO1g7Wyy7cY4Pb9Av6hxXpYY7R50Vu5rclJNuVVgdhY/G8nJQ+L9eWrL3Wxr7N5a/tcUvcqat3W6e0+aZzG3ufbQsl/bGubHaMwKbCtrLClpyrrrpqOKTxrJcxSeNAjsDVpp/ZF8pSNjSrMDFvgzyPf6Ob2H4mApcd83J6x9aJ5pO9YGTrpOl8j7JZEonqSmVWmk0JXJZwxctwpCex82dWGmLFoW5SqvdtnaR8NHactGThftoTvYwD1v4/+OCDO6vWgxd9HDslfjHNYsk8GN2JqM+U6D7asVYlS+FI+ilfV0kfs/2BvIsQyjMQbBjrDyztK7ZemtprT+Qg/z9nQ9j5fMrnAMlVNom113hSiT+Q+3J2AudcHAFHwBFwBAaPgBO4GmLsBK6GQPlljoAj4Ag4Ao6AIzAUCFhH/tlnn12de+65C+SL1ZwIcCFalcISuKwDSjcTNGCZe5y/BLJskFTXEPDAQYbIwa/7OBY7jjiGrL322oEoQTCOAJKCSqyCgPMgJR/60IeqyZMnh1NaYcESSLh39uzZYQlxu7qGTcs6pTjOW+OQHyBHyFi213f7LocRGEGmiIki3G8deiJwWQIQjrbb521dFYtdDv7qq6+ujj/++HBJW+cKgQjeCqeutAJJKYGr3/hZ51SvBK6SvNlghbYWiOvBblME9rQztfG2fcMGhfTWq31jn3aofqqAsQ348lyCAiVljctlf+fqQc532rUIZfY+67hXAMEG7uJV7nQvb7BC2ATLK664IugYOZxt8B0ihVYIoh/FAnlUZA0FUqxzVTopvs+uXiICF6QlAtoI2yuJqBXfq60TqHsRPeJr7G+LbS7ooKB//Da6TYfv5JHtZVgp5U1velMIxHM8R+CygVOuQ6ZNm1atvPLK4TvPZVsmK2zjx5vkduWmUv1s023zvTTw3XY8sAQdVmpibIzFOsjZPgUCHKKAIO0AvcBnL2Lbcioo0DZt9acm90H4hYBpx69hJnBZ0hTBpauuuiqs3hgHbLqVnWAO+heCDG2BgJbt15bYJP0Wp7nNNttUm266aTis4GUv7SpOv+lv6Wqbf91rV65IBYNZmUjE9VNOOaW6/PLLAx6sVIi07Rv9zoslLQhjlU2frM5Fu0BsIKtbP7Xb+9ixx66m0U0vKw98Spe3IXD1azxvMj6IoFtXJhHKLY4jPQa01eVgb+2r1JyGa6yg29kuCrIF7V8ruEG6SBG4RFK3afA9F3wtHcesDcNWrxClYsF+IiCNWKJNfJ39nZs7WDvYpmVJDPGqhaSrtp6yD5k7Yq8izMuOOOKIYL8MwsYKD0n868Wmy7WlUvs2kb2iQ1YXNiVwtZnb5NpCSX+sK2DdGMF9dvsvjb29jKsaB0RG4Rml/cySd0gHHwZ1AQkt5Sfhmjqx/UwELktgkb8jTsOSnfqBkeok1Z95dmq+Fx+Px2el2ZTABUkI4gm2mK0rnoPtDUEf0Ys/4UfLf030vq0TzcXtY1idSqvkai7P+X7aE6Xtk3zI/rdjOMcRq8PqxknqAaxkU9r7ms7t2faaciBtfF0lfaytjirVJ/2y1wIoT/7L2RDWf4CvhNUsY9Gcvc6m0z3d/IFcl7MTlIZ/OgKOgCPgCAwWAcWk8CUwN8UmYk7DH9/1u2kuFpkXXOjNS9n0SSN8nRO4Rhhwf5wj4Ag4Ao6AI+AI9ISAJXDhJMdZHgvLlL/jHe8IhxXstwQurexj75Pzhi0xWDEnJdZJr60kdB/Xi/CSutceU8CIY7lAOMaqhDcvIUPwfMhlOJ2tkAZvg0PgwLGqbT24hqAkjptYcDCyPQrXK3gTXxP/ltMldjTa69hmku0mERG47PL7ufJyvcpMvti2AunmXMHJxrZsYIMzNCWlBC7S6id+1jnVK4GrJG82WGGDtxYzS3hR/amNt+0bpKt7RZDh7VRWZMNpzlvyIg4pOCOndbxC0EjUg5zvufZtV5JTAGETQ+qMtySwuMbf5XC29VBC4Np8883D1kWkr1UB42fxW6tHiMBltxjlfK5fqk9yTZNtVm0bZytDVmuKRatEpBywrAAFmYa3hRVgju+XTue4fV6q/GzTh45AwDcup5zBap9cV6qfubdESgPfbccDtkdjmzRJjIWOq85tf1dAMN5yT/e0/bRBgdSb1m3TU38ioKitaW0aHIccMGfOnAVWd+E6245yb87b9KyezAUc7fV8L+nf3Md4CrFV9cIxhFU0IL9BhCXYEQt9CDto4rytFAlMxfdzPW1AxEy76hBjvbYatelaorWCl720K5t2m+91utqSm2ywUemnCFy9lKHfedl9990rVpxCsEWtPacy8CndRTvQ6oRN+qnus4FruyIe7Snecs8+134XqaUNgYv7S8fzNuOD1TEpIp/KIZKyDf6O9BjQVpeT9xzpRuWClIe9teaaa4btOHGEpyRH4CJgj26ORfMAixfXlI5jNnCfC9pSFtotYu35cCDzLzd3sHZwjsCllwds0rJlU3YL9RcTuAZlY9k82e+9PC/XlkrtW5uvXr6XELisTW2fbcdszW1ybaGkP9pnxd/rxgiufe1rXxtIXHzX3KKXMUnjgJ3LlPYz7HBWFqaNxIIOgBDP6p/dVvDWvZYsJAIX9QG5HEn1PY7brVT7gVG3OknN98iHJXbJBuI4ojSbEri4h7GbFYYRCJ/YdJDttWp4U30XEpj3r0Tv2zpJ2Rx2LNUcUs/rlz1R2j7Jh+x/Xg7Ud+WPFw548QCRL03n7KetB+aoq622WueFzaZz+1JfV0kfa6ujetEnpfaaxdd+z9kQdh6Wms+ThmzO1Djc1h9Iejk7gXMujoAj4Ag4AoNHwAlcDTF2AldDoPwyR8ARcAQcAUfAERgKBCyBK+fos9uwaVtDS+BKOQZEcIiDErbQ1tkrApfuaxrYJoiaesPcPif+/vOf/7y68MILw+Gll1662mGHHcKKPimCA0G5gw46KDgBlQ7Bude//vUVK/GkxAYxUud1TGW1ZAed06d1EslJblcQ0XV1nxDKtBVDnXPFrlJm0yMozWpjvDWKWAeorUNbbrt1TNyu+oWfdU71g8BF2drkzZb9d7/7XXXUUUeRxHySCnKo3tv2DRK2b4lCPoQswKpK2sJMbYPfOOZwBtNH7CpsymCbsuqe1GeuHuR8zwWjUw59G2CaMWNGdf7556ceucAxOZltsKkbwYOV/NgOEdEKXFYfnnTSSRWrUKVEAWk53y0RI3V96ljcL1LXWGx5ezZFLEkRuCBfkj6kk1ggKFAnehs9R+BKPc8SuFKrmin4IJ3Wq36O897kd5PAt9pLvFpVm/Egpy9zebTBv1RAMHdf0+Nqk5aA0u1e+ijBJMSuAil8VI/d0onP2+CRdFN8jf0toinHcmQnez3fS/q30mAFGsZWtmJNEbHoH6waRJ0hBK3Ygie+lrGRwAcYYj9YApcNPpFXzsVit+5T8LKXdhWn3/R3na62BC6thmPTTRG4eilDv/PCahus2mjrxuZf3/VcS8Rq0k9tWbWdl115k5VWsZ+aiIJpuTHzfe97X1ilg7TIG6vjSNqM5yXjgyW05Owd8iIbRPbNaIwB5KONLuf6HOmGc5AhsOUI5sciHQGpE8kRuHLjfS74WjqOWd2bmpsp/7JDcyss6Tp95uYO1g62cwBLYkjZCiJwpcaYFIFrUDaWyhd/9vK8XFsqtW/jvJX+ts/XSx6ptGyd5vp6am5j77NtgWe07Y+pfOmYdHVOT6YIXFZPK526z272Wi/9TKQgXgaT3ojzcuaZZ1YXXHBBfHiB37aficBl7alU3yMRu3qzCFy9YNStTlLzPfLRbwKX7bfattyuyFanE8mPlVK9b+skhX8dgcvWQS/2RC/tU/a/nU8LF6tD6l62sBgwR+Vllw033DAk03RuL1tCz+72aX1dJX2sjY6y9dQtX5y3+oTfbew1rq+TnA3RxH8gmzMmcOXKhx2d8weSx5ydUJd/P+cIOAKOgCPQPwScwNUQSydwNQTKL3MEHAFHwBFwBByBoUDAEhZw2vDGXSwpJ0A3ApecAnbVkThd60C8/vrrKwKYuq9NEFrbukD6YtvFbsK2X2y/FgtB3XXWWSdsiSKyEtfEjg3dR2ATByyOMrZqw3kt0VaT+p36lNOljqxmt0sUgcsGhqkzsKoTAjSUAck5V6zTEwcNxBS2lmLFDZzkiAgCvRK4QmLz/vWKn22X/SJwtcmbDVa0CXKojbftG+TN9jsINpAPIQ6wMhMrNMnpRhCabTN32mmnUKTcdqScHFQ9lDj07bYekLdw9DaRlMO5G8HDroYgAhdbC0LORFIrC3LcBqRF4KL/8zwEMirkrzqhzzbZss228RShimekCFxqB5xntSTeeKY/z507NziSIf2xAhgySAIX6fdLP5NWE+kW+LbBk5jAZdPvNh6wUiH1gxAMTZHrbHroUbZRRJoQQ+y9Tb4rTa494YQTQn3X3cdKMmxRgsSBUPWnVHC9Lk2dYwUErWJkCTE6H39qbOE4q1NovIqvs79L+re9n+8EecABMidvm6MLJQpAs3oDZC6Rt9i6mdU56VNaJQMy7fLLLz8fSciuwJUrk92eWgSuXtqV8t72s05XgwmBNKQpgauXMvQ7L5ZokFoNQ1ipDdoV7NSn4uCb7uHT6n4CiIcddliHOG23TbX35L7LNoj7o663q4nRJtX+dJ7PJuN5yfhg9aYNlNpn813EIBG4ODbSYwDPtNJNl3NtjnTDOa0qxnfaB4R4dABjPXMJyJ2bbbYZp0eMwGXrw45jNnCf274UAh9tDalbTS1c8OS/3NzB2sHSmdxiA/gpEkNbApftZ/20sWwZ7fdenpdrS6X2rc1XL98t+WKkCVw23036o70+/l43RnBtisDVy5iUGgf61c+WXXbZsCooK2Jhl8vOoBysqqxAHL9TYvuZCFx29aPp06eHbYHje+0WcCJw9YJRtzoZKQIX+PEiCfNijdusfohPB98GtlrTrSpL9b6tk5Tus7pbc0jVj9U7vdgTvbRP2f8pApftW3UrcGFr8YIEgq3OfZD9kaZz+158XeFBT/4r6WPddFQvfcXmrYm9Zq9PfZcv0dpcXNfEfyCb0/o5S/2BPDNnJ3DOxRFwBBwBR2DwCMhuxJeALYRd5FsoJnB3AlcCFD/kCDgCjoAj4Ag4AkOLgCVw5QgLlkSEQwsCjyWSpN5oxPGIExuHGW+epwKy1gHyy1/+sjrrrLOCw5L7kNSWI6wow9ulGKOQzXA0ycnGsyA5pchZOLP4Q8gv5Bmc2QiBGIhKVlidhudrmxScfty/6qqrVhCuIGjFYldHSG01FF9v35JNlZXr5XjkuwhcNviW2/aSFSdERLnyyivD9o6kkXOu2MCgnLlcL8HGJWCIlBK4IMf1Ez/rnOqVwFWSNxu4akPgKu0bYE97JNBM+4dIoK0icOAR1KIc9GmE3yussELYXpG2Iykpq+5NfebqocShb1f7s1t/2ufaVcikj1IOZ0vwSBEodt5557CNCGmLwGWd3rnnsxXErrvuGrIk5zsTZOoFqVvRgu0McSazJQjO725isW1D4FJwFJ1I8CleAcZugzFoAleJfsbpUCroPYh4SGrbSbslsALfEHnajgeMgWzFh+S2qqKfvvWtbw1EIZ5FQB1JBQTDiR7+sZocRCTEklBySdogX7zNofpTKYGL/qDV/3h+jmzCORvsbkL24h6kpH+jL7XtJfaGCHVPpFjNpz8V0LBET/V3Xa9P29/IF0K9s0UdcswxxwTCV/hh/tmVmkTgssGTtu3KJN3qa52uLiFw9VKGfufFkuROPPHECnsoFhs0ZVyl7SJN+6nyTPs97bTTQt1zv8YVvjcRBdMUeI7vsStSqE+VjOe2vbYZH0TEigOFyicBWwK3iL1mJMeAEl2OTW/10G9/+9vqhz/8YSiH3X4LncCKbvEcw9rP/VqBq2QcI8PWhtEcKRTE/LN2olZVNqeTX3NzB2sHD5LANSgbK1nYeQd7eV6uLZXat7k8tj0+GgSu0v5YVzbp2xzR1ZJMNJ/tZUxKjQOl/Yxt2lnph/nZxRdfPF8xaXPM+bVy7qmnnhpesJjvouiHJQuJwGX9OoxHkPViwXYFE6QfGHWrk5EicFEei8nhhx/ema/J3ueabtKL3rfPb0vgIl/Cshd7orR98nzZ/ykCF8QmXk5A2O7z+9//fvhu/zHvEYkOshztskT3sWXx5MmTQ9JtfF1t+xh29kjN/0rsNYtt6nu/CVzWnpFusM/N+QO5Jmcn2Pv9uyPgCDgCjsDgEHACV0NsncDVECi/zBFwBBwBR8ARcASGAgHr6COYyXYvBP0lvCmIMwmHDMcVnOxG4LJkplmzZlUnn3yykgyfEFEI6iyxxBLht4JRNj+pbd9SpAvrbDjvvPPCyjn2YThFWaWGQAOBF60MIiLMfffdVx1wwAH2lvDdpgsJbfvtt+8EyHFwERyxQlmUTo78Ya+3RCwbMNI1NqjIMRG4bEDZBht1H582OH/IIYcsQODiGutYZDUe3v5FUmWzwehSApclGKSe0RY/S27JEbhygfNQUPOvJG82cNWGwFXaN5Rd+1YqxyzpgWAJzjz6q+SOO+6Yb5vRkrIqrdRnrh7khM4FWVIOfVaGILgsfUMwmGC2hLkWwRTO25XrUg5nG6BgNTL0iYQteiB1CScF2iEroJf0fLCG7GnFvllsCR0KinOtCHX2PrvioO1D9pr4u8W2DYFLq8mkSK2UjbKDATJoApfVo031cxwgj3Gp+20JdjExiXGH9kNwBlFAxxIjm44HtFVWN0AIUkDytWMnx+32cnYcTAUEub4Xoe8z/pAvRG/vox9isfqcPNPm7Wo+6k+lBC6eZ/sJ2yjSP+J6xb6gLWo74lTAKM67fpf077XWWqtC/yK51WnUj7VC4jbbbBO2UOQetlSNV9ezAS1rI9kVLu6///7OKlakg9gV0PgtAhfB3dJ2RTolUqerSwhcvZSh33mxW+XSxkVEtzjZ1dLsFttN+6kliTFesS0WbSEev+wzU99ThH1dx/ZDCjJyTDZzyXheOj7wTPk7LVFJeRQ5nd+WwDWSY0CJLkfP5Ug3kOAJQCOp9sNY8rl5L5WgfxGLy07zVkBlroS03UKxZBzjOTZwj+5Ht8cEbvQLfRRhxTRsxG6iwCzX2bmDtYMHSeDiudLNfO+XjUVaOSl9Xq4tldq3ufy1PT4aBK7S/lhXtroxgvtSBK5exqTUOFDaz/CxQNDCFmJOH9uM9uWKuhWOhI8lC4nAZclqVg/rHnwikIX5RETS6AWjbnWSmu/xbLuFYjxfV5q88MJcTKL6EMlex/VpbTKNx5yDlIuvpYn0ovdtnVhdqefWrcDFNf2wJ0rbJ8+X/Z+yx7Ft8Kcxj6QN4zuKVzTjJQm2KEe0WmeJ7iv1dbXtY5deemnnxbhBz/9K7LUAZM2/fhO4Sv2BZDFnJ9Rk3085Ao6AI+AI9BEBJ3A1BFMOjYaX+2WOgCPgCDgCjoAj4AiMKgKWMEVGcLZAFGAVGQL8OJ8I4CE2eNmNwGWJENx79tlnV+eeey5fK5xXBHS0epB1EnGOIDROXwRiBatz4SgigEXgWeQKgl0YqSyNT+BIx3FGQkpAWEnrox/9aAjO8NtuAWSDUWeeeWZ1wQUXcEkQnF8EesmHiCI2IAvZjcAHzlGE8hKswXGI5FYzCyef/GfJKByaOXNm2DKOsk6cODFgpKAQ50Xg4rtdiYGt0VhZAkcwDlny/dKXvpTLqjgAbx2LrE5DEJvt3KxTiWA7W+4pqEUAm2CzxDqEc8EbsIgDV/3Gz5JbLIHLHiegRGA8JuGoLPosyZstexsCV2nfUF633HLLEKDQb7YE1XY4HJPzUufjN7BLyqq0Up8Wb1sPcr63IXCR/jve8Y6KVZIQ+jfOZAKn4IY+EvHI9uWUw9kG7nHif/e73w3bFnKcN3tFHuU5InDxnW3l1N4JEBDs5vnoAvoWukFiCVw2wM7zCOTeeeed4VLePv7gBz/YIdc0ebOeGy22bQhctg1AHEInEsxdeeWVq3e9610VK/RJWIHwuOOOCz+7PW+PPfYIW81xcSowwVvX6Cyrd0r1M8+weo60Iax2E3QgdYagE8EN8t6kSZMqdBmBGYkIXPxuOx5wj10NDsc/K9RI12jLEI1LBJ0USFcAinairSxJD7HtKO7bT1yR/8/b6hCDJfQ96p8VIRnTabv0f8iEktR2JupPth51fdNPayNwD6TFCy+8sGK7ZNoEeWWFKvq1pG6rV12jz5L+bYNIjHu81Q/ZGqHdsuoNfwgEbXCwpC9sAeqY7U/RH/QXApDUMUJ7I4ChoKzqmXNsX3rKKaeE8Xbq1KlhyzXZOZwXgYvvpe2qtO3U6Wrqp+0Wir2UYRB5sfqQ+mZVDvQh+G+77badVdni/qj6i49TPiu0Z+xRKzlyvb0m/s74QntDWKHlyCOPDPlkPGTcVzvjvAhcJeO5xaPN+GBXbgI/VqIlIL766qtXrC4pm578WTtxpMeAEl1uXyBA79EOGf/py/R5Ya9+yjiDnYC9ItIs5T766KMriMNIyg4OJ8w/tXeeQyAW3cIcoHQcs4F7HoMNxSopkLQIvn/4wx8O8wvOxcQIjuUkN3ewdvCgCVxWv7WxsahbdBh1iN7XSzy5sup4L8/Tyzy2LVGvJfatbUfME1lFukRGg8BFPkv6Y1351Gdyc4sUgYv0SsfV1DhQ2s/siubMYY466qjQ38kfq/Pir9ALBhBD41VCuc6K7ZcicHFeeeY7tgxb+9JnGM9p/yJwcl4ELr6XYtStTnIELjvniOfrSjPWUypbjsBFOSz5kt9t+j3XoytK9b6tk9Q8qRuBqx/2RGn7pOyy/61vjuMS6y9ifGHcYtVfhDERO0bjJQRibH+kRPfZOWBTX1dJHyvRUSV9pcReC+DV/FM/iW0I27dy/gOt+mr7kq3fNv5AsmjbvvUx1mTfTzkCjoAj4Aj0EQHGZQQbmbkk4zH+Dv74rt9NH7nIPKfV/y/t0PSuMXCdE7jGQCV5Fh0BR8ARcAQcAUegg0BM4NIJHAFywHAMYxCSlMQGZ1NbKHLd1ltvHYKUuoc0FZjQMd7c420+CBKS173udWHlEv3mPsTmxxIuOAchAwehhHtw2BFckBBYpwx6W9AGZrmGgBTnCHLYZ9m3YHGoQgpDeAYOUYxj+xwcITyH9LqJXd1DaVqMeIbyYglcvJ2P80jnuI6820ASxyAyWNKDdegob6wMQJlw6io9zpF/yoaQlsWTZ+E0kvOOa2zwxgYc7MoD/cTPlsUSh2zAmXwhMYnpiaPz/2+bNxu4akPg4qmlfYN77VvS/IZ4CAFREvdpCCK0Uytty2rvjb/n6kFOxVyQJefQB1cCzJZgRZ+wZAfKQ8Bc5Uo5nHGS4/xVG47zbdu31SfcR4DA9iWu1cTX9klL4CJ9++Yqv6UDbB5yTnGuj8Vim3PAoj/Js3XAEkzfaqutOsmRZ0T9m/4LYUW/CRLRJro9r4TAxXNL9DP32SAMukyEOM7VCTpHRL/4Olt/lsBVMh6AO45/q/9jbHl+TBBWAIr2GxO4bN0RzKd+28iOO+4YSFpN7slt0af+1AuBi+dbQkq3/PAWPHq8qZT271g/QsqiTdDf1R/4TWDjnnvuCfqDYDx2gYSx0JKrqUedp89jE7E90iqrrBICskpX9+vTtkURQzhX2q5K206drrbjqSXNqgx2lTkIapdffnk4VVqGQeQFYj2BJVsPVqeTYeoCIjxBQkldP9U1+rRjKsfarPahNNZYY42KVWZzYtsLekd2s30219Ae6+xS2054FvcgwifWYRofuIaA9MR5Lxh0E0vg4tqRHANKdDl5VNvjO6KVdO24x/G47UAwWHzxxTkVsLzpppsCMSNnB4cLn/xnA54csmSFknEsDtzrWbbtcIzfkKIhCjYRaxvoevKH/iSAj9g5gC1XisSgsT01xjB2kzYS65wSG8tuh0bdMZdqKiXPI+1cWyqxb+3K04wrTbbfTpVvtAhcpf0xVQaLbW5ukSNwlY5JqXGgtJ/RFklP8xn6IX0IfW1tCumeHAY6bvuZJXDFPgKut3aL/W4JXKUYqb3n6iQ337P2hcqk+brStDqRa1Qfds6je/UZ23nXXXdd2Mpa55t8lup9Wycp3QfGsuvjOaTyZcd0jrW1J0rbJ8+S/Z+bq2LrYn/Ec2TsB7Vr0onnFyW6L27H9BfsE/tsjllfV0kfK9FRpX3F1i1572avgWWd2PbGdeovdszO+Q9SBC5ecirxB/KSkH2m8sxYzjjv4gg4Ao6AIzB4BJzA1RBjJ3A1BMovcwQcAUfAEXAEHIGhQMA6uVgl6w1veMN8DhgySZCIFQvYBkjCW3asVIPI2aZz9nODDTYIq2ZZp47O81Yeq1ilJvasrkEg2hIfuA/nO9twQbiIpe5ZBCl40zV+m5VVDQhmKXBl04Q0olWxdJytMXDqsQVCSigTq3pYrFLX2WNsmQChJ84DwWNWyGLVMcQSuPjNKiq8TW+JLhxHeBsShw0rr1jB4cRWX7zhKYFshrHPaijbbbfdAvWPMwjs2GJReeFegj8QDPhEbPDGvsHHNkJalaaf+JFf5ccSuMiLfTOS3zNmzKhYaaZO2uYNZ7vKnnMOT5s2Lax6xHPj+qtrr3V9g7QgFKjecQRTDxK79U4cRNU1bcuq+1KfuXpQHmmL8eokpJNz6HMORy9vor/whS/k53zCqkSQHfRWLyfRI/SfOOhBm2XbV+vs5XpwYVtFVgJELIGL3ziAWUnH9hOO42w99thjw1ZWBB5Sznf7ljH3SLiX1YdwxKNbmojFNufATxG4SNvqdvssdBMOeragYxUkREHNbs+zAY1UYIL+QL+ICb88o6695/SzgrzcT9qWjMqxnFDf6Dm70hjX4vgnWMX2HuhCS+DifNvxgHvohzjQ7eozHEcIzDFe8WdFK+DI0W7PWXIFYwAYtBUIK+hg2nFKeC59iP6SEgVwUsH11PV1xyBkY1fY4KS9HozYtu6yyy6zhxt9L+nf2BS77rprZ7XM+EEEUyDy0FcljLXUsUgaOk6/YWy58cYbwxY/snMuuuiiirfPEXQtAfjnP//5nTGe+yAj8gzGfsRuvcbvknZV2nbqdLUNsMZbkpJPS+A64YQT5iNAlZRhUHnJ1SFloA0eeuih1e3zViqxUtdP7XV833zzzUM75zvpoX9KhNXZdthhh05bURrkDaI4eCOWwFUynpeMD8oL/Zm2pnGVsQ2dwopckETQrYzPrL5hZSTHgBJdjq6iHtWPWTXvoIMOCuRM7AFWyLFCuen76FLKSl9BpLdzdrBNA13N6pwigNq2UzKO2cA9AXQC00pbz+UZzA+uvfZaHer6mZs7UO+yg+0cAB2LLYqkbAWN7SlbwRK4UjqnrY3FOKi2KFuna4HNBW2fx625tsS5tvYt7UOrKqdWzCTNJoLdw/ZmyCWXXBLm7qn7Suc29j7bFnhGSX9M5Y1jdWME53MELs6VjEmpcaCXfsa96IacTcS8nVXzIFl1E9vPLIGL+yCQ4yOgvVnBxmF1KW1LaAlcXFeCUbc6qZvv5ebrSjO2k5sQuNDVKh9l4mWeNn4Z7kFvluh9Wycp3Ycu7Ubg6tWe6KV95ubTYCKhjTBHxwaPhXER+xeyaSxtdR/3l/i6SvpYiY4q6Ssl9lqMo/2dsyG6zedJI0Xg4jj3tvUHMj/P2QkiFJC2iyPgCDgCjsDgEJC+9RW4umDsBK4uAPlpR8ARcAQcAUfAERgqBGwQZ/r06YHYgDOAFbZ4uxGCAtsq9iI4wdhCjD8CoARFeKuat7W6CQQOViVQwJO81Dk1CXisuuqqIXCAowjy2ZVXXln7LIIuU6ZMCc5O8so2V5BECG6kyGXkGccoJDNWQyI/EGjmzJmTDYh3KydOIEhxOKrAhWdrdYW6eyGtrDxvWzQwgmzCvbz9z5uTdcJzWHWLIBvBJomwgIwABmyPo6XxuQZnHW/n4QAmmNiUiKL09dlv/JSu/cRJRl6ZyLB9TNO8jkTelM9e+obS6OVzJMtakk/6BSQj2p36WJN+ET8LZz59hP5CP6U9NJGVVlopBEC5D91DYAVd1E2YE5Jv8EU/sA0BK9MQOB1JmTBhQtg2DyITATWCoTYPBFgJboIpennQUqKfCUAxdvBGcBPsbRkoG/WAnoasxXjWLQ3pQOqu6XjAM8GaNobuxGly+zzCBdtTlgg44VjPEUObpkm9M56TJ9oh7R4McuNa03RLriOgMnHeyj3UBQEexlnaJH+9Skn/Bpt111036BYIG2yBSQCCMY/8xcI12AmMt9hG1K0dO+nzL3rRi0KwlH4m0rJNZ/nllw/YQyBFtt9+++qVr3xl+E6QWFtshgNP/mvbrvrVdmweev3etgy9Pi93P3VIG0TvQRKhvukPTYmhuXQ5brf0jFeeqLsvdY7xhnyi99AljDtNxr2243mv4wPBOoiJ4Cf7CmIAOKNrCATHMpJjQIkup0zqMjQfAABAAElEQVTgzjiJDrf9GF3KdpGUAfIlBFjpCuYbtCvmONgYTerLYkO64AaW0g8632Ycs4F7beWOflxvvfWCbmKcZ1xRfekZTT9zc4em9/fruhIbC0II42DqhYJu+Sp5Xl1b4nn9sm+75X1Yzpf0x0Hlvdcxqdd+BpGcNLAZ6d/oeeY4zKuZL/RLeA7PwDbB9kVvNZ3/9IpRmzKUztdzz0BHQxJlLMUvAumrVAap93N56tWe6LV95vIVH6eNMHbhq2PMw55n/Os2z2qr+6jHtr6ukj5WqqNK+kpbey3GPv5dZ0PE1zb5LSxK/IHDYic0Kadf4wg4Ao7AeELACVwNa5OJnYsj4Ag4Ao6AI+AIOAJjBYEUgWus5N3z6Qg4Ao6AIzB+ESAoTtAVwuhnPvOZ8VvQqGTaYpWVoCAgu4w9BCDgsIoo8utf/zqs5hmXgjYNkQwSCCs9iAwSX9fmt7edNmj171oIpgS6EK1q2r/UhyMltkSFFM9Khqy6FAdpCXTvtttuIbOzZs2qTj755J4zvrCOAaXApQL3pWmNp/sg47MaKyumsbqRiyPQCwLez3pBb/D3spoSK7Eh8Qpjg39670/o1Z7w9tl7HXgKjoAj4Ag4Ao7AWEPACVwNa8wJXA2B8sscAUfAEXAEHAFHYCgQcALXUFSDZ8IRcAQcAUcgQoBtwphfswUhq4ksDLL++utXbNnESiH77bdfIK8tDOUeb2Vk9RVtk0NdssKWnGqUlS1MWRUHYTvFb37zm+F7L/+87fSCXvt7qWNWaLLEpXhb1vapDu8ddqsrSImnnnpqJ7OsoMKWVdoW7Ctf+UpfVpJZGMeADqgFXzxwvyBorPbCFmqs4sJWqf1Y9XHBp/iRhQkB72fDV9uMx7zsQX9nRXlW4WIlxdLtjEe6hP20J7x9jnTt+fMcAUfAEXAEHIHRR0C+Jt9CsUtdOIGrC0B+2hFwBBwBR8ARcASGCgEncA1VdXhmHAFHwBFwBJ5EYPfddw9bws6cOXOhwWSttdaqNt5440COYEsdl7GLwN577x1WLKIErK7FFjOsKMSWSWzvghBwZJW5eNu0cLLlP287LQHr8fLPfe5z1ZJLLhmIIUoKIh6EvPEokyZNCitsQYRB2Er00UcfDdsGEjSXXHTRRdXpp5+unz19LoxjQC+AeeB+QfQgRkybNq0655xzwvaRC17hRxyBdgh4P2uH10hcjT+Lba41PvHM888/v5oxY8ZIPL7nZ/TTnvD22XN1eAKOgCPgCDgCjsCYQ8AJXA2rzAlcDYHyyxwBR8ARcAQcAUdgKBBwAtdQVINnwhFwBBwBR8ARcATGEQKQtfbaa69qwoQJyVJB3mKlooceeih53g8ONwIEXJdaaqlOJi+55JLqtNNO6/wej1822GCDavvtt+8QEOMyLgwYxGUept8euB+m2vC8jFcEvJ8NX81afxa5u+uuu8JWv8OX03SO+mlPePtMY+xHHQFHwBFwBByB8YyAE7ga1q4TuBoC5Zc5Ao6AI+AIOAKOwFAgsMwyy1S8VY9ceeWVYdumociYZ8IRcAQcAUfAEXAEHIExjsCKK65YTZ06tcLeQn7/+99XN954Y3XPPfeM8ZIt3Nlfe+21g/384IMPhpUC77777oUCEFaPY5VA2jUEtvvuu6+aPXt2wIAtq1xGD4HFFlssrEJDDtgqkFX/XBwBR6C/CHg/6y+e/UiN8WjdddcN2ybedttt1c0339yPZEcsjX7aE94+R6za/EGOgCPgCDgCjsDQIOAEroZV4QSuhkD5ZY6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj0BgBJ3A1hMoJXA2B8sscAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR6AxAk7gagiVE7gaAuWXOQKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao5AYwScwNUQKidwNQTKL3MEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcgcYIOIGrIVRO4GoIlF/mCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKNEXACV0OonMDVECi/zBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEGiPgBK6GUDmBqyFQfpkj4Ag4Ao7AqCHwute9rlpmmWWq+++/vzrvvPNGLR/dHrzRRhuFfD744IPVxRdf3O1yPz/GENh8882rxRZbrPrDH/5QXX/99Y1zv95661WTJk2qHnjggeqcc87pet9qq61WTZkyJVz3ox/9qOv1g7zguc99bvWqV70qPGLWrFnVvffeO8jHjdm0F1lkkWqLLbaoFl100WrOnDnVDTfcMGbLMtIZf8YznlFttdVW1XLLLVctvvji1cknn1zdcccdI50Nf15DBF784hdXkydPrh5++OHqwgsvbHhX/WWDSLP+id3P0pfR+U996lPDxdgef//732tvfMpTnlJtueWWFZ+I68xauAZ2ciyOW69+9aurF7zgBdVf//rX6txzzx0YNk0SXnbZZavXvOY14dKZM2f6uN8ENL8mILDSSitVzIWQk046qfrf//4Xvtf9K53jjeU51zDoqNI5TV1d+rmFG4FhaNcLdw146dsi0NZH0zb9hf36sTxO5+oOvxi6rkSwqfF1lNjY41W/pmyRXFk33njjaumll67++Mc/Vtdcc01JFfg9joAj4Ag4AkOEgBO4GlaGE7gaAuWXOQKOgCPgCIwaAl/5ylfCZPdvf/tb9ZnPfGbg+YCks8IKK4Tn/OlPf6oef/zxRs9UPh999NFqn332aXSPXzR2EPjOd74TMguB6+CDD26ccdrs8573vOof//hHte+++3a9793vfne1zjrrhOv23HPPrtcP8gKcVNttt114xE9/+tPq0ksvHeTjhj7tJZdcMjjtCErOnTu3k9+nPe1p1YEHHhh+33rrrdWhhx7aOedf8gg8/elPr770pS9VfEpOOOGE6qqrrtJP/xwyBKxO+OIXvxgIJ71mcRBp9ponSJlf/vKXqyWWWCIkddttt1Xf+973apN9+9vfXk2dOjVc85///Kfab7/9qscee6z2Hj/ZfwRsexor49bnPve5aqmllgokQdrNSMiECRNC+8Y2seRsix8k8iuuuGIksrNQPyNnW4w1UKwO3GuvvSr0YDfR3KntHE/39TrnKp3zdStX3Xnbx0ZLR5XOaerKNRrnRqP+RqOcI/nMiRMnBiI6Lx499NBDjR89DO06l9nSMuXSG+vHS/tN6X3DildbH82wlmM081XXJvo1To9m+eJnMzd85jOfGR9u9Pu4444L98q31sbGHmb92qjwmYtStkiurF//+teDzwj/vPxumWT9sCPgCDgCjsAYQMAJXA0ryQlcDYHyyxwBR8ARcARGDQFN/ts690sz/MY3vrHiDznmmGOq6667rlFSymevwYRGD/OLRhyBlIOhSSbaOgedwNUE1dG55uMf/3j1whe+MKwq8bGPfayTCSdwdaBo9QWiIu0dgRSHjj/xxBOrm2++uVU6fvHIIcDqUjhNWZnq6quvro4//vieHz6INHvO1LwEWBlst9126yR1yCGHVL///e87v+0X3hZG10P8Qk477bTqkksusZf49xFCIOf4H6HHFz1mNAhc9GPGLlbTmz59eiffFr82waVOAv6lNQI526J1QqN8w1gkcJXO+XqB2vYxJ3D1gmQV5uslc/benjp+78aW2X///UMBf/vb31Y//OEPGxd2GNp1KrO9lCmV3ng4Vqr3Su8bVsza+miGtRyjma+6NjEefaNf+MIXqmc/+9lFkDuBa0HYUv7V3FjiBK4F8fMjjoAj4AiMZQScwNWw9pzA1RAov8wRcAQcAUdg1BDQ5H80CFw/+MEPGm+XxzYgbD3DFjhnn332qOHlDx4MAikHQ5MntXUODhOBi/ZMu0Yuuuii6q677mpS5HF7TS7ICmmDwCWkFshHvoJUsyaw4447Vq94xSvCxQSJCBa5DD8CO++8c7XGGmtU//rXv6q99967LxkeRJr9yNguu+xSrb766iGpOhsEHLRy53333VcdcMAB/Xi8p1GAwFgct5zAVVDR4+iWnG0x1oo4kgSufs25bPC5zZyvl7oZBh1VOqfppdyDuHc06m8Q5RiWNHshOw1Du07h2EuZUumNh2Ol/ab0vmHFrK2PZljLMZr5qmsT/RqnR7N88bOf8YxndFZntuemTZtWsZoqcvTRR1esEhXLI488Um2wwQad1e3bvCQxrPo1LmPb3ylbJFdWJ3C1RdevdwQcAUdguBFwAlfD+nECV0Og/DJHwBFwBByBUUNgrBC4Rg0gf/CIIJByMDR5cFvn4DARuJqUb2G6ZrwEWYelzt73vvdVa621VshOv7bjG5ayjed8QFQScYu3ia+55pqeizuINHvO1LwE2N4TMtaiiy4akjvnnHMq/qy8/OUvr3baaadwiJXk2BYUIreLI9AUASdwNUVqfF43XmyLkSRw9asl1AWf+/WMYUyndE4zbGVZWOtvUPUwHslO47FMvdZ/ab8pva/X/A7q/rY+mkHlYyynO97aRGldsP35MsssE27/5je/Wd15553JpOzqUm0IXMnExsHBNraIE7jGQYV7ERwBR8ARMAg4gcuAUffVCVx16Pg5R8ARcAQcgRIEVltttWrDDTcMt/7qV7+qbr311vmSec973hOCoazeEW+/xMR3q622Cteff/751R//+MfKErg+//nPV+utt14I+hPw/cc//hGuueCCC6q77757vufox4orrlhtscUW1fOf//xq8cUXr/75z39WGAps/3TxxRdX//3vf8Olz3rWs6q3ve1t1XLLLdeZgN9zzz3V/fffX5111lnVvffeqySTn7xlhpOQfMRbJy211FLVW9/61pD2M5/5zOqxxx6rWKWDlXp+85vfJNNbGA/yVttb3vKWauLEieEttv/85z+hrsDzsssuS0LCSjCvfvWrA7bUL22CujrzzDMXePuNYPy73vWusM3V9ddfn1wp6fWvf331ghe8INTPjBkzOs+sczCwBdgrX/nKirZPG37wwQfD1pu0sU984hPV8573vJCvfffdt5Ne7ktM4MLRM2XKlLC6G33mz3/+czVz5szabebAjz7Idn9LLLFEuOfGG28MbY2211SWX375gC3XQ1p46KGHwq3bbLNNaOtz584NK3NBYJg6dWrF9azS9etf/7q64YYbwrVsQwY2K620UuhrnKc+b7vttnBe/7bbbruwndPs2bOrW265JaS39tprd/DkGHUKQSIlTft5fC965E1velNYPQes0A+QL37+859Xf/jDH8Ll0mkveclLKtooQvuhPnC+kSd0B3Lttdd26mb99devJk+eXD3wwAPV6aefHlbyASfyympdYIEOyK08xbNe+9rXhq3c+D5nzpyALfdxnGPcm3rLM2TG/Nt4441DH8GhSP1QZ+hS2gjbvt5xxx0hj5pEmVs7X1ktC8IVmCGkBQ4QeOJ6oe74I70f//jH4fsb3vCGirkHbYk0Vl111WrppZcOaVGOxx9/vKLPsdKRpE3/5p73vve9ne39WA2N/k5/+Mtf/lIdfPDBleoEPXHyySeHc/Qx2int+4orrgj9hLpFn6PXV1555TB2oFfI5+WXX67szfdJ2Ri/uB49z9jy97//PWy7x6qMfLeivPTSPtrqS94Ofs1rXhPKzbjE28DgNGvWrICRzV/dd20hwXiHQ9UK+lCEJrCi3VL3kyZNCroQ/Xj77beHlSotJoNI0+ar9Dv95J3vfGe4nTol4EP7QSgrNspiiy0WfmNT/OxnPwvf+ccWddtuu23A+znPeU44jn5Bz/3iF7/oXKcvG220UegX9Ce2cI6FNon+RehHObsnvo/f6Iu27aWtXtXYQP0yTtG3Xvayl1UTJkwI/Rub7qJ5qznmAhs237yBje2GoOPOPfdce7rzXf0XzNiODFsOmwCx41bnhnlfmuoyCCmMC+ikU0891SYRdBo6DkEHovutENRiTNR4Ys/F3y2B67Of/WyoY3QSbQYdSvkZQ+g7KWnTzjbZZJNqlVVWCfXC6pG0aewD9Bu2LngyHiOMbzybFQPQ17InbrrppgobvUTA5aUvfWkoG+Mg2NIfKN+///3vTpK0VXQUtrK1xXTBmmuuGeqRe8in7t18883DWKc2s+WWWwYbij5KgI22yJiPnHHGGWE+Qj1ju2GPc0zSRl+W6L1utgU2cBPBRmSMZzylLTCmM54xN/rd7343XxIl+ZwvgXk/6NPYENjKjNu0B8ZO7GdsHGSvvfaqmuS/dI5XN+dirEF3oC+pd/oQ7Zs2i25C2sz5aBukh93C1knYB8wNmeNi58fSrQ3mbGul08Qu1bX6bNMGuKduTqM0U5/oB+qYvklbop3R1xj3NJ6D0fbbbx9uz/VfTm666abBXkJ3o191f7/rj3zKlqYuqT+2Q8b+lp1vyyrbbBB2on2Ovmvc7OecSmm3KTt6EB2CrkeoD+w3xjbGuDq7Hn8O9mTd2Nu2XffDd9KtTMKpzRiqe7p9NukrSqPXOXBTHdVG7ylvfDa5r5ves+k1tcG4BzsUfc94wryNdiHdgU2CLmYe88tf/tI+ovOdPlDio2mihzoPafClaR2V2r70P0kb24V7us2dla79bNImUuM09aE5Iv4IxmTGFNoE82jq8tJLL+2s+o4/C3sD/yw6meuZU8snZfPE97Zlj+8v/V1K4GpqY+fsBo1XvfgSND628TW10W91mKZskVxZ6whcjC/4GhB0xQknnDCfDTpa7aKu7H7OEXAEHIGFHQHFHvAH4RfCN4WdwB/f9bspTovMMyjS0aKmKQzpdU7gGtKK8Ww5Ao6AIzCGEWACztZYCA7SQw45pFMaSBwf+9jHOr/jN5RwYDEhRI488sgQFJBzn0k7ARocw7EwUWMCGAcFcYri5MkJgQdWy8BwYLL4yU9+MnkpwaNuwSrlk8DJPvvs00kHxwPkLYyPlEBwO/zwwztEstQ1C8MxAoq77rprWPEkVV4cdF/+8pfnOwU5Y911153vmH4QEIA8YIOuOKanT58eLiGoddRRR+nyzifBU4KmEHlse0g5GLgJZwfXKTjfSWjeF9oXgQKcXLTftgQu+g9B3JQQaDzppJMWOPXmN785BPIWODHvAGWCxAIBqInQFxXIJTCOQw1RW6ffUEYIarGQd4J6IhvY89QN/ZugreTb3/526CPkDbIlZLtYeN5BBx20ANGkTT+3aaIbtt5662zfhNSCs5hrNttsM3tr5/v+++8fCJkHHnhgOEZ/PvTQQ8P3j3zkIxWkL/QTgT4CrCmByHHeeefNdwpi1e67757EAbIVDm0mNhCK2H6wm4ggQDskKIMjORbySVqQsqxQF5Rl4jwiVEogR33jG98IOOg8zlkcrtQ1TnU51Th/3XXXdVbe0vX6xOGmLSjb9m/SUD/lGQTHRayhzDhXVSdcS8AZck4sYMRWSh/+8IfDRDI+f+GFFwaCnz1OPyXtnJ4HhyOOOKJD7uNe5aWkfZToS+oDgiiT41jI309+8pMsUTa+Xm9ccx+EJgVeuY5AmPoD7VPEjzgN7kGngzcyiDTjZ5b+tlskWruGsR3SFaI2pmfQR3fbbbcO6VPH9YmjgrHf6mPsI+wkZM8999SlnU+2c2RbRwRyEn9NpKS9lOhVjQ0E7SELo8digTD0ve99Lxk4t9cybmKfIbQzCCHcG4sc+lyD7YVNkBq3uK+tLmNlQPlL0B+2nfMsAm0IxGrKbuVb3/pW0NEEVSAn1on0M2M0Y6qeae+h7NgzjEtW2razT33qUyEQZtPgu2weO+4TSEOPpoTxG53WVKhP2jekmpRgK1HfDz/8cDiteqWfoGNigVQp/Q2+4IyICArZjYCm+hPn6FPoQcYnhP7D2CS9jc2CbYK01Zcleq+bbSGnZshQ5p/VTalL0C+Mz5KSfOpePt///vdXkOdiof9BkiLIirQlcDEOtJnjSdfEcy4C0SJWpvIIEZR6bzrn6zaHwx477LDD5rN/urVB28esbU1+m9qltmxt2wD3ylaCwMS8oJtgN6ADU3Md7qXuvv/97wcbB9uUNsen9HLq5RF8AKTLNegldFC/6w9bl62t0D8pSdnQss24vp92Yur5HFNb7uecinTbll31wb1WIJYx90Jv5ux65qOMr7mxt2277tbvmvpOupWJcrYdQy02qe9t+oru72UO3A0rq6Oa6j3lS59N7uum90irrQ3GPVZforshCqNbYknpslIfTVM9FOch97tNHZXavtKxbW0X8qzxIDd3TpWrSZuQbrPjtLVB0C0QwTkWC6R+5o/axt6ex05m/mhJa5wvKbtNt5fvJQSuNja27QfWbtB4VeJLKPE1lei3OlzV9mz/zZVV8wJIb/IzkDY6HPuevCGM27zYIRnNdqE8+Kcj4Ag4Ao7AggjI1+EErgWxme9Iyjk43wX+wxFwBBwBR8ARaIkAkycctwRE7ISdZHijXm9n8puVeE477TS+BtHkl0AVW4ogmvyHH/P+4eiFzIOjgkCBJv38/vSnP63LwgoDH/3oR0M+uAcHFgEdHA4QTjTJU3ADcg8EIhwnrJ6CEEgiXQhcWlGo84Doi/Jpy8w4i0MLLMgDKzFAMuMZL3rRizoOKFaXYpWahVXAg0Cpgmg48Vmpg22rCL7puCUt2ZWqwJaVSMCXN7AIMOgeVm1iFQJkEAQuG+AlHwQR+SRIqTzwbIJTbQlc3IfQDikb7dYSGGkzdmUyS3ghDwTRaY8EmVmdDOH4d7/73a7Bc67NOVDU1rkG0bOor1RwlvqkDNSL+h3ONwI2Ejmv9Vtp4qSl3MISZx0kOwnEhDb9XPcR4MNBK2HSgC6gjSgozzlIfmCnt0i1lRoBe4R84zSTIylF4AoXPvmP5/DWKGXCsSyxWwjG/QEsaFdcL92k+1LBJ52znyII2GPkg/ygR1UvPAtSGsclIjXym7LS17geop3wQE9yHXWN7PRkoIfvpKn64/tF81bgYWUt3qIWwUp4nnLKKaFtlvRvniVHIN+tiEQhR2d8DvIA5VE+dZ6xiL7Hm5sEHiQQCoQRZaD+dJ5+h3OR9s5qLyojz9DWg6STyktJ+2iiL1k5jS0rJdQ9mMQ6heAIf92EcRdHKnhB7MSZLLHOeR2j3gnu015sf1ZgkOsGkaae3+sn9U8fUvsg4I09QZvXMQKclAehn9ImFGSiXhn76cOMUWor6EGIQCImDYLAFeuTJu2lVK/GYwNYQPDkmehV6Rl+W4I016VENiHn0A3x6neQi7DbENlyuXGLa9rqMquHbMCEtGxgGL0IYUUCfnvssUf4CXkXcmSdxPqZ/oKdi92AfpYO4Tjtir6LlLQzAuwQi6XvaHs8i/YModDipzzTTum/tCU7xrPqDqvONREFd7kWvFjhFvtk4jxisOx4dC3lQxSooe+UELhCIuaf5hUEchifYgFbVpqBxFyiL0v0Hi+81NkWjKt1wupirJyAgCkrm9K30DHYMtJNkKhZXQ0pyWe4cd6/D3zgA2H1Lf3GHoPUSFtS39a5tgQu3ae2T9nr5njSNXbOZVdLVBtjPCZ/BIklvFiEXuo254OoBmFNwj3YPxBj0GfS73Yc41rb1nUvn2qDto9ZvdLGLtXKaiVtgLzIVrJBU47n5IMf/GBYOY/z6APmaOgnAqcidYE54xltEPKyXkBJvaTAKm60J0S6u9/1F+tH2hR6h7m5naPF9ZeyzdC5vdiJoaCZf2rLOk0foN32MqcqKTtjFnNMbDSEesZWxP+BH6TOruflC1ZDShG42rbrfvpOupUpxqmpraa6Sn227SukUToHbqujSn1dTe7rpvcoZ1sbjHusvuQ3Ql9k3khbwT6XxC9cYk9wDUK/oh/zafs/56yPpo0e4oWObtK2jkivxPblvhLbhfs0HvDdCnhRrylp0iak2+w4nbJBIACj8/ALoEdiIcBLnVu7BnuHl0EkpWXX/b1+2jpjfhC/VKz0U+25iY1t77N2Q2q8KvElqH908zWV6DeVPfWptmdtkVxZNS+wBC5IaJC0ZYOyqpt9eXe020WqzH7MEXAEHAFH4AkEnMDVsCXImG14uV/mCDgCjoAj4Ag0QsBOYiGt4BhB7IoF/I5XLZADy07MNPnnepzCrGpAIAlhksm2igpu8TYWgSiEN3EIDCGsiAOxQsJ9OCQIosaBRK0CwrWswhKvRqM04k/l0zopbFrxhNIGVa3jKE53YfhNEEWrPNi6p+wEBxQYxcEB6ccG0wkYEDgnmCBhBQ7ITAgBEwgpBJlw/PRzBS5WXmEFFoQ6xGFDABKBuEEwXm2zaR3bgDHp4ISg7Ujsig3gAWlRzkiCnATscISBiXUesXUpW6wgYEU/6iY5B4raOveTBxwqKnecfxz/Rx99dOdRNlDN9pLkFVHf5ztpokN0Dgc7/VxkIUif2qa0tJ8fcMABHSchq+vh9JXYgJh1EEIqxVEE3tStxDojcwQu7mGVLQVRudfqQ5sHVn7SFiZsFUBdcj/CNhAESBSYLSFwkRZtlaCZxOpsu8IQz9NWODhyuU+r0FBuyqDgnSX/2EAPzyBNAuM4FSUQinCsITbYW9q/SUeOQL6jG0488cSwSpnws45OjkHQY9UVxAYU+Q2h4Wtf+1pnZQ3btnmzkzc8EQLw2upNq7aFE0/+g6gCcRhh3AFHJM5L0/bRVl/yLNveY+In2y+wzQxtKh4PuTcnBMYgqtgxj2ttf+A35E0IT9IRdhUp+jh6QDKINJV2r59sD6OV5CAu8CdSQKzndt5550BS5Jm8YQ0xRoKTmf6mfnPllVeGdsr5QRC4StpLqV61YwN1yzaQIhpg8zD+0j6Qr371qx1bLhxI/LNjUGrcsmVDv6AP7T02yFGiyywRy+p2xgG9ZKBs24ANW+qy4gLSpJx2XCRIRX/R+A1JhGeprTEmSM+VtjPyBemYuqB/yi7iuMWP37FOs3atiBdcVyfWVkLHEMQVOQk7nJW3RGrEXsOZp0BNLwQudDwrJjI28VwkJnDRj1l1y46HJfqyF72Xsy3qMOWcMGKsA1ONzZyzbdT2ndJ82nEZXOnbmh9Rd4zhlvxux3TykxOrM6ijpnM83WfHH+HIsyBp2eC6rXfbl217jud8zCmxPxG21aYtSSC5YP8oaIidqxd9LJEh1QZtH7M6yrY7axPyzJxdWtIGSE86xAZNOZ4T6QvaGvMO9JTE2nuQVSGt2nGeeTlYWrEEL+w0xsF+15+1sVhZxm5LzBwN20PzCtqTXiSw9/XLTrRlj7+rLXO8X3MqW4Y2ZYfYgg5G4vmFrWfOx3Z9v9q17ZP98J3UlamXMRQMUtK2r5BG6Ry4VEdZjGO9lyqTjtXd103vldhgPNe2K37fcsstYbU/6SDmxFyDWB+StTva+GhK9FB4eOZfSR3ZMtvxW49I2b6cs2NIm7mexgPSSM2dOZ6TujYh3WbH6dgGoc54vurT2sM8k5XQtfIs8Uv8QdjFsW1YWvZcudoet34UOx+I07F1y7mmNra9z9oNVtczXjX1JZT6mkr0W4yB/a22Z22RXFll76if4+fHbyAyffxCGc8Z7XZhy+rfHQFHwBFwBOZHwAlc8+OR/eUEriw0fsIRcAQcAUegBwQsycQ6EJjQ4uxmgkmwGCeByDmsCMAkFLFv62ryz/Fjjz02vCHPd4mdgFonFA4YguYEp/RGv+7hU9vIkBdLxqhzRNj74+/Kp3VSWNKMJTfoXra7o9wEXexbZDq/MHwS+GFCjtAeIPzhwLZCsABnOwJJaYcdduisAmBXFrD3sM2Ull0/44wzqpkzZ/adwGXf7MT5yipvVuxbXyUErtw2jziv9MYpjg8CujYYYrehs/mRE4M2D6YxzvZavuccKGrrXGP7HL8tCYZgHP2M50nslmOQGiA3INZ5TfAaR44Vm64lfpb0c0vwSwWWcAThoEJXgZFWCpNTN9YZ1hlpA4PWqRYTPCjbBhtsENoy3xUosWnxbNq7nJpch9i2rfueOJP/bx2irJjCyilWIFfgfKfsVi9DPoDwSplJA31qhTdlaVcIgXCCyIgN9LDiAUSoWHIELrvKR5v+TfpyBPI91Y5snVxzzTXVcccdx6Udse2Qdq6AHhdYModdDXDbbbcNxDowom9akhr3cX6TTTbha9DzkAIRm5em7aNEX66zzjoVYw2Seg7HbTAgNc5yTSysroHeQSxJ2rZhMKFNaHKuNGxAA7KQZBBpKu1+fFrdq/TQczju1U8pP2MaNg7H7Kpruoc+Rd9CrI7pN4GrpL3Qfkv0KmWxY0NqHLK6i2A6geU6QR9pKy7I2NiLtCmJbEqLc27cKtVleoYdw7WaLHkRmdauKEsAnKAxbaPJSmNWP1tyqMpJewFbnqVVfHppZ6SrIEwdgcvqdOUFUrr0eXyvrok/IWgxxiApvWy3fhZRRoGaOEintLttoch1F81b7ZFteKxYIg8kQ0g46rtcx1bHJfqyF72Xsy1svlPfqQfslBtvvDHMj+Jr1HZZOUrbkZbm047LKVyxBdDr6g8lBK7U2JOb40nX2DmX1c8p4iTjFe2QMZgAKJKb8zFuQtpGckTFqVOnBvIz19jgoyUypLBK6ahSu7SkDZBf2Uo23xzPiWyjeLzjelZN0da+zLUgYyHqw+jJeM6htom9CZGKa/pZf8yRSA/B/hApKRx48p8leLD6Hu0PsbZZv+zEkHDmn9oyp/sxp+ql7HVkp252fb/adb99J7ky9TqGZqqzM59t01fUv0gzNUam5sC96Kic3suVScfr7uum90ptMNuuUn3Z2rnWJin10ZToIeETf5bWUYntW2q7kGeNB3xPtT+O56SuTUi32XE6tkHwc1jyuX1ZKkVeE1GKMUP+217KnitX2+PKF/cxvukljDgd257b2Nj2vhyBKzXH77evSbqqjX6LMbC/1fasLZIrq2wKCFxgsPvuu3fIWxDsmT9YGYZ2YfPj3x0BR8ARcATmR0A+Yvw9+DTwI2AD8cd3/Z7/rvyvReZNzv/fU5i/bsydcQLXmKsyz7Aj4Ag4AmMCAess0+oTdiUl3jbC8Y1okmuDMUyCNZnX5N9O1C0I9q1kSxaz1+g7jnvywUpEbF2CQRCnW+eIUDqpT+XTOinspJl7mMwT5MNRbANGqfQWlmOWFJEi04ADb7oT2Cdwyao3kLzYwoS6IziawhLCBMQJRA74fq/AJSeGdRiGB5p/cjbY4K85vcBX67giMAP5JRa7EoycOHJUKrgb38PvD33oQ9XkyZPDKbtaQOpajuUcKGrrcd/hHmxLESZTxB1WKmLFIsRuiSUs6wLeuqYblt36uV0dJbd9Ke2S1SSoW63QlAuyWmdkjsAlPRcK/uQ/VuAR4Ul60vYH3jI+7LDD7C3hO9sjiRRSQuDCOc3qUrGIdMBx+hXBba3UxtYGImrF9yk4b9uDDfTkiBo5ApecoG37N/mSIzDXJ21gzpKOVCb1V0ti0zl0DnlDZs+eXR1xxBE6tcAnJAfqCYIu7Z0JKQJRN0XgKmkfTfUlW0AR/EHAB8JnLHbLkKuvvro6/vjj40uSv6UL7BZEtj+wWqaISjYB9SWO8R29JelnmtgZWs1O6fOJDsFeaCusggQxAdtBAgmQMUZi+3Bd/0S/005sv+k3gcvmpWl7IT8p6aZXuUd1l+o/nLf2VTd7jesRHPSTJk0K37VSCz9sQNMG31PjFk6gUl1m2yokBFaO0uqJEDwJlkOwErGYtsGz+LTjQShA5p8IXDncuI12h/2KQCpGv/DmOdK2nXFPEwIX9haEslgg1bFyTbexWPdp7M7ZJwRgCXQiENcJakkXlxK4aMfUXdyeLYErhZslKrXRl73oPbUxqwuEXdtP5l+0E2zgifNWJ0ByBK42+llBbfII4Ya2Gotdua8tgStX9twcT7rGzrksQZS2xjgNYZ1+mBOrkyx5xtqJ9rhNh9W5IK0h1uYQkYEypdpgSkfZ57WxS21+9L1bG+A62Uo2aKr7U5+WdA3p+KqrrgrbJ2tF7NQ9bFPO9nmIXqThO/MQ5iOI1ZH9rD/m+DvuuGN4Rm4bW0tGZatA9DAyUnZieNi8f2rLqT5QMqfqpezWfxPrx252fb/adb99J7kyWfsoLqvqhs+UrWbPx99L+orGyDZzYKsz2uqonN6LyxL/rruvTu/1YoPZdpV6AYk8yl6wNokwtbo5Lk/qvhI9FKer373UUVvbt9R2Ia8aD+qwUpniz7o2Id1mx2lrK9lts5Uu20Iz7iMQghk7rGB/aEVavQDUS9lt2r18l++CNFJzeqVt23MbG9veJ98fadrxKvXcfvuaSvSbyp76VNuztkiurLa/Lr744p25eE5/D0O7SJXZjzkCjoAj4Ag8gYATuBq2BCdwNQTKL3MEHAFHwBFojYAcXhCxmNTqDXuc/hAFRAbQqgUKYMVv12nyb1epsJmxE/04IEhg7R3veEcIYuCotUFXpRE7TOscEbon9al8WicFQXvIRjgPY8FJwlvzOCe0tVR8zcLw2xKt4m0ecuVX2yKQSkA1JZZsoaW2+0ngsg5h63SI8wJBB+eJdSrG19jflsBFMJ/2GYsNXGt7CZw2Iomk7iEN2/5ZgQlHaJ3kHChq66k+aQNaIiXZZ3QjcLG9HE7glMhpFPfZtv3cOkW17VbqefGxXJDVOiNtMMo61RT4t2naVXhE1rJvC6beJuR+6wjPOa3sc/gu/RpjZ6+zW4lANCKQQJklTdoVxALahQ300AcIyMeSI3CV9m/SlyMQkpK+2+faOmFcElFY1+jZqf5qdUpM4ILMwBi35pprhi2XqKOU5AhcTdtHib4U2YT85OqQc9IPdgtNjteJti4hXeqeNmP7g92C1KZjVwyMA/39TBOiI0SXWOqIMvG18W/bbjW22Gs23XTTaptttgmH6vSsDUJotZh+E7hK2ovK0lavcl/d2MB5q99ie43zKWF7ZVaIQ2y/tnpc+HFNatyy28lxTa4fqA9wjXSZxVBbuWi8xX6F6AtJUG3KkhNSq5CRdizSz7H9a6+zq5KwQsJKK61U3M5ItwmBy5IubF5sEAUbt05yqzTW3cM5PaOUwJULhFsCl13tV/kp1Ze96L2cbaE81X1CQkBnMj5BhEtJjsDVRj+rvaTsPj2TrctZSQqJ9bquiT+76YzcHE/32TkXGEA+py6s0N8hWEK0pc9q+06uyc35rH5Jjc9KX2RGux2wiAy5NpjSUfZ5bexS8tG2DXCP7KO6+QvXSahXXrSyOpJzlJtVUngxS9tK657llluus4KtCK6cs3aY3eqyn/WnVRKVl246384nbf76aScqL/Gn2nKqb5XMqXopu53bxvOLbnZ9v9p1v30nuTL1YqvFdWh/l/QVkY3azIGtzmiro3J6z5Yj9b3uvjq914sNZtsVK2qyqmEsWt1Lczdb53U6LuWjKdFDcX70u5c6amv7ltou5FXjgbWxVYZun3VtQrrNjtPWVkrVzWabbVaxkwNy6qmnVvi5rNi5kwhcvZTdpt3L9xICVxsb2/aDHIErpQf67Wsq0W91uKrt2baQK6vmBXF6uXnCMLSLOK/+2xFwBBwBR+D/EXAC1/9jUfvNCVy18PhJR8ARcAQcgR4QsG+w4VAnMIkzV05ckTH4zYQMJziO4Xj1D03+c5OznHMfxxwOgNjZjBMXBw8TWpyE/CZvkjpHhK5JfSqf1knBdQrs80YsJLKUnHnmmdUFF1yQOjXuj9m3E2fMmFGdf/75XcssR2fdm4LWQaQgexMCV84BGTsYbJCWFZqOPPLIZL7lPJBTMXmROdiEwGW3G8WxRSAcTNqItimquyfnQFFbT/VJG2yISS48qxuByzpw4rxphSjbZ0v6uXWysXQ/5WgiuSCrbWs5ApecjPY5KaeaDbykttLi/l4IXKkAkfIE2XX99dcPP2nPrLhDwK6NKNhlAz06FqdjiTA22Fvav0k/7qfxM21gLlUnInCJeGzvx6lPWRDbtlntZNq0aUHX2+v5Dt6IdH+OwJXKS6p9lOhLjbUhIw3+5YL6qVvp76zAxjirbYJtf8htA1tH4OpnmoMgcNngQmyvgJGto7qVDm1bFAGpG4GL1UxY1QQR3uFH5p/NS9PxlaRK9Cr31Y0NnC8hcHGf+iWr6qArEGxGbLjYDkiNW3aFuXBzg3/SW7YfMtafddZZYQUikoDIxUoAbOus37QP6olxKrdSUbjY/BOBK7Wdii6z+hk9wkqyG264YTjdtp1xkwg5dfjlxiAFUZrYNXaVE7tSn8qV+9QzUnYG99jxA7uNwDciGy431lkCVyp4Vqove9F7OdsiFKjmn11V1V5G22OVTV4eQHIErjb6WeNyamzUs639Ysd0nU99dtMZuTme7ovnXNj59EdW7dVLDfa5EC0POuigQDrieG7Oh23I6r/W3rTp6LvyIQInx7u1wZSOQt9gYyBt7NKSNsAzutlKXBMLmFLHbGkfz625lr7KCryyezhmV2jViskivaX6aL/qL4cLeUqJzYsdm1O2mcajVF/I2YmpZ+qY2lBK15XMqXopuyW+1BG4ND6qDHz2s13303eSK5O1j0rGUFv2+HvbviL92mYO3IuOyum9uBzx77r76vReLzaYbVc5myQmcPXqo2mrh2Kc9LuXOiIN6Zomtm+p7cJzSsYD7kPq2oR0mx2nu9lKdo6llyaeeNIT/1MErl7KbtPu5bsdw1MrYSntJu1Z9q+1se19OQJXaryycxi9LGhttVyfqvM1tdVvKnvqM9X2cmUVLkoHu0t23qWXXtrZGlvnh6FdKC/+6Qg4Ao6AI7AgAk7gWhCT5BEncCVh8YOOgCPgCDgCfUDABkmYHOJQZ5LFm3O8QScHI5MvtmvaaaedwlNZWQAHlkST/5Rjk2tSzn2IBziR5WC+6667wpvXrPCk1a5wJC+//PILOOfrHBHKU+pT+bROivg6VmrgzaU11lgjkNmUP66DnCIDJr5vPP9mezFIPQjkLYLM3UQTePvGdHwPpAoFeq+//vqK7QWaELjkLGNbRm1TR9qxg4HADo45pO6NRW1taJ0w4ab/Y+/cY3Yrqvu/m/zahLYhtrVU8MLBGyIqKWgBJQjeUbkoKCioCIhgVUKaeqEYS1tpTaotXrgLyKWIwUtFpNy9oSBoURDQqkiJGiXR2sTYP3/nM/B9+J45M/syz/O8533PWSt5372fvWfPnll7Zs2atb6zpvLPAVw4WAGf5bTXXnt1hxxySLr86U9/OkUUYMsmjC2s+KcPDdFPf/rTDbYtK6WvGVDU1kt9ssXZwLtlvO7bqk/vlaOstZ9j4Fr30NZCAk6U6p9fqzlZ3Rg5L4DLAQ61lcYYzgDAQLmDJS+zfgsgUNvCinTutKLt4UARuJV2eOmllyq74pFIDNpOB3nOGACVHD1cdwe8O3tb+zd55v2Ua05ex5KhU/1/imNODkreAwADUA+RPuAZ/AZIjEEamhfA1SIvBSLl/fCH79RHACGQV2NJY6nAJ94fpgAE/H2LyhOZj0zKydtqfm/otzsXSgAu10n6InCxUhpHOEQ/E1CAyEpQqX16NKgxAK6W9tIqVymzZHRpbOC+y7exEbh4zrfYJjoNYx1jJZTzoTRuwdNWWcY75JikXmwdxnfQOOTbcBHdB7m39dZbd7UtK8kvJ8ln5Znf57fLLqKDUU9tMzO1nZHfSgG4iOTG9nuQwPTpx8A/jQO1tuSRLdCzpNvrWzkgw1/lc5MSgKtVXs4j92q6hZc7P3eHOLIDxxxtk8in8AxSpLhFALjUXmpRpXifb1fjYzr3ajQkM1yeuszQc31zLnQltuZk0YPAbJTD9fHanA9Q9g477JCKXYpqofqIxxr/uD7UBksyqkUvbW0DlHFIVyJNjdANiTYKUBWApo+xrgPzvDuK0WnR8Yn2CpXGz3TjoX/zfD9F8iQr5HIeHczfwzl9hm0UIZe1pXG4RU9MGVf+qS2XZF3LnGqeutfAThR9SK9fVLvO2TSv7aRWJ5ctLWNoXs7S77F9pWUOPI+Mqsm9Uh38Wt9zfXJvHh3M21UNbJIDuBZpoxkjh5xHfj7PNyKfKbpvq+7Ce+YZD/rahGSbj9NDupLPscYCuOapO/VfBK0lAJfPxeaxNY2Vb338LbU97/MOVtO8AJ338ssvT4vCmQtAXEMOsDBctBrahcoSx+BAcCA4EBzYmAPyfzIHwleMfxQ7H3+c6/fGT5av/M4ee+yx8f415bRr6moAuNbU54rCBgeCA8GBNcUBBl0M2wy6AKiIUgAJlIJRXQ44nDpsc1RyXmnyXzJskp8b4GTcx7l20EEHcTs5Nc4444x07v9kgJXTVPf6DBFKUzqqnG6k2G+//RJoiPrl29WhoBC6HecWVAoTXnrP5nbNo0nVtu7yaG6AJY4//vi0LSXfDoNJCWzgBuxrrrkmRc14xCMe0eEohUpRZtwROwTgIg8ZXP2bc12EkQpjA33AHUa6Xzo6gMuNFp7W+UEkAaJaCEQCT3CcARzJia0X+YNwXNKn+qhmQFFbL/XJFmcDZRAva85BtmFiJR28lGO8tZ97JJOa0V78dCdwzcnqxkh3Xg05gUqrInEYYvCFauAsnPbIvb406ab9E0CAS+9973tT1Bq7nU4FNhTICxmFDIf6tvV4/vOfn7YRox/AT+jIOQBcHmltSv/mvSVDINdFQ99E48JYABdgF9oKRB8H1Jn3PQcazAvgapGXbO3IanTorLPOStHD0g/7h7PjRS96Ubpy6623dsjiseRRoRjfcdByhFoBXMvIc2x9htK5c6HkgHaAJdskn3vuuRtliRxTBCkfazwCF1vT5WObb3OaA5c2esn6Cy3t5WlPe1qz/tQ3NlA+dxpIXyuVO7/mDlgA/sgmHIKMd9q6U8+Uxq15ZBn5+phB+95mm226n/3sZyk6AvclOxmb6Et8X7bHZpwdQy6ftXVj/pxWkks+z9POyFuAnGVH4OJdGt9roPvnPOc53ate9SqSzgB5ctT4GJwSPPRP/ODnIgFcWlxCvlPkpesBU+VeTbegDDXycaXUl7DzwRdoEQAub6O1bZG1HRXvXGkAF45EIupAAKgBsjmh/1M+5qYQIGH0zdqc77DDDuvW24BT2ksuuaRjXMzJQQnMc5HpUB+QgfslGeUyZqxe2toGKMOQrkQaEfP3vffeO/0kAqGATrrv8/l8ruPtEJkJ0I0tZyHGCzlbF/39HNwGeAvgR07IaeQO72YbyK9+9aspyaL1xPy9+e++cbNlTjVP3X2szecgQ3r9otr1om0ntTrNO4bm35HfrX1FY+SUOfA8Mqom90p18mt9z/XJvXl0MG9XYwFclFk8nWKjaZVDziM/n+cbkY+33SHdt1V34T1TxgPSO/W1Cck2/wZDupLPscYCuOapu9dlnvO1BOBqtTW1yrc+vpbanvd5t4VqXuCLQY477riO7Uah3Fa1GtpFX93jXnAgOBAc2NI5EACukS0gAFwjGRXJggPBgeBAcKCJA77yhQwcoIWRBAcSBlTRfffdlwwu+s1Rk/8SWIT7JQDXgQcemLYA4v4tt9yyUeQYN9r1AbjOP//8jqhdY0jldCMFUQcAaOFww5nBu5zYpoiyQjWDvaffHM9/7/d+LzlBaQfwh5XuOOxE6CoYBbkvw6ZH7rn55ps7DDxOOGgAVQD6gdy5J4MeyiJAESd3jrtTnTQlA4NAPtwnwheRvpwATuCwhnKnhqfzcwdwUUbANg4IAfRz6qmnJicUZcQJBd/ckXPttdd2V155pWebHN6sTMNwRn6EoPd8N0j80I+aAUVtvdQnW5wNvE7fhXMirOA4cDriiCM6tiGFFCa9tZ/71go4ov72IVCf3sc2gjjTIJw6RDeD5GTl3FfluzFyXgCX54W8pD/wnUXIE9ot8hPKHSxKlx+pIw5MqPSM80Tb3JJWgCbOkdcYzZw80p07iYccPeTh/didvX59av8u9VMv76IdcwCPFamPCDBymOudALzgvb7XvACuFnnpQEd3MquMHH1Lio9+9KOTAFw8LzAIW8wxbvIbmgpkSA899G8ZeXr+refuXCgBuAACI2sZs5CxAHK8D/NenOFEJoEcTOyr7YlMSv4itmYC1CWdaQyAq6W9tMpVytk3NnC/FcDFs5JhyEWtynP5TBqoNm61yjLydKAGvyEHaLnMevDugyAObeuna7Wj6sb9Up/BOYGTAlIfnqedkY/610oAuLx+55xzTqojZRCVwFh9YCC2jhRYhzxcx+tzIpN2KAJXq7z0sbv0DXl3bevYmm7BMzVCrrAtPcS4RyRYJ4ApAOMgH5tby+mOsJIOkfcRH9O9XPn5kMwozfHIQ89pzuWLhmqRXF1PxtkKUNudzz7ncxBxaWynDP49fVvyoTZYklGug43VS1vbAGUf0pVII9pll12SvsZvQE5EvshJ8rUE0nRQPmMiYA6PWEZei/5+HnHZ50pebt/S3XXNReuJ/s7SudryouZU89TdASN5Px/S6xfVrhdtO6nVad4xtPQtW/tKyxx4HhlVk3ulOvm1vueG5J5kBPmhf4ydT3q7mgLgarHRtMoh55Gfz/ONlI/0pyHdt1V34T1TxgOVS8e+NiHZpnGaZ4Z0EJ9jjQVwzVN31WPe41oCcPk3mGJrapVvfbwttT3v80MALuyizCHQKyCfG6+GdtFX97gXHAgOBAe2dA4EgGtkC1gUgItJEdtQYTgS80cWIZJtZhygTaF041xgtRtbNK1GQtHDsU452c5t2eWkj6CIQhiItMXQauRNlCk4sEgOeKQY8qWvsXpGJCOdfms7OP3mqMl/ybDJ/ZJx3yeYgH4AYNDvAPTQFzE2yAkK+AXnicBVPmkEjAEwB4P0EKmcbqTw7UTuvvvuDseVQDNsCfC2t72tw8EPYaDJVzUPvXNzue8rz9EjmMzjMAEMBFAGxzUED4mIwHUMc/qGV111VXf11VenNMh3nDSK+MaKRd9SUAZGEt90000pQgbGbsYuVi2LMPgLmMG1koHBV53zXXH+3HHHHSkLjAY4wlXGFgAXGeGgxlnCke22aDPUH7r++uu7K664Ip2zPQxOEt5HWyYiw9e+9rV0D+AOzzEWQeJj+tHzz/uCG1DU1kt9chEALnh54YUXJkAcddl///1nW9DxGwAbkU7m6edyKFF9og197GMfS/38cY97XGo/AB+gyy67rPv617+ezt2pQ8h5HFlsxeaGsHkBXLzoqKOOStvTcI6D8YILLkiyEwAbUZL0/bmfO1i4ViIZgHWP7Ur5Q7bBxze84Q2zyBQuh91RDrASANL999+fsiG6zzHHHNOJVx5FcMjRQwYOenBn7zz9u9RPVWeO/g0dhKc0cjCMjcBFf2N8UT8XkBNDIlsMIdvEH95BO5OMGCoLsgwgEMT2WIokOVVe8rwDJNhmi4gi9CXKyXcg6hKU19u/owNWUmL7J+AphmCA2xrnpwIZLMsUoRMZu8g8Pf/Wc3culABc5Et/wnEDMabhrMJhDdEu4LnaDGMZAAvInT30tw9/+MNJd+E68xaBkknrRmp+12hqe5lHrvaNDZRvHgDXi1/84o7IHE7en3S9Nm61yjLlS3QdATG5hqwgogzEN0Vui1wP1LW+Yy6f0U2Qz+iE5O3ymaiIksGt7Yyy6FshB2if6MiMvc6/mrNUq+DH6jVsG65t0+jPRKVDksxcxwAAQABJREFUD6EPHH744TNwtssfHx9w9J599tlJFgCwZm6h/kNdFgngIr8Weel6wFS552OB6xaUpUb+7QFdo+fCP/Q8dE/ajchBeq3l9MUU5Mt4wJbntJl167ekRu/2/uFjuspROqodlvRJ0pfmeFzXc97XaAeyb6IboyOLiD5Lm8JBr8Ug3PP2ns/5fI6KnnjmmWemNkgeRHlWVKo8SpzmGfl1lcXf6br1VL20tQ1QjiFdSWXl6EBgdF7mYYrSyTdHL1UET+zBylt5EKUVPd6ptNBk0d/PoxUD6kNX03yedgWACzmCDOSb/epXv0pF9P64CD3R6106V1su9YHWOVVr3dExmGNByBPKhpynn7s+KACk12dR7XrRtpO+OnkfmqKreb39vLWvOIBr7ByY97bKKP9Wudzz+uTnfc8Nyb1WHczfWdNJmCsxZ3KdpNVG0yKHcj7579ZvpDzG6r6kb9FdeE4yO7eZcW+I/PvkbUmyzcfpIR3E51hjAVyUsbXuLtf65rlDfFhLAC7q0mJrapVvfbwrtT1vU64jae7hEbjI29MzntPnZFNvbRd9ZY57wYHgQHAgOLAYDghDxBwIuzTzMubZ/HGu32PfFlsoFjiF45lBn20ChHYmGQMmE3qcTSjQQVsWB4ZWta4Wbnj0ndK2A4supyuVroQu+j19+RG1A6MufZTttjYlLbosgBowFEMYDzQITK0j8gxjHg58AA8y8jIBYBsUDPJB0zjgq0B50gEn/JbTl3OotHWMJv8lwybPlIz7jEsYIFldKcLo7M4FDOu6j0OJb8w2h24s1bMOaNC1/KhyupECcBZtkrYE0f9wHFA+LwvjJlFPtlTCmMNk253TGDDFN/jC9yIiA0fogAMOmIF6+A1vecZ1EkBYGPUAg4k8KoGu+ZF8UBTHALh4jghYgNhFPA+RB6T83KiYblT+5X1CyZSPfgMGgGe0XZFH/OIazwhgpDQ4L3ASUb8hqo1dauulPun9pwQU8xXvDo5y47XKlfOS6/RTDGzQPP183XqH49vf/vYN2lje5vKIgM6PVID1/zDY0adxgEOLAHBhIMOALPmUMrZ/fHO181YAl7LL25VHA1Iaj/LANbU5lYFrudHXDaIlRw/PuIM+d/a29u+SIZB3iZbhmKMdPf7xj9crZg5eyQDGhK222irdh99EqQLMO1SWGoCrRV4SwQeHnspEOZABtDUR1wCHEOFH5FEJGR+1Rabu64h+iVyBACBoC+OpQAblx3EZeXr+refuXKgBuOi79GHnL/0G/vu4lm/txDfH2Ox9y8vpfX8sgGtqe5lHrvaNDdRjHgCXOw7Iy0EY/Ba5nM7nXC2yTPmytS3bjkB8B807+M031Zbh/K61e+6VKAdwKU0unwF/AloTtbYznnf5w2/prs6/mrNUTpSxeg35e7QiflM3yGUS8ptxF3LQV7qQ/XPe0Nek5w05kcfYKlrk5ZBTkuI7D3zMc56rmrVxU/eJ/kjkRPGP6y4f4I/rf8h7+qdACqSvtdNaOT06H8/zDte7/Zt4/UhboyGZUZrjkZeeU7vlmoNP+Q0/qDf9xPnkEY9dZ+UZSHM+7EX0E3+WPOnvukadAUQDjBYNtUH/3i6jpuqlrW2AyIBDupLqomM+N0H+UnfksvMC2SBgq56lb3Dd0wH0FphK6Rb9/RhPkQ28X0R7YP6tsnA9B/u5bFyrAK7WusMP9S3OIdkohvT6RbXrZdhOanWaZwx9kDsb/2/pKy1zYN7cKqP65N7GNXr4St9zQ3KPXFp0MG9XNZ2kBODifS02mhY5xLtq1PqNlN9Y3Zf0LboLz00dD3hG1Ncm1O98nB7SlXyONQXA1Vr3sfNc1bd2XGsALtpVi62pRb7VeMb1UtvzPu86kuYeOYCLfE4++eTkm+bct7hvbRfkE7Q2OYBvENmPrn777bd3F6zHJkwl9EQipbOADn0KnRH9n8UA6IzY8kS0VwDKU4gFsYDkRVPep2fy4wtf+MK0mAL7PD6CIZrqQ52afuj9897nm1Bnok2jNzIfZYE3i17xFbT4hFvzbH0u5wEymfZGVGdt656n4TeBClg4wxG/NfMx5lS0TxZU5/Mw5UE7w1+E3Ro7K+Mx8yJ4deutt6bFoUqbH9k5CP1om222SXZt+E0bBltw8cUXj/It5XnyW98pAFwl7tg1rVCzS6NO+dBs6eKT4fxBhBuGzGVHNsrfG783LQfGGEU3bQkffPuWCODS9gwI9xNPPHGTfoZFl8VXz+EE0YqLKZVkGwqMzQxqNRrrKK89v6VeB0glYA4GFo9Ax6RK28L4CnHnlZ7Pt1tQmppxn629MMLKca70jE+sGr/zzjtTpBA5U7/4xS8mxzPpfOUqv0lPNIQ+UjndSEF6Vn3TRmtt65577knRCHC0bMmETkGUKACUOaFLENlGUUp0f8899+wAZOkb6jpH0mIQ9cmR7rvM0DVkI5GWiPbFGIEy6xG4ZFyVIVvPobADaOc754Tijc600047bbAqNE/nv7VVoNopkSZyZz7RuE477bQ0kfRnOe/jCQYPgCNjZSQTIvgLuQFFbb3UJ92QlzubyccBXCj7cnqJv4BHMKgr6hrPQHyfG2+8MQG4Hrzy4P95+jmLEQDfUGYn3kXZ2cKMdiBiQoVBgGhnIkAr9Hmis0AO4EK2IeOgkhOI/JCJkEdY4jfyAoATbUftm3Lx7egLAg8QZY2J2hAxNjLBw+mG0YEJGG3XibZNZIWSLPIoPv4MZWLrULa9ZCIn8j7GnCF31JGuD8DF/b62XOvfakd5PyU/aOibKAIXk0kBkh58skttEgMt5G2b9opeQ3tygjeMM3wvoiypncmwOFSWvvbRIi/pK8cff/xsPPay0pdxjDAeORFhTdG5GAMZC2sEv5B3RIMRL9iCmMiEOdUAAnm6ZeSZv2Pqb3culLZoVn7oPYxp2uZM1znSNgC6AYrLifTo6RjXndCRkElEuoHGArhIO7W9tMrVvrGBcswD4OJ53+qzBp6rjVs8D02VZQ8+1XV77bVXd8ghh6SfJeeBb/lXigymfEpHtkpmwQOyAdlGn8vlcw1A2drO0HPo38gvSIA4519pO2PSyokyBcDFcwcffHBaKc+5E3oa8idfKLPHHnt0hx566Ea8+PH6KD+kRZ+AHMCF0ZqFQrkurvd5tDQBdXTPj1PlJWO29ICpcq+mW8io6eXyc74VPJWOoHvUHV0PWSIdjnuUjzlYaznJAwMuAOu8fdJ20Xn1vrEAriGZUZvj6bn8O6MzE/EpLx9lR0dR5DB+i/rmfDVZyLP0mdNPP72jPToNtUHvY65bk8dUvbSlDaBnD+lKXh/OmYegsyA3SsTiGoBs+VbySkt7gJdQvjhCaTgu+vshH7EFKCqzv4vvRyQw/pyGdLMWPdHzz8/Vlhc5p+IdLXXnORxiRLuUXME5w5xzSK9fZLtetO2kVifxqUVX49kStfQV9cepc2De3yKjeK5P7nG/RrXnhuSe8puqg3m7qukkNQAX40CLjaZFDql+pWPrN1JeY3RfpeVdU+d6an+1ubPyrh1rbUKyzcfpIV3J51huJ9K7nRe5baWl7lPmuSpD6QgoWXNfj5Sbpx3Tnks6tj/nesPQeNVnS2ixNbXIt5wH/rvU9mp1FV9KczB4j71Jup/LipZ24WWM87XDAb4/c0LtcOL2urG1wF6H/Ym+UyNsOPiOII8ami6M+AfgmcUU0NT31bLXGKz5fC2drrtOh514yD8wNb3es4wj+ik2WeyOJcLuTDso+YFK6bnWmmfrc3k53D7xhS98obvmmmvyJOk3iwnRWaWjlxKxMw071DhhV0RGuh/D73OODQo5Sxty8m/v13VOenYNUIR4XR9zlK0jAFwD3GoFcGnSOpB9QtJhzMRIHbRlcCAAXOXvjEKJ4QBioPfoBuUnFn910aCpeUq4yLK4gk+ZxigfedlxaqOkCSSBERKABIY00Mk4VUQYJ0EnB60NDqDY7LrrrilqAo6mb33rWxsg0hkHn/CEJyTnJgq+gwxQCFndjFKBsdnBCVNrT9vCGElEASYUKCg4UXBA4ewOepgDGJx33nnn5HiCRz/84Q9nkRUeTvXwGc5HtnLjD7AehmYiP2lS9HDKDc9waAOuwajPNwBAM0XJ3zC3B6PFsEoHeYHyihODNrcIQg4xvsIPHIND5URB33HHHVP9qCeRKZBbQzxZRFlb85ABB0cYq/GYzLKlJeMn/GQFVa3e8/Rzyktfx2GOPOBdRKGqvYv0GIIYN5jk1Fa4kG5RxESLb8r3h9atWzcDhPn2oX3vY2wUgAsQGkY7QFz0G9r/GFkEf+ibtEdAXjwH6DGf5PWVY+q91v499T2LSI9jE8Ad34r+jyFccyDJG2TUkEybUpap8hLDFhN/xiLaFTIBQB8R1IKWwwFWLNIuACcjiwF50gYA6fYRso/vxDcjvSIT9T0zdG9Ke5lXrg6VZVPe3xSybEp94T3yGVmLHEGncTBxKa/WdiaQMHNTQILLJvRh9BN0b+ZZ6GzMuWpE+1+3fsxjrEJ3BmSqaFu1ZxZ1faXlZYtuga7EPAddlsUOLDaCryJAXMxlmNsyzs8zl1GeyBEMzZSXMQT9bKW+icrQdxRP6D/oEGyNB2/69Mi+OR/9kTbInIHFBfQV5gzLtOdM0UtV35VoA0Qrf9aznpXmaPCF1d7wgXYnfaf0bRywz0IBZFqNVJ9FfT/eg3xkPIVHyBHmGtgEtgRqqTuOS2Qu+j28cvvIPDyb0q4XbTsZqlPrGFrjx5S+Ms8cmPe3yqg+uVerF9dbn1OeK62DUd6pNpoWOaT6lY6t36iU19C1ldZdKM+8bWKoTmPvb4q6jy3bak431dY0Rb6thnpHu1gNX2H5ZfBIobytBcDlC7LQgbCTMdfBl6MFxsylTjrppKQbvfa1r0168VDtaIMij7I89X3KQ0dsjJSBMQ6izNh6+2iqD3Vq+r53L+IeIGVsCRBzD+Z4+OyYqwl4hy0foJx2bxl6b2uerc95ebARAsriW0I1ANfWW2+d/N3oExC+JuZg2J7JQ9fhCdvGO6BKiwV5DhskQFh8C+wkw9xIlPcZBRfgPvlq3oceh+1b7Rr+41OfSgHgGskxGD6VHKDDs3xAGheNBrR8viqrFPZ06jsj/drhgLcP3+ZotdWAdsrqf2gltlBcDfVfJGhq3vrMWxZWG+NwACgh0JXK1ALg8pVgGKDZvsYN3H5/6mpzlSuOwYHgQHAgOFDnQG68rqfc/O+wipOxjYlVKXrRYYcd1hGZBGLrVRz8Q5QDuIbSx/3gQHAgOBAcCA4EB4IDwYHgwDwcwLjPNuMsHAg7yjycjGc3Vw7EHHhz/bJRr+DA6uTAMmxNq7OmUaothQMsfGKxgFMORvF7pXNAUEQbgtBX8Qv6ol6i/gESgqZEQt9///1TdGCe851d5nkf/mwAZUSdFoiG/GsArqk+1KnpefdKEGBhAYXAo2ALZ/G1yAFx7N7BdoRD1Jpn63OUZ/fdd+/22WeftKBW0dBVzhqAiyjYBx54YErG4tsPfehDeiS1gRNOOGHWPm+++eYOPA7kux6xCJvtjX3hFeBEIsmJBDDEH0HUbtoXvGYnAF/8QtuD3/LJsysIu4NMoQBwjeRWC4DLV0/xmi+ujyZE+EARH1qoVK6xNQ6hT4PWNgcYWEDdsooT4wursADtAX7yUItDAC5WarLdgfZNZXChw7INBltDOGgGjrHvMBEjQHmWtm1h9eczn/nMFAmCrRfybX9AkiIYWcEHGhUh941vfCMhVPsAXKQlZD9AL9ozq55xkNKeW6IUgGrde++9UyNgoBfPEL4IfVa2EdKf/ZMRnqwAUxQo+pgjZ9WS+A7sX0sZ6cvwjmggDF7wCgELIazZk5lVdEIjExmCOsEzjwDACl62NwEYBfqXcMeU9frrr0+rdvVujvAIhDtEBA6iA7z4xS/unvjEJ6b9x1ntSr2IDiKlZ0pZUsaVf2xFpLCoeRKc1OJvfq/2G2Q2aGYIZUCDiKdn4ILnEAC0vK162jgPDgQHggPBgWkcCOP1w/zCWMDKX+jMM8/cYFs7n+gzjvs2nw/nsPEZY6NH4No4RVwJDgQHggPBgeBAcCA4EBwIDszPAexO2Ayxd+F0gKY4vOYvQeQQHFgbHIg58Nr4TlHK4MDmwoFl2Jo2F95EPdYeB7Cb4tMDSIIPk4j30FQA18knn5x8mTzLjhD4M52IcoQvECKSLBFlh4goyGz9CggGnzKAI9E876N8JaoBuKb6UKemL5VlGdcOP/zwWcSzW265pbv00ks3eA2+bKJNQUQW1/kGibIfrXm2Psfr3/SmN6WdLbKipJ81ABcYBgVNYjvEPDquA8rAa+DDhhzHUwu0g0+BtgppC1r3O+SAsZRw/T+2nAY7AbFTTGnxebpZ+Sffe2yhWGGQLrcAuHzPafIhLJuHvQdQAgBE5I1G1+K4djjAIFPbN55aABK64IILUkfldx+Ay403pM0JwBKDhDow97XPNZ2ZwS0nQkWyxRLk+wjze6+99upADTsamesQwgdQF5RH4AKwBXK1BhAiRDwCbQoBzKIskO9ZDvIV5QKAGsIXhSAngEIf+chHNgCOAfD6q7/6q1mIxfwZBioMAQCZDjjggBQdL0/DbwcrucAupc37MgAvVlNC8IQQlmypkBPgLSYIINjHliXPI/9NPh7mcd16FDxhI6EWABf1oD4AAJFxJdKe0tyjPg888EApWVwLDgQHggPBgQYOhPH6Yab5Ki2uMqYDtkYv0ViH/nXOOeeksN4PP1k/CwBXnTdxJzgQHAgOBAeCA8GB4EBwYHEcQK+HZItj0cE73/nOWAS3OBZHTpsJB2IOvJl8yKhGcGCNcGAZtqY1UvUo5mbIAcAs+Eixj55++umz3ZamArg0FuOTVpSnedhF0Iu///u/Twtz8XcT8MaDjszzvmOPPXYWYIIyAuxB364BuKb6UKemn4dPU571CFvvf//7u5/97GcbPf5P//RPs+AlJaBT/kBrnq3P8X62o+dP9Gd/9mcpmhq/awAuAW/BDlCvnAgco+0z77vvvoQJIA3YAYLpQGAQfv7zn6dz/0fkOQBb0LXXXttdeeWV3X777ZeCtHDtuuuuqwbVATMC5ZiBdHHgn/AfAeAaYFQLgAtwFo1CBBgFASE67rjjUsQf/R6LSlX6OK4uDjhSk8GQvX8BrgCeEcAJpyKCC6BODcAFWIo9XRlQyAdhQoQm2hL7ZyvkXt7hWwFce+65Z3fooYfOmKlIUqBxFYVKNx3ABXKbAZaBFkIwMiDQV0CzygB17733dqeddpqyGDwOAbg8A6Jo0acQ4OJLPgg7Gvp///d/E4iStPBZ4Rf5Tgh4IpS98IUvTOVXBCkJbBQG6uioWb4nUbx4JwhceKZ6X3PNNWkwobwO4FL5+bbso8t7+K56TvwaUxblNeXowNIWABcoevhGuyQqWU60B0Be8Jg6kp5jUHAgOBAcCA4shgOawLLSqbaiaDFvWhu5sN/8brvtNhtHvdSMP+edd15aVebX+84DwNXHnbgXHAgOBAeCA8GB4EBwIDiwKA5Iryc/FiSy6JMo8EHBgeDAhhxQX4k58IZ8iV/BgeDA8jiwaFvT8koaOQcH6hxwX+YVV1yRdk0i+AI0BcCFL/Xtb397eo4dotgOjm0Z2UWIXQzwC5PfPffck9KM+eegGPyM7AYlWvT7FJQi9x3rfflxqg91avr8fYv6LX2pbycKviP8hUiPn7ePWvNsfa5Ulpe+9KXdi170onSrBuACpAg2AbBTCbj21re+Ne2IRSa+W95rXvOajqAn8ExRufIysK0uGAToYx/7WGrr7MKmncQuvPDC7ic/+Un+WNo1TWnoN2yzOIUCwDWSWy0Arr6sAfWA7BNog7QRJruPY6v7HsAdgFl8T5yF+d6yLsDZV5b9ZWsALt8rGET097///VnlAVQRPYuoEvlg0wrgOvXUU2fRoFxw8dJXv/rVaUtBFcABXG95y1vSVoPcy8P/ISyps0BQNeSq8vXjGAAXSGzCDX73u99NjwKQI9wjQCkIFDHgKIBRikYmkFZKsP4fQCOeYS9aiFWOIL0hQEdE+OJbnnjiiema/onPDsbTPQ8TSrQ9ou5BOYCLCCHswau9dHfaaafuzW9+c0qbR7bqK0t6YOI/b4stAK6h13l4SwBz9Iug4EBwIDgQHFgcB1jxgT7AOAfoN6hLE7R99tknrSgjWieTz7vuuiuBrKeCiNnemAijRMNEvwkKDgQHggPBgeBAcCA4EBwIDiyDA2yp4U4vbChBwYHgwMYciDnwxjyJK8GB4MDyOYBPeFG2puWXNt4QHNiQA0QVwreIz1rBQPCFtgC46AcHHXRQesHNN9/c7brrrrOdD/yt2KrPPvvsFNzEr+fnAL8IcAPhIyVQiNOi37elALi00Lu2Sxc8PvLIIxM2gXOAR9/61rc4rVJrnq3PlQoyBsCVPwco62lPe1oKjIPfXkF2StHe8mf99x577NEddthh6RI+BnAEYDOGCGzCSSedNAuAw0Kd22+/feixDe4HgGsDdtR/LBLAtcMOO3Sg/RQxiLcCBuHDe4jAemnizmrjwDHHHJOEAeW66aab0laDXsbtt99+BgQiYhPb/NUAXAyqRNsC5JMPXORJO9l22203AhcJWFQTzqUtFIlY8brXvS4VFVQqYRVz8u3wBODygb4WMhME6qte9aqUHYIJATWGxgC4Lr744oQW9/wAQAGEggB34XR9+tOf3h199NHpmgOq0oX1/1A0iLgFnXvuuTNAVR9oCh7Rd++8885inT7wgQ+k+0QHI/oX5AAuhDygJgnflGD9P4V45DcgPlFfWZRmynFZAC62hOQb0NZFgNTYgjMoOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcWB4H8F/im2SRK5gD/JEsVHW/7pQIXPm2oio5UYsIlOFYB/zTbLHYt6CW++ziBJWiQC36fVsCgAsMi/AEJVCcvtkhhxzS7bXXXunn5z//+bT9n+7lx9Y8b7nlloWWpQXA5ZgN1Ys2SVCXMYtmAD7CK3AOohtuuKH73Oc+p5/V41/8xV+kXc/UL37605924DemkjAE9Cnyokz0N/441++x+f7OejTaZrlX1qIAXIR5Y29MGOskYIxfi/O1wwH25yVaA1QDrWi1EGjn+++/vwrgKtUadCiIaQBHbK1H+0HYeHSoFgDX61//+pQn71Tov/z9AJxe9rKXpctqp5SBMLLQV7/61e7yyy9P5/6PbfYECPuf//mfjmhP69ajXtlatERsMYogGgJwAXYkel1OL3nJSzr+IJWTCCVE4xKxFeXXvva1DpS4hJ/u+XEqaAplg++zz3okOnWEagAuUOheppR4/T+9k9+cEz5f56VoYCB/d9xxx5TG/6GIUf8aLQPARRuB9xqQaJtXXXVVxzaSQcGB4EBwIDgQHAgOBAeCA8GB4EBwIDgQHAgOBAeCA8GB4EBwIDgQHAgOBAeCA8GB4MByOUB0K6JcQeedd95si+5WABfbzO2+++6zQuNbPfPMM2fbxj35yU9OwR3kH8QHyzaLJSLAzQknnJBuKdhJnm6R7yPvLQHAxRZ/bPUH1YK1cO+Vr3zlbOu/K6+8srv22mu5XKTWPNkSfpFlaQFw0YbAMQjspAoCaPz4xz+etkHUtfz41Kc+NeEfCFoi+q//+q+085p+l45E3TrqqKNSgB7dBywGTuI3v/mNLo0+CsMQAK4Bls0L4EJwEXUL4eQE0IG9Z0HuBa1dDtABASxBDr7pq1EtAhfPMJBq71XyzQF/pFkEgAsgFMAjCBBaSYggrI499tiURsAoF/LcoCwlUrn/7//+r3vXu97V+Z7LeXr2lwXcNgTgIsQhUchyAhj2ile8Il1WOfkB0AxBndNvf/vbDqH75S9/OW2x5PcFpsp5rDTsEXzwwQcn0B7bWZaoBuCqKSV/+Zd/2T3pSU9KWfFdAKpBtbL89V//dffoRz86pfF/NYCb0iwSwEV52ata4Sd5B20IMGBE3hLH4xgcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcWB4HiP7DbkwQQBoAXKJFALgIPEEUI4FLlLcHAumLAOV+TQJdEPAiJwdwzfs+8t4SAFz4qRXl6YEHHkg7PuV85Tf+XILNQJdcckl36623pvPSv9Y82R1rkWVpAXB5fZ74xCd2r371q7ttttkmXQYQdfLJJ3uSdE7/eNOb3tQ95jGPmd3D307UrS996Uuza/kJILHDDz88BcsRJgJsAdtTwmMFa8mfG/qtPhYArgFOzQPgAtzwjne8o9t66603eAvhBdkPFhBJ0NrmAGEe6Zjsfcq3HkM1ANe+++7bHXDAARuBtujwRFciqhSAwBxcNBSB641vfGO3yy67pKL93d/9XdqHmKhYj3jEI9I137bPyw/ASxGvBIxCiO28886erPdcoKtFALhqwrUG4KJghIQk+l3eB1Xo73//+93pp5+un1XQFAlqded7EOHrj//4j1M+NQDXd7/73e6cc86ZvUsnaw3Axb6/RAETMZDdeOONHWE3g4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYGV4QBbJyrgAsEk8FuKCBbCbj8QPtv//u//Tufnn39+8j2nH4V/z3/+8zu2NYTYXeoDH/jARqn+4A/+YAYaAvsAUCunRz3qUSnQB9d//etfJyBYnobfi3qf8t4SAFzUVTgFoj6xbWaJ3A9d203Mn2vNs/U5f7fO5wVwkQ8gK7aYpJ1CeUAbIswB8lIUOfrNXXfd1V100UUdAWpqBNjrLW95S+fRun7+85939KkSOLGWT+l6ALhKXClcawVwAbY55ZRTZtGZlDWCEcBI34dX2jiufg4IPEX4PSIcjaESgIuBFSEilCYRqUBpglgFtQwBENtuu+0mA7iIAAfSFOId5OfRmEBNM2jm5NslCsBF9CkiZUFEsPrGN76RP7bBb0BXbKOIEBPAaYME638Q1hEQ0FAErhYAl96FcKY+T3/607vtt9+++93f/V3dSttAsh0kVIt65eh1BPj3vve97rbbbuvuueeetP0jz6K8IOSXDeD6oz/6o9lgw3tFtMG+gcG/OQA+vstUOvrooxMP9dzdd9/dXXDBBUnp07U4BgeCA8GB4EBwIDgQHAgOBAeCA8GB4EBwIDgQHAgOBAeCA8GB4EBwIDgQHAgOBAeCA8vnwKmnnroBmGTMGxXwo5bWd2mqBajgWe1URcQhfKw5OXiob/u+Rb1P799SAFyqZ43/8OOkk06aRaLKQUzilx9b82x9zt+t8yEAFztmEVkMuuaaa7qbbrpJj25wdL/2Zz7zmVlUrT333LM79NBDZ2nBThB8CSBWH7HFJHgNgb4I8APg64477uh7bPS9AHCNZFUrgIuoRuvWrZu9BdAHgum6666bXYuTtc+B97znPd2f/MmfpIqAbAXh6kToPcAyALN+/OMfd//6r//alQBc++yzT3fQQQelRwEHnXHGGZ5NOtcgSFs68cQTZ/cFIlO0q9mNh07e9773zQA/AnC97nWv63bbbbeU4tOf/nQCY+XPEW4T4BIkAJcDmQBv/du//Vv+WKrrq171qu7//b//l5DcAkdtlDC7sEgAF3XbcccdU2S0yy+/PHtT13lUMkBY7NsM1QBcDoITLzxT5AS8hZYN4PL3TjmfF8D1nOc8p+O7QigCbJeI0hYUHAgOBAeCA8GB4EBwIDgQHAgOBAeCA8GB4EBwIDgQHAgOBAeCA8GB4EBwIDgQHAgOrDwHjjnmmA22gPMS4J/GVw0RCIJgGdA///M/z87Theyfb71I8Ai2PizRBz/4wRTpCOAJPvOceA/+YnzbROiiDCVa1PuUt8BEY3fQmupDnZpe5Vr08Z3vfGe37bbbpmxL0bXYEhF8Ae2A3b4Acw1Ra56tz5XKMwbA9fa3vz096n7+PK83v/nN3U477ZQuE5Dk9ttvTwFnaKsKqgN2Z+wuUw6WvPfee1PQJqLPLYoCwDWSky0ALg8HqNcQZai2ZeIPf/jDFGlJaeO4djjgQKhvfvObCWXppT/22GM7UMPQf/zHf6S/EoDrwAMP7NhCEbrlllu6Sy+9NJ3r35Oe9KQOlDKUA7gAjim6FVseEs1K9OxnPzuF/9NvAbgciFUKWQlyFGCYEKQCLRH9iYhdEAIJlCnlcXrZy17Wse8xdPPNN3ef+MQn/Hb1fJEALt/PF9Ac4DknIoIhZCH65Uc/+tF0LgAXP3xrSR90SvkBbALgBC0awEWeXhZ+t9C8yoQ/TxRBtp8MCg4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB4IDwYHgQHAgOBAcCA4EB1YfB9hCEQAPRJQggjOMJQer4DP+6U9/usGjL3jBC7qXv/zl6Zr7WpWIXaXwI0PsPAWYq4/mfZ/nvaUAuBwHwM5LBJVxYhtMtqeE/vM//7P7+Mc/7reL5615tj5XKsQQgAtQoNoTeIV3v/vdG4EDwQIQaY60EG0RQB9tlrYLfelLX+qIzDWGHP/TB2ock1ctTQC4apzJrrcAuBzAkmVX/OmAj2KCuLhqOcBWmQwo7KMKAdIiVB8RihBUAHtAcAJyAmhFxysBuHbZZZcUFYo8EB4glun8CBeATS95yUtmSFDyAmgk4JRHk/rJT36SQvwB4mLvVoSQEKTkLQAX5witrbfemtMEcCLqF1G82G4QsA5gLZEAXPx2tOovfvGLVFZtCYrAo/2rzrzjV7/6lbLpPS4SwOXbPzJgse+utomkfkceeWQHKA7ykJ3sWfvkJz85Xf/sZz/bET0MNLgDwgifCIr5N7/5TYq+BvjuGc94RnqGfw6IY6tGlASoFmLUw4c6AK9WlpRZwz8HYDGA17ZQ1B7F1JtnRK7sADIcIgY8+kFQPwf222+/tNXuj370o+473/lOSkxUP/oDBAiyb2vMlGiOf4A5UaKJHnjDDTfMciqVa3ZzSScoUYQt3WGHHbpHPvKRqS+x7TAKFHIxKDjgHEA/e9aznpW2CKYNM16iqDPmMJH83Oc+V11BRDsj1G2NkF30OwD22sbY0zLGMfFh7EeHG4o0+YhHPKIj0ibEahD+hoj6Pe95z0vjKREv88nx0PMrdR896BWveEUq5xe/+MVVW84x/EAesuX0L3/5y6TP8cyi5DHyjUknbYYtspFtyyJ0yp133jllX4qUuqz3rpZ86Z+Ev4bXd911V1pVtVrKNk850Fv5ruiZN9544zxZzZ5dRp6zzBdwQkTdxz72sSkn5liab7Rm/ZSnPKXjD2KMWGk9daXl+mqWz6yqZc4I0Z5Z7LbaaFO3l9XGj7VWnmV9PxaqPeEJT0ir15kTo/ts6bQsXi+Sr6U5b03HY6xlfMTec+211y6yGGsyL/Qq5DW6LHOjRW0LshqYwXyQeSG0CD1jNdRpbBlWWicZW65lpduc27F4hv0fGz+2Buab0FqQzyr/Io+M1X/6p3/aPfDAAyHHF8nYyGsyB+iXjJ/ojIua7ywjz8kViwfWJAeGAFx//ud/PtuKDhs0QC0R9vfDDz88/cT2TgQj7cxDUItDDjlk5g/+h3/4h41s6diNn/vc56bn8Z1/4QtfUNbF47zv80zdpykQmd/Pz8f6UPXc1PR6btFHbKDUVQFZ+D7svIXPj3HxRS96UXolmALwDIyRolNOOWUWnQ0wFCA7qDXP1udUHj8OAbhI698A/AVbIN5///0pG6JuHXHEEbMdyhxwhS9eNkd8O8Jb+Pv9/Gtf+1qS5fhNX/ziF6db8Bk/Qh/94Ac/mBzAKQBcfRy1e0xqppJH8RnzbAC4xnBp9aZBAMoATSnV0ZkgihR9i98lABeCFcHJQCoCQIOSJwJcpfsAtHA8AGzwfYGV1o+UR2VxABcTOcBYuscz/k4/dwAXzgDyAZwkYuCmrJ7XFVdc0V1//fVKMnhcJICLl/3tepASTnMIHsA/+OzlJlwkYRIVtczLkB5c/4/9gMmHgcDrxzMaEMkffilv+PGP//iPCbDXCuCqlQVDYgv5QAZvagAuIoxB1I9BDCLCGwDEKUTUMnge1M8B8RsAF8BAyL/9pz71qe4rX/lKuk7/f/SjH53OAWsugr8CcpIX30xUKpfuLeMIcJK9qCXj/B30rU9+8pMdIJag4AAcePrTn56AuJLBJa4glwElMhbl5FsL5/fy30xoWJXkQMo//MM/7JiQQijUpdDQng8rXAB8QWNXOZV0Bc9ztZzj4FKEUNcVVkv5ppTj5JNPTuBRDyVdk8dT8iWtRzCdEpK57z3r1m/VzqSYiaKP6Q4AX0QEz74yrMZ7Y8Dzq6XcgFhwICOvCLndR94WXZ/ve2bo3jLyHHrnlPseZRi57QamWvvvy9/7BiH70ddXklZarq9m+cw4jt4HXXbZZd3Xv/71lfwUo961qdvLqEIuMVFLH1ticSZnvYzv5/NpCjRWp5tc+FX6QK1NLIPXi2ZBac7rY6DPuU888cRu++23T/YdvvmWQjWdxPUqorATjX1zIRwvOGAgtgPy+d7mUsdaPVZaJ6mVY6Wub87tWDzUYtzvfe97HQu0oUXKZ6JNsGiOufJq7yvY47faaqu0oI85/mqktcTP1ci/RZWpNvYtKn+1Reb6p5122kKyXUaeCylYZLLqOYDPpS8CF8FAXvOa16R6OMhFFfMdiriGHQtyf+mXv/zlBBpKN+wf/lWAtRBlGANonOd99uoEakIP2Ny3UKTO2IAY+/2bOC84x8/I3MfJ/SQf+MAHZuAn0rTm2fqcl4vzMQAuAkHQxvrqTV74u/GLCxwFSJGtJccSAUvAY2i+OPa5UlS6oWdVRrY6xf9F3fAB8Me5fg/lo/u/s8ceezzYY3VlMzm2ALiY+AFyGUsB4BrLqdWbDoQySM7cmYxQYNUeAC4R0ZqOOuqo9BMULAMbBNoTAcskw4k82Hv1zjvvTCEA6aQQqz2JEgWt738p2kAupH784x8nNLQAZrnD5/GPf3x3/PHHz4BHKbP1/wBzEAWMkINQ7pRl0kZZH/OYx6T7/o/BkDpPXa0IWpuIZZAb0BQyE7RwCUQEehsUN+TlxCHG/rco4yWi35111lkbOIPotyCxtSUlz+GYR2BSvoMPPjgJSc/vt7/9bXfOOeekPYZVfu6DVkbZUQjHb3/7293555/vj6bzWgSuvrJslMmICz6wjAFwAZqRwdJ5POJVKUkAuMZxqgSUqhmTicTHH0Rbok3NSyVjNnmWyjXvu2rP01cxqkh+IfPo70TLkxLFpACZJPR8La+4vvlz4NWvfnWKcOk1ZcwCwILcRG/TOEmaW2+9tbvkkks8eecTkw1uVH7QJhl/BKANANfDjFrNAIGHSznubC0BuJCbAg7moa/dUB8Aru8mHW1cC1j5VFr0wxiHntZHyDWtpitt2973bO3eMvKsvavleg3A1df++97jfSMAXH2cWv69AHAtn8fzvKG1j83zzkU/u+j+zmI1zespKzohwEPsNFsC9bWJRfN6GfwszXlrc27ZTdwesowyrbY8azrJ5gx8CQDXkakZsjjYo7Gvtra5iPJszu1Y/Fk2gEsRTHzXCb17tR0FcMHpuVoBXGuJn6vt+y6yPLWxb1HveOUrX9ntvffeCeiCn2QRu0ssI89F1TfyWd0cGAJwedQrAgcoKIXXqmST5z7RzVl8z04yJQIUhN+cdPS7sdT6Ps9fIJ2xAC7NBcijz4eqd0xNr+eWdawtesfuCHgLPEJOLFLH1wGVFjW05Elerc/xrIjIYYC4ICK3EcGtRACj8bPX8DzshHHeeefNFkDjQ6KuU0gArqn4nwBwTeHyxLS1Dz4xm0i+hXDgcY97XIqIhbMXoAHh8TD8jCUcKbvuumvaQoxVLYQ9dkQy7ZGQ/Uw+WfHpW4kAfli3PhoDkWyYpBA2ubT1U14WBk+ieJEv5WYbt/vuuy9PVvyNYORZgFy8E8CYQjUXH9gEFwGpAbAj+gXfAlAVod8RnDUCTEfULUBezn8AJXwf6ksoUZymAE1E2267bYqQBJiA8ImL2JqlVha9M45rmwMloBSh/InsBwHUVNhSB3ChcGjLxXk4UDJmk1+pXPO8p+9ZX1GRr9R43ete17GFEhRg5z4ubhn32P7kta997ayyyOePfOQjM2AVNxgfAdOSVsS+7shrkQO4aH9OOOhYdb/jjjsmQ4uAhb5iLgBcD3Nscwdw1eTxwxwYd7boCFxjHakB4Np8AFy0tGOPPTbp3USOAoC0CFpGnosoF3ksGsDF9qIYkCC2F12Enp4yG/lvpaNdrGb5zHdY7RG4NnV7GdmslpKsb4xZyguXkOmiv59HPa8tzFpCNVZNln1tYtG8XkalS3Pemo4nJ0wAuB78EsyFDjvssOR0w8Z42223LeMTbZI8A8B1ZOL7lgDg2pzbsTpPCcC1SPm8lgBHAeBSq4jjEAeWDeDCjwQwAhnku/MMlavv/jLy7Htf3AsO5BwgsAdzI+wNBL3A/z3VB57n2fd7pd/XV5a1dA+cwjOf+cwUIAH/Cdv/gTeYh1rzbH1ualmRtfh0wAOAW8B2il+V9ukR/afmuynSRwSukVwPANdIRkWy4EBwIDgQHJjMgSlAqc0VwCVDExGUWNngRAQuQuuigAEw1baenibOtxwO+AqHoQg0AL0E4gLMTARKkQO4+gAu7vz2lToB4BInHwylvDlvofhwTec7CwDXfPyb8rSvsAdMT5TU1UpTDcZsoyzg1oUXXriQRRPLyHNR/F40gGtR5WrNJwBcD3NuLQC4Hi7tlnfWB9bZ8rjxYI1ZEPaGN7wh/ViUA24t8XKtt4kSgKvG/wBwDUcFrfFurV0PANeR6ZNtCQCutdY2W8oru5pvodiST+2ZAHDVONN2fS3xs62Ga+OpqfPxlloxf2fOXdtZZrXk2VKOeCY4EBwIDgQHlsOBAHCN5GsAuEYyKpIFB4IDwYHgwGQOlABc2223XYr8Q2Y4CFj9S8hYIrxpv3CQ8yDHCR1KVDkRUfxe8IIXpFUQpAVpTrQ5Vsr+6Ec/UrLZsWbMLpWLvdB33nnnhNb/xCc+0a1bH/GPrScAuQC++sY3vtHdcsst6Z0Y+YkitsMOO6RtYSkjEZDYasQJNPy73vWudImogh/72Mf8djpXGfkBgAsg16Yg9IH9998/1YkVT0TuYPsU+HvVVVdtEAXq9a9/fUckJ/h/0UUXFYvLaoB99tkn3WMVBCuaRWz9+vznPz/xmGiATPS5TyhiIpHlhEOJaIqAmkh3+OGHp2dJ+6EPfWiWfEodZg+tPyGsrNoV50QxpMysYuA61/i+hFjOidUerPzEYAERnZLocURrJHzvWPKQufCVyFl90VPoC2x1w5H3YDRRRMyxAC7KpjDPnLOtLHmsBgAX/ZvtkeHr1ltvndoaMuGrX/1qageUt0REdeS5bbbZJvVNgGlMCmg77ONe4ymRJ5EBtFt4ijyhzwOybAVwTa3Dfvvtl+Qg7ezqq6/uXv7yl6eomIQBZ6/7MUTZ99prr+4pT3lKkqe/+tWv0na01J/v+8hHPjLJuJNOOilll8tjZJ2IPs4WzshA+ha8o8+xsoctlbx9DwG4CFWuyEC/+MUverdkIgoC+RElDkIO0Sdvv/321K98KyMAisht+iGRVikT/ENW59vw4pzmj/bAltSc49iibsgxl1GstmMrAMYltv5mFRdy/oorrthIDtBGkEkAcen7pcgN9G/aWK3uvA9AJt8DeYi8YXtU3g/fiAL7xfURK6EcwHXZZZd1e+65Z0rHFtm//OUvU8RYvpFHstW4RR7khZxh++ynPe1pacXaz3/+8/TOfBxrqR/t79nPfnaKmov8hOAN7yQyVN84pzGRSLCEgneifR955JHpEuWkXfANn/jEJ6a2TXsnWm4+Zi0jTy9X63kJwDXU/vveRZum3xApl9DnUCvP/D3kS1uhPSIXaFeMj/Qjlxk1ABfbYDDW19r/M57xjNSHGX9oHxrLVIZW+UzdNbaP0Rl53xS5p/KVjjmAi7bJlu3UhXIxntHPGZdqRFRknkEe0u/Z1h5+X3/99SkScuk5+EyUTr4VuhzfCr4jl5CLTqX24vcZfwnlz5FVwYynAMb//d//vahz+7P5OX2UsRk5xJiGHEamXnfddanPkp6x/pBDDkmP1toKN/fdd9+kqyLv2cZC2z+PrfuUPjZFx1vGPCIxo/Kv9v0OPPDAxGdkIVuGMZehPTInQTawtQJjgLZtp20dfPDBSYbynSDGAyKz0m4Y80WL1muWybMpc4KhNlHjdStfpn4jvafvqHGOb6zouzUdrw/ARf9kXk67gJirMx6Lps7h9Fx+bNEtyKNFlxnSSZAl1BlC15Q+yJyC+T73P/WpT6Xxgf7EmIVOjJ5JdG30NsYOFqOxW4B0MebQV155ZXXuga0BXYkV+8hY+t2dd96ZZLXrb6lgjf+GAFzM/enXELYKdE/R1DF0aB4jfqIDYmtBZkueU3/GRXhf2zaGck3hWU0nUf38SLQ6xihI8zC/r3PZhtQm+PaiqX1jjH1jik7C9yq1Y5WPPn3QQQclHtJGIcb0u+++O7VTpdNR8pl5xWc/+9lup5126vbYY4+O74hdBj2QMQJdZpGE7EbnpC9xznt4B+3zgx/8YJpvOYCrTz6P5d8+621W2AAYK5nP0b/pi+gp2CMZP7W1EWBA8mXcoO+gW3FNNHWc1HNTjx6B65RTTknzSNmkmLcy1qMvuvz2d7SUc1H8nFcW0P6QzdQX3ZFvRp2xISNz+W5Oep/6LfVYa7Lc6zNWDg6NfT4fpx+Nmet5Ofyc57UIABsl/dapRZ4sI08vU5wHB4IDwYHgwKblQAC4RvIfhTgoOBAcCA4EB4IDy+BACSjFZBlHAYQxFAfsO97xjuLrcUDj3IEwMpxwwgmzPavzBzDssJWcU8mYzf1SuRwUAHBD0Y08PwwDbO94/PHHJ8OV3+P8xhtvTI4tXcfYg2MfwqF811136VY6YjxgZRpGBze4b5BoBX4A0KD+lKNEGDvOOuusmUH71FNPTYZm0p555pmz6/6sVnpxzSOpYExl60gMLznxnssvv7y76aabNril74WhHEMEzgUIp52ALVProBdgNH/rW9+agDq6piPb7WLwwSCaty8cD/AMA0qJALpgvBhrgKcP4GiBxkY+wLnLMxhfMHbLCDMFwEUZaYeQAISbGsCFMwMHdK098l3OOOOMjXiLQwxHQI3oY4R31yRB6QA8IZNK7wPIhUEXAqyQt03lkR9b6iB5haMfhwBhkUV9kdSUBqAM7UhGeV3nSN0Bz/BtkWMCcOXy+Ctf+Up6DNlFe5ADz/Pi/Ne//nXHKmiBN/oAXAAQAIJB9PFLLrmkCHJKCdb/c1ChrnHUNp8uq4lABYCrRABYHBwB8Af5QxnoLzi1RJdeeukM3IDMBnBWIp4FOARoQOR1r0XEElDHI93peYydGChz4l044AEKc47DFeKbMG5AOJn4ViV5SjunvfPtIQeU4AQBuFVr86effvoMRNNSvwMOOKB73vOel96b/3vPe96zUR/0NIrGSZ1PPvnkGTiDNF53ZDJgJZx+OTE2IAdp69Ay8szf2fJb7YJnKS/Oy6H23/ce7xuAeunzrTzjPbSPo446KrWd0nv5RhdccMEMLFlzlgLEY8wEiMg3zckjSiIHcRaKWuVzi844Ve6pjKWj9zecaOgaJaIPf/jDH95oPNNK9tIzXMMxwvjtNDTufP/73086m4DMpfai/BhL6cclGUEaAPc44MdQTcbxLG0IwDJ6D7oWdeLI9Xe/+90b8YVn1EdIA1AFuTql7nqevJw0xnCtRcdzfi5qHuHly8/9fervpJFTFxAcOmip7dEG2KIbHcd1vvwdvvhkiMcl3WxIr/E6LJJnU+cEQ23Cy+m8hl8tfJn6jfLvUvotXvt8sqbj1QBc6JHohzjEIeYyRAcWqLZlDlcqK9dadAuec9k6VpcZ0knoJ9KrkJPoQJDzj4UQzLtymYgcQoajY4pv6eGH/iHjkWukc0IvRj8uETKNBUq5A7yUduhaH4BLW0yTB/oCYzV6CNQyhqoN1uYxzk9kPosbkPc5IZd8gZbuT+VZTSdRfn50Oci3Yg6ksdLTSZ8hjY9RLX1jyL4xVSdxfc/bMeXHnsGCJC2q8Dpxjn6GTcfbnOQe9gUWUNXaK4CZa6+9Ns+y6TflxN4n24RnAjgHntAHHcClcpLW5fMU/qFLAHzPSfM2b0u0XfqVZAH2vbPPPjs92jIe5O8c+1vjCHMdZHRJ/vDtaGcCbCvvlnIukp/zyALaMLIGPa1GgNcdVOfvW6uyXHWdIgeHxj5sBbTjKXM9laN0lHwEFIrt2En9dKo8WUaeXq44Dw4EB4IDwYFNxwH5ZtBDsWkzJjE34I9z/R5bwt9Zv9JgwxnX2CdXeboAcK3yDxTFCw4EB4IDa5gDMky5Ic4n0AC4WN123HHHJQcC0QIgwAEYUwFwYZzlOlvEycjHPYw4jGGsxGVQh9z5wm8ZEt2YzfVSuTSp5L4IJyIGTaL56B26h1GPVYustnQDAs5JlI8xhCENJwPEKt2PfvSjYx5baBrAUPBWdcDhg7EZEAiGGoGl4ANGMQhwDSu0oNIEHcULYzU8w6BE5B+IVXJvfOMb0zn/AH/AY6ICudEpBzDpe80efOiEZ/nGLXUgC4y11F3fFmMseWIYUlvUO3MAlzveMUSwupF601ZkdKSdkg7j3xA56Ioy0bZayfPqA/7Q9hRdSgZK3ulGbBRqABd9RDQ1VnBD7uzre8YNob6tBavaMSKJcB7BWxwJrI6WDMj7OiCrt73tbelb8h1xJOJAAOBG+xLAJXd6E7no0EMP1etmEU5wLOVG7rEArtY6SF7NCvPQCbIGQOQQ0W6k16stc3QZSR5jAFwAf2gHEGAyAAjwED5LJuDkoa1B7ogDdIvshgABAKyDKAtgTvpSH7397W9P8oDvBiG/kRW0LfItyWrGDL437cT77t+u37ZWILMjHwJwkSdl8X5/8cUXpyhtgEt32203kqQ0tD36IhFokIl6hkhcrGqGvO5TAVxeJvIiwg7gI96lts51ylsCcHEPQgYxJvKN5Nzguo8r7vTkHoR8po5EQXAZDFj2/PPPT2la6kfUGlZG0/YkD4ksAQH8QzbWCAcUxlp4DaAQPUHkzildgzestOY9tBl9I5cRy8hT75/n6OOIAFxD7b/vfd435MRq5RnvYbxm3IbgM+Mj/Z4oUpIPtD3qQbutyXUZ36cCuFrlc6vOOFXuJcZU/pX6G7KM/pbrrkQcwXEqIvoikcMg+EsEJMZo5BD9UW0cICqRISDyZAzhHt8KuYXDju9EdELJE0DIjGVQqb1w3Ve885vvxtjJuxmHRWzfiszrIxZDANCDJKfQM9GVaEci9F/klevFJaew81Xj+dS6j+lj3jcpN99tSMdzfqpe9JllzSP8fervvFdOXZWBI7oUbYjvJ31Ieh+6DuML0SY1DqD7oSsAqmKsW5Ze43VQeeflWcucYKhNeDmd1618mfqNxJu+o3RIn/Pmc26B9EsALtoBi2KkRzJmI7tp/1DrHK5W5hbdgrxcBijvIV1mSCehjkMALr2LfoFe6fM93dM4SRvUGMk9ohayyEvkCwV4BpmI/kf/pB9CXAcYhg1lHqoBuLC9sCgKQha8//3vn809W8dQtcG8vJrHeHtUGuQjbY12h01F5Iv4uNbCs5pOonfkR9o/8wiI6LZ5RFr4Bd8gjT+ct/aNIfvGVJ3E9T0HcOXfkzEd/YA+j14hGxCyA1CagGsu96gnxLO0f/RtnyfPa22PaTYAAEAASURBVLsgb+Y+2Byk49AvGbt8bCIdNAbANYV/LOQi4pdsfvAAfZe5JfqZt6UHS/Dgf/opUftYRNo6Hnh+U87zcYSyUGbmWIDRaA8Qv7UjAL9by7lIfs4jCxiDabcQcw9kJDIXm4/LXcDHisRVet9ak+XUd6ocHBr7aBtT53qUo0ayo9B3+U6SJaRvlSfLyLNW/rgeHAgOBAeCAyvLgQBwjeS3Jugjk0ey4EBwIDgQHAgOjOaADFN9AC4ZkxUhg8yJcsV2SyKf8Lljmfs4qwEIyUmMMUOOYhkS3ZjNM6Vy+TswgOCYUsSs3FiMMQdDp5zQmliSN9v/4PAYIkAqGFAgDKeEPscIsdKEk11bBpQiKnhkKPiJYwUHD+eQHEBebsBdgLwg/14euSsHwxBWm1D0GO3yPPW9yA+DABF8AIHwnaCWOvAckdS0TRuRbk477bRZnnnUDwdwcU/b/MAPVu7r22Esw/iJERLKwWjpYuEfWwLIwdoHuio8utGlPgAXRlqcs/wB4JIjz4FXGL8w0kErDeDysucOD4zF8FZlZktSyg3Bs3UPRUNj5TyGaxEGZtordc/blrfJL67fUogtIkRsQ8EKVVHeZnU9P7bWQfKK/GjbOHtoP5R5iLzPYZCkTQoEiIzEWSfgFff7InDBZ0XJcZAWZaCNvve97505WVixjHx1R5wAXA7soz7IdX2vofoA/BFw0Psez+WyGqM5xnORR/9zgEMOlgIswLM4IyAcRwC+kEHIGeQBwDURUbkwnEIYJCkf/d7rPgXA5Y4K+IOcA3gI4Wxh9b/kCPdrAC7GO+QH3wEi2gppqYcDb/NxzAFOPMc2jkRjgHgfMoA21Fo/8tG38PJzfYhwpgMWxKFJnxe5c4prAPeI0qC2zvYub37zm1NyBw9zYRl5phfN8c9BIgJwkV1f++97nfcNgQxaecZ3p3y0I74fABvfSg29SwAcyWp3cDkwtxXA1SqfnQ+ug8C7ms7YIvf6vkXe35BjAFjhJUR/e9Ob3pT4y293NIlfyCGB40gD0b8F6EU+0fch16HZAtqjBDKmC6zu8t/5pPZCXs53yXOuQw4uoz0QxamPJANII5CW0nt7kbPZ+3A+/vCcA7zQBdnuq6XufX2sVcdzfvKdlz2P8Pf593OnLnIQMK6AdrQFxnA5dr3dOXAv13eWpdd4HRbFs9Y5QV+b8HI6r1v50vqN1HdKR+mQPud1pzVgaM25cwAXkSzRC+X8BhAOoMmdry4X8vbRN4crlZVrrbpFLlvH6jK8U/KItiadius+TkoWcd35x2+fJ8AreK65G/IaecwiKMjL6boh7Qwdm7GV/omu6dFxXv/616dIX+ThMp7fLVQCcLkcpb0gB7BtiLy9jx1DeVZtkHN4nM9jcn4Cwjn33HOTrsozgGhIA8FHgepaeeZjjOsk6QWFf16+Eu8d9Mb8gXEdau0bffaNFp2k1o490loOGGdODXBN8w3GVMZWyNsB35MoWwKNcx/9XKDuXFfg/lRy/tIn6Bv0EYjFNUccccRMZxoCcLXwj/fQ5uAj8wvGSpG3Ja6xuImoWwD5RK3jgZ6fevRxBFsB8keAJWwf2BY19/c5Rks5F81P72vUe6wscBsk4D7qQtsUecQpt8Hl71uLsrxVDsKb2tjXMtcTr0tH5AjzFujTn/502mJY6VrlyTLyVJniGBwIDgQHggOblgMB4BrJ/wBwjWRUJAsOBAeCA8GByRyQYWoeABdOdQwQUA1M4uAFHPkXXHBBSi9DohuzuVEql08qv/WtbyVHW8rkoX9EDcHYCjlIjN84mnEOQ7fcckvHdlx95MAQDFOAwXBUbQo66KCDOpxVGD/gs8AMKgv399lnn/QTR50cuBifMeZAfp3fODe0Mk7OId9GzcFCpBe54Y5vKFCGvhfpcNbnq5Fb6uBGTtoH9QHs4AQQAGci5CAS6oRhDJ4B+MDI54QjBGMuhIFPhgxP4+deFgdceJop526YG/McbRDjLXyAcEpsCgAXxlnAkJCvbE4XHvq3PiJuAvrx0+UKhilWXvItcFTnpK0R+GZy2vj7cFbRD3OiT2hFeO6wytPy2/OcWgfJK/JxwyK/h8ijbyGrBATSc7463B34btCUc8+dTiUHBlvI4CCFcLzkIB+M+LQlAUPhuTuyVaa+41hHqst75efjgfdbB3CVvvfRRx892yrOgV/Kl6PLBDmDWp2Q/j43Mut9DijzduvygusAyXK5LWeSP+ffNe/zeqeDkb/85S8n42tr/cizZjDW+2pHjw7ogMy87shWTfqVl8s/B8MuI0+9s/VI+eU0o9zSA/raf9+7XI8RyKCVZ8ccc0zaZpP3edQmvZ8tXiVLBeRxB5f6B+kFSKKdChyqfDiWtlB0WVrqrzxXks+tOqP3j7FyjzLUyPOr1du3FmR7IraQhhiLcKoSoVb6rL9H273hvNJY7U7/kjxhyxWiSwA6Feiq1F4cqFoCUAFWwMFJ+XLd2suoc/9G0gd1jyNAAnQO2pCi7am9IL+IGiHdhPSqO2AJQISkaal7Xx9r1fGcn8ueR8ALf5/6O9fdqUt0ydtuu43LM/JxDHAXAA2oBuDyvrhovcbrsCietcwJqH9fm/Byitfz8KX1G1HOGkmH9H5Z0vF43gFcgBQAbwEch/jG9DP6lmieOZzyyI+tuoXL1im6DO+v6SQ+TtYAXDmgg/yYAxLBFCrJXQAVyEz0ZM1NHDxV6p/k5TpcLgO5P4VyABcLkJ74xCemLGgrvMvnsa1jKBmqDXJemsd4eyzZdFhog/yHnN+tPKvpJOkFhX98K23jC3iRhRTeDzT++Fx9nr7RZ9/wdj5WJym1Y67B03xRh1cfmwbjHuTyw+VeyXbjUVJ9vuV5jz3HdoIeTDkZ31lAyNHJdaYhAFcL/3jXGAAXcgfwmtuN5hkPvI5Tzn0ccbuZ8vCFiloc21rORfOzVRYguxQFD52ZxadOLCxR5H9fzODvc9miZ9eCLG+Vg9SxNva1zPXEs9oRuwT6lI97pJ1Hniwjz1r543pwIDgQHAgOrBwHZMvFXoZ9CT0QfZw/zvV7bIliC8WxnIp0wYHgQHAgOBAceIgDMkw50MIn0AIMkNxX0MvIwHVCP7PiDnIHV7rw0D9Wlwl8QVh3JuGQDIlujOJ6qVw+qXSHMekhOZQwJmHQcyLcPgAYqLSloKf1KCfkhaEQ5+RqIvgJAAtnH0ANFCnIgVpusMTJCJADckOhg5ccrAD/iXiVk2+1881vfrO76KKLUhJ9r5LBJc9Dv4fq4KA7NwLqeY7wAEcNJMMkiqQiXrDlhYBaKZH9w6mKUxLDr5zcdnujUwEE3Si8UaKRFxzAMPQIDmC26HDj/aYCcDmw0WWA1wEHE/WDhtoD9XjsYx+bVrIjR1D+/Xu409ejefn7PJLDGADXPHWQvKKMGNrcaeBlKp2r/fTxRDJsCMDlhnzexap8jKAYSTXBysvgjjii0gmUQroab/M8/PdYR+pZZ52VZK4/S4QdbRXhK82PtC0U3WmtZ7V1CnzHceCGeaXZZz2YFecwJGez192jLOgZjgLqeAQ4vY/7vK8Uaa0EPHQHDbym3eSE3BKIlraEI8qN7wD8aDM5OchJTszW+pF3zWCcv7f0W44Jj67hdWeVuRxO/rzeyTXVXfcXmSdgUkVwVP4c6V/IijGkdkHalQBwTeGZt88ScJoyA/hAXuDwJ1pCzVkq2VMDMpUAXK3yuVVnbJF78KBG3t9uuOGGDkBbTi7n1N/yNPpNWsYzZNC6h6JNOoDLHak8w/cAhAnItSTLSON6r0ApPoaVgHs8h/5E1A3GG0Wq5XqJHCyEHEJH/tKXvrRBlMz8OXfSOhBw5513TlHLSO/8aqm78176HfnOo+M5P5c5j6CckL9P34/rknOl+Qr3fb7lek0NwOVtYqpuNqTXeB2WybOhOQF8qbUJ7nk5xet5+NL6jShLjcRrn/PW5twCcNEniSYNeAKiTzOHzvXPeeZwtfK26hYuW6foMpRD+gH187mZ6xYuW5x/pQVaJ5xwQrfDDjukKjLOsdWqE3N8IoQDnJWdQAsu4D3lKRHRGZF3UIsO7Xk6gIsxQ4uveD/gvVyvbx1DeafaIPylbnk7cn4yDnzmM5/xoqZz6Qs+V2nlWU0n2eilduGtb33rDOCmKI/c9nbni0fm6Rt99o0WnaTUjt3e4WOdVTmdYkdDTnrfcLkHeM0jxfGQR8bx+Vae95jfzl/Nr/LnPPqS2268nJLPLfzjfWMAXCU+zjMe5PUc+1vjiH8zf9Yjpmqsby3novnZKgu8fjpHb2PLSCLUvvSlL51tA1wDcK1VWd4qB+FTbexrmeuJ77Wjf1vKrCjZ3k+nypNF5klbZqcI2k1ObJuLDhAUHAgOBAeCAyvDAc1DAsA1wO+IwDXAoLgdHAgOBAeCA80ckGFqHgDXK1/5ym7vvfeelSE3BuoG4AwIQ7Sc9zIkujGbNKVy+aSSyay2xCM9JMOWGxQfvNN1UwBcHm6+BD5Qnit5xLhMdAa2dAQgU5rQUh4HcAHqwjAN3x2UgHNRAIerr766u+qqq1JVvN61b0hCfUe2N2O7HUjfC9CXztMN+ze1Dg5AY4sHtoDKyR15Mtb59kWkr9VF9SCNtpnjvEYywnHfo8bU0vdddwAXjuOcMNoDGoTHgOxy8hXQKNSseuujF7zgBWlbJdI48K7vmZJR3Y3m9GFtUZrnI4cIK2C1ypI0rBp/zWtek5zbGKH9GygPN3ICxMQhDpX6PNcdcCnjJ9drNE8dJK+8P9Xe49fd6eiy1tNwLrCIyzA3iDmgFtAsTpyc2NKOdgMwQNH4SOOOuPyZGqgpT+e/vU7qe7o/JKuZ22AwhNzA7wAueJG3fcl4H0P0Th1d1mt7F697ra7ivX9bRXmpOdl5p5x43m7dQVP73u5UpJ3zDneOsM0v2/3mhBylf0ECh7TWjzxqBmPuDZG20qHuyFB453VX1Kc8H1+hrLorzSLzxEkkkJzy59j3PT0d52oXnK8EgGsKz9QfKBvfkXFjiEpynWfkkJ0C4GqVz/PojFPlXh8/vL/VoqzwvMC3ebQrnK60V2QOY3KJ1Ee5h05GFB1kZ06AMgDaow/IkUIal6VyevoY5ltE5XmO/U35AajSd53o12z/iqOW8cTHexxx9HmINOhHkJfXt2NsqXttjJlHx/PylXQK9Skfg1PF1v/zsWVoIYie8ffp+3FP+mQ+99Fzrv+6XlMDcHmbmKqbDek1XodF8mzqnADe1NoE97yc4vU8fGn9RpSlRuK1f/eajicAV54X/RL9CXC40zxzOM/Hz1t1C5etU3QZ3l3TSVy3qAG4HEyqemhrZn7DU/jnpPmKA7gUxYl0eXo96/MXQE6AnVrJAVx5HkQMB8zgNM8Yqjbouq7n7e2RLeuJ0pWTdGOXk608q+kk+Tv991Oe8pRZhB+3O3h/p4zaqm6eviGbhr/HyzJVJym143333bc78MADU7Z9bcm3xVb9XO6VZL+Deny+5XUYe77ffvt1tFWo1NeUj3Qmf5+XU/KZ9FP5xzNjAFxXXnll2k6S9CJvHyVeKZ1kQm7D0P0px6FxxO0zGuvnKeci+dkqC8Qfom0///nPT7YfLTTVPR1rAK5S+1oLsrxVDsKP2tgnvVRpxsz1xN/aEfupIvW6XdD7aamP9MmTRebpC/3yOjAmMTYFBQeCA8GB4MDKcCAAXCP5HACukYyKZMGB4EBwIDgwmQMyTLmT2SfsDhjwFeG+wlsO7LEvd8O1DIl+jXxK5fJJZQlAowmuG2FVpimOF+VTc2Qqz5U6AmDB2Y+zIyf4BgGGgRzAxW83dsBTDJAyZGKUZnKuPBxUxLND5I7m0vfy51vq4AZqgAw4AXIqAbg8Slievva75JTK0yrSDtcxyOG0HCKMcE94whNSsgsvvDBFCeOH87rUlofy5b4MpGPACL7tWg0Ml7+zZFTXtpy0HV8Znz8rg6WXDQP1AQccsBFoi7xwAmAUwsDnef/t+igD2vqkxifaFkACSMbPvDz+e5461OSV518696gkREM5++yzS8lmfdOdIjV5TAZsrfqiF72oY8VzidzJ5Y440mIAxAElJ5Q720t55dfGOlJL320MgKvUJ9Xm+6KYuXNkCoBL39adWjLE9o0Fil7j7dbLUAOMDQG4SsZrfQPxQdG9/NvW3leqH/nVDMZ6V98RMDGRDGlD2ppoTN37AFyLzBMn0VoDcNW+X4lnagfeZvu+F/dKcp3rQwCuN77xjR3bvEK0Jdpeq3yeR2fk/VPkHulr5CCDPiCU+OxgrFodkANERFSEQ3+Gcgi0AvhWelteviuuuKK7/vrr02XXe+X0RDai00KMZ8ineQkZcuihh3ZE+Cs52hjLTzvttI5tokQexVQREeX4zHV6npla99oYM4+O5/wsjU3S/+edR4hH/j59P+5JR6qNLVMBXMvUa7wOi+JZy5wAvtXaBPe8nOL1PHxp/UaUpUYah71/1HS8HMBFlD7GV8hBk3qXzyt0re/oc7haulbdwmXrFF2GctR0EtctXLd1/pXmij4PLrVfySz1efQZZP4UYoERc6tWygFc/q2Rvch8B9DWxp/a+729ldqgPzfET9LmAK55eFbTSbxMpXPJauYymgPyLRm/8jnCPH1jyL5B2aboJKV27BGX+qK5uYwrAbhK7bsPcFHia981tyWUgIV6VnOnMQAunpnCP9KPAXCV5M484wHvbaGhcaQE4Jq3nIviZ4ssgEcsaEBmsWgvJ0Bx6D2y7dQAXGtRls8jB+FTbezTHGTKXC/ne+n3sccemxZBMt6gM0EuY1rkyaLyDABX6YvFteBAcCA4sGk4EACukXwPANdIRkWy4EBwIDgQHJjMgZJhyifsYwBcipjBy1mlXwLaeMGYuLONIlQzJJbKNTSplDFPRlh/5xQAlybKbDmEMXBTk5xklIOINKyUIiIC4AQMl4Binve856Vi5gAuj0707W9/O0V0weiHkQFHnLYa5GEBuziH/xhZ+ggHLkATqPS9/NmWOrgDq7YK2LcTUxSg7bfffgYugkcYGfuIemqVbl86d2DfcccdacuMvvQ4Z8Vr0vkWbG5MLhlI+vLVPRlI+U3b79vm07/tWEdDyajuoJPSyry8bNqik60SiRhAu4PoW7Rh2qSijcCf7bbbbgMAl682ZhsRjPI5+VYiYwBcrXXgvTV5lZcp/812KBhkIUCU6i95OniE3j8WwKXnAb3AB5xmtH85+bh/+eWXp61t3RFHm0fO7b777rPoiUTuwuA6dmXnWEdqqX23ArgEMumLwOWAvu985zsdYGOvew0go/HDDZlyUrnzSzzXsWRodwdN7X3eDhWFyp2ete2IPfoefZ5yt9aPOtQMxqrf0FH9Vn19TN1LYCR/z6LyhC/0jZzGynyeW+kIXLX2UuKZ+gP18UiHeX39d0muc1951QAlHhEAOYXcbpXP8+iMXpcxcs/T5+fe32rAZm/PijrqACIAWzgpb7vtto7tieAfpPE5B3B5Gdji8FnPelZyoBDRSuMjaYiqibHM9V6BUpCp69atS1nJiZt+LOgfetVuu+2WtugWEI2sfVzit4Ps0dHYshrnDeQr+tOF7N+YutfGmHl0POdnaWzSODDvPELV9ffp+3FvyKnr+q/rNbUIXD6eTNHNKMuQXuN1WBTPWuYElLXWJrjn5RSv5+FL6zeiLDUq8bo253YAF9vBMZdh7qDFPHmkHtfzp87hauVt1S1ctk7RZShHTSdxWbxMABdlYG7MAiEc5mxPPERsyzhWdy7l5QAu5tbIIfiAnIQ09ujZecbQUhtUvhy9PZZAFKSRbuxjQivPajoJ7+kj39YZADbfC4ARpAUFen6evqH5mi90VL75cYxOUmrHDuLJ+7W/A9kOsAFSNDmXeyX5vEgAF1vfsWgIKkW44rqDWMYCuHgOGsM/0rUCuOYZD3hvCw2NI/7tNdYvqpzz8rNVFjjAlDk1th505HvvvTctGvUIrssEcPG9WuVSy7ee9321sU/zsylzvTHlf8xjHjObO2oRy7zyZFF5AsSlnZSIRTLo6EHBgeBAcCA4sDIcCADXSD4HgGskoyJZcCA4EBwIDkzmQMkw5RP2MQAud2bVtkrAoMNe9hieAQ5h0IVqhsRSuYYmlYtyvBDGHic5zjiAOpuSAL7g6IAwlAIWyI3E7ljNAVw8p4k/IASiOmCAgzyKGr/d4FLbOhIgiox3t956azIq82zpe3Edaq3DDjvskCKPkYfAWZw7vfzlL+8wfkFKw6Qf5ymkCDXpR/aPsO4YxwGuYDAdInec4bTlW/QZEF72spd1L3zhC1O2KL2+zeEiAFwAmnCuQLmxOl20fzJ2col+0gf20mMlo/phhx3W7bHHHinJJZdc0tEGcnLnqkCQvm2nG3T9WfVfj2Tkq30//elPJ4CoP8O5G/Fl/MzT+O/WOpBHTV55/rVzAUNpbzgTcsKoT19FVrpTpCSPca7vuOOOybkEQCsnBxsix84888wNQD7IAaK8QKeeemr3+7//++m8Jr/TzezfWEdqyaHQCuCiD/Fe2ghgM/iUkzu2rrnmmu4LX/hCWulLtCCoFHUCsCXtD3IAF+8A/Mv7tEVgSmT/FL3B2607aGqAHDfQlwBcNaAf315OqptuuilFnWMlc0v9qEbNYGxV7D11uYicAcTBEarVvQRG8pcsI0/Pf8r5agZwqT9QH8qZbznK6nfaBTJF7akk13ne9QRtjcd1kY9ZAnC1yudWnbFF7qn8paODDBwQ4GkZ7xgzIPU317lKY47LNwdwsf0QYzbA8ny7LfQWvqEiFnzyk5/scGy53itQCtsQA76Fas5eAWT6wKc8j05O9A8IUDXjhROyhXLhHIcciO71RKeg/TEuQTgtPUppS91rY8w8Op7zszQ2SQ9ZawCuZeo1i+ZZ65yAdlVrE9zzcqqvzMOXIcd7DWRHWWpU0iFLOh7PC8DlTlu2pGIuDRGZib6pecg8c7iUYeFfq27hslVjT559SZchTU0ncb3K5bXzrwQ4mhqBizJIfqLXoZ/l827SUEf+IKL9CLybLkz85wAugXJp68wzGb8hj8zUOoaST6kNcl00xE/SlQBcrTyr6SQqT+3osgBwFeMC88+Svj5P3+izb7ToJKV27IvR2Er53HPP3ajatANFGPN5isu90pi2SACX9+scVKgC+/aWPt/3cko+t/CP98imkUdaG2pL84wHqt/U49A4UgJwtZZz0fxslQXSo+iLyBsAN06+ZeiyAVytcsnLO+V8nvfVxr6Wud7YMms8AITMPND7aas8WUaeY+sT6YIDwYHgQHBg8RwIANdIngaAaySjIllwIDgQHAgOTOZAyTDlE/YagOv8889PK6p4oa/QxaiEg4dJu5ODWW6++ebuE5/4RLqtSV7uZCqVa2hSKYPBohwvXv5Ndc72Txi6ICJe4Dh1whGCg1YrsksALneyErkGY14pDLeDbAS88Xdx7hE3fMu10vfSs611cCMnjgpWntK+RDg6MWqo7gJwcV9tgXMMfThMnTxKjztYPU3pXMYV7qHIsrIvN0xxDwDC61//+pnhPXfyujO8ZCAhjyF6xSte0eFAgvieGAlLZdEWb6TL+xnXalQyhDqwotQeycvBGYr2BSgSgx10yy23bBQVzY3XyA4cV5A7KXJDLfcx2GNw4gjlfE4Xs3+tdSCbmrzKXlH8KaMeN3PwJNf8ew4BuN7whjekNsZz9D0cZE4AsgBmQTKyu5y+7rrrus9//vPpfs572vT999+f7vX9c+eJ9z2eGZLV7vh3A/+RRx6ZtngjD8BTckzyG3Jgmo8jD97tEsgAPguQJqAJ9wWgy8GU3FPIf87dMXL00UfPHHQCb5BG5EZ2b7cuu2ogpiEAFzKPuuR92oGb1Om+++5LxWmpHw+6TGuVRXKmsD0ousG8AC7KtYw8yXcq4SBXFCLk9gMPPJCy6Gv/fe/wviEn1pj24nJVgD8f20sRj7xdC+RbkuuU1+up/FWPZz/72TOQD9fUr1rls8uiKTpji9xTHUpHd0bioEdmog84ueOEsQYnh2+nXJK/ACwAWkCuX5xyyikJoMW74HGuJ7tTS8CsUnvxLXmJZosO6AS4C5AXlEda9XScA8wC8I5zGPClxg1P54C1XC6LP9SFejEWKxqf59FS974+1qrjOT9L8k75Lmoe4e9Tf4cvQ07dGjjI9RfXd/z6FN2MsgzpNV6HRfCsdU5AWfvahJdTvJ6HL63fiHLWqMTr2py7BOAi35NOOqnbZptt0iu8f88zh6uVl+stuoXL1qm6TE0n8XFy2QAul3nXXnttijbkPELOAWKiTMg95sYcW6kE4CKvQw45JG0vxznzPL49gL7WMZR8Sm2Q6yJvjyVAHOlKAK5WntV0EpWn78jYB8iQNsZYxjjmfULPztM3+uwbLTpJqR17xG7aETqG2zuox957752iXnLuC1Fc7pXk8yIBXER0Yl4CnxnzWYzkW3tSNo8S5vM7L6fkcwv/eIfmB7ldYKgtzTMe8N4WGhpHSgCu1nIump+tskARaGkjOQCWtoMc0zbgywZwtcqllm/NM/O8rzb2tcz1xpb/JS95Sccf3wrAMG3oyU9+cnq8VZ4sI8+x9Yl0wYHgQHAgOLB4DgSAayRPA8A1klGRLDgQHAgOBAcmc6BkmPIJuwO4/DrGU0AIMtw4SAQnECAA3cM4AYBLBh+Mh3JK1wyJpXK58ac0qVyE44UxFwc5ZQXEoHJMZuyCHqAc8JIjJOAHxuNnPOMZyUlHtDCRr9DVNd8+UteItHDhhRfq5+zowCLCnRNliUk97wM88bSnPS2lzZ1b4lNpi4F56nDUUUelevJS3nnBBRckByrbxREJDGOiyEEk7nAGtASwTaAUDBPHHHNMirLGs4p0oXz6jryPNgs/IIzGAEkwUgKiAQzDd9lll11m2ZTAcM7nUluePdxzQlvFKarIGBj3AeawbRyGXxwobM+Es0x0ww03pJXi+t13rBlC5YjlWcBBRHeS8fyggw6abcnnYDH4QfuBKCdtmm0rAdogVzD2qI3T3jBicYQkIziHx2eccUYCovEtcJrgyBC5Q1PXSseWOpCPyuJ1K+VfuubRBjDQA3RRhD+cC4DcxIMhAJdvG4kDHwebtpeEL0euB0LRFiFtc+EOHwdwkcZla8n5TpqcHCRG38RITbmpm+dXat+tAC7qJucB5bnqqqu6q6++OhUNJwWGU8L3Q7ks0rfjHmAsIibAE7afBYgicgDX1ltvnfqYvstXvvKVjqheyFQAGnxTEe1VwEN30LQCuMiXyTIr8QFp4eA5/vjju3Xr1qVX5pHcWupHRv6t2AaN6Jg4CCHaEXIA6pMdMi4jB3DoAHSBanUvgZHSA/ZvGXla9qNPHdjkAK6+9t+XufNbTqwx7aXEM9o8gBuNAYC0aJ/0QcZAgES0Xdom9aA91eS6gyMBPJ999tlJrgMGItKl+gB1E4CLc293U+Rzi87YIvcoY40cZEAa5BiLC4iAgaxB/3j84x+fHocnAia6k4woU2yxxbOAS5Dj6AAidzA6IPTuu+/uzjnnnJnTn2igb3vb21LEUp7FMY1sL7UX7gs4xTnjMLofOvfjHve4JAelF1522WXd17/+dZJVie8pexORGYnQKIJHtA3aGGM3CzSciGS6//77+6WuBHhoqXtfH2vV8ZyfpbFpEfMIZ4a/T/2d+0NO3akALvJcll7jdVgEz5AlrfOavjbh5XRet/Kl9RvxLWokeek6pM+tfc5dA3ARCRkgpWTyxRdfnLan4p0+t5gyh6uVl+sqM+djdadctk7RZfw7uk7i4+SyAVyAtpGxGj+ZWwA0gAAMIauR9xCynKjVIv8GRExiDjhENQAX7yc/2j2kbcE5bxlDeU7f09sg10XeHqcAuFp5VtNJVJ6+o/NN6Up2EO75d5nSN/rsGy06Sa0du15Bf0HfUFRVdArGYfV55kECm3t/KcnnGoCLNgVPyNOj/ImPtaPbZZjzAdYBNIyOQBnp+6IhAFcL/8hbshndFj5hT0DvHdOWWscD1zkUrUj17DuqrLXtyUsALvJrKeei+dkqC7zs2MiQn8wPiax/+OGHd4xhIrdHDr2vJZpiq1yifC4zxsryed7nfdnHvpa5HuUX+LqvfzNXQO9FDmDjYC4yL4BrGXlSn6DgQHAgOBAc2DQcQC+F0GXwgzFmoPfxx7l+jy3d76wPcb9huI+xT67ydDKorfJiRvGCA8GB4EBwYA1yoGSY8gm0G5NxaDGZddK2ZkwucQJhmBLhCCc6EgO6KHcO1QyJpXL5xLZkpFqE48VXd9dWsasuK3V0gwXvFFhGfMWRv9VWW6XiYMwiCgqOQSc3pnDdjX+ejtD3GIWVN/nxHeUMJC3XMNi5Ubr0vTzf1jrwXtoV4IUSwQuBqRzARVqPksFv0kJKz3kO8uDaEOHQJvKOeNSXHqcvkcowcjq5UajUlj1t3/mee+7ZHXrooX1JZvcwcLJqeizVDKEAg+iLXv+8TdJGAP9hJIfgOUAD/44YlBQ9jTQ4E3Sf/ADYsM1U3iZJ68/6+VgAV0sdeG9NXnFvDOH83m677WZJ4RMkXvKb8yEAF8/g4MeRBPEc/IPPLoPJBycU/OwDcCG/aZPqGzfeeGNH9LQhkmFa6RTta0hWtwK4eM8BBxyQQFd6J3XHcK+ycx2ZRVtHhos8Ko6u+VG8dwAX99n6CydRjfQcx0UDuPROvcN/Y0z2yIKt9fPxXvkryo5HcKIv1raa9XETozNATmgeANcy8lT9phxrAC7yqLX/vvy9bwhk4A69qTxjm14A8iLaCiSZwrmib3Fek+tPfepTUyQ60pTI26ADuFrlc6vOOFXuleqiaznIQNe9rlxDfqLz4LSDAEUDHnYek0YyiOcZlySLkSm0FQBQjCEYuyDScY3nfCyUHCVNqb1wfd16ICd6lfLiGnLQfwP8xIEzRA6wJi11ocyMx15HRQXz/KgjgE2lo06lqBxEi51ad97T18dadDznZ0n3WsQ8wvnj71N/93rVnLotAK5l6TVeh0XxrHVO4LwTn9VfvJzO61a+qO1N/UYqV+lY0iF9DPY5dw3ARb5sewqgAKKvEjkDHTCXx/RH7g/N4VJGlX8tusVY2Ur5cl3G+aEioZMgKwWiXTaAi/d6VFx+U1aX61wDNIuODY9FkiH8ZtzQ4iHdLx0diITuynxNxDc97rjj9DMBhpm7to6hpTY4y3z9ifN/CoCLPFp4VtNJvEy1cwcKkKYEMtazrX1jyL4xVSdxfc/bMeMtupX3VcZixlYf1/Ot7l3uleQz7URzfwdU+Va2vIdISWOI/GhDeTkpI2Wln0gf8Pd5OV0+T+UfZfS8+K0FLWPaUut44GD1KfaUoXGkBuBqLeci+dkqC5xXfB/aBKR2gbx0+7AiyQ69z/WGUltnLCFfFlQwZoha5BLPtshynmt9n9effCDNx6fO9XhWsmuofysiOP2I8WpeABfvXkae5BsUHAgOBAeCAyvPgQBwjeR5ALhGMiqSBQeCA8GB4MBkDmh1jgzgZEBkEQy2kBuT+e0rPvnNNlxEc4FYzYdRRRFQ0sWH/mFUY1U+f04CFsn4onulcmHAxAAHlSbummijYGBQdfIoVES9YYVmiRzksFoAXBj1iEjEqignDCJEiiAqF4AsAHbQz372s2R08LQOQpChxO/7OdsLEulFK379HqtBMejec889fnm2ysvbkSeYpw4YY1jVudNOO82MmNSdelB3GR09DLvezTZCAK5y4nlWMn/84x+fRcDI0/T9pp0QUaLU1nkOYwmRj4haUyK+F8ZTqNSWS8/UrtEniFaj75+nw6kLkIrvNoVY8ctKW0hATT1PG6GvCzio6xzp66effnr34/XRspxqz8Ar5AhtGcevDNVf/OIXO8AgEFFQaJNyiCtfHFZEcuA5aCyAi7S18nCvVoeavOKZMYTxEp76CmU9R+QjQCu0cwdw1eQxaTFmspVoiVidTVQAbflGOsoP5RG4uLbP+ihgAt7QP0pOeNI5YVBEtuib3Xvvvd1pp52WnE19sroG4PIV6GzDoSiO/k7OAS4yRum9fp96M35gwM3J89c96kqEGsYIDOYYlnEuOAFuYdWw9zHayK233pq2MuI58hGAC5mFIRn69re/naKteX6cy7jJOTKMfuBOT5w0ACuQnU68l758++23++V03lI/nDEAC1k1LGL8ZBwlUqGiLpbajNJz5BnaJBGJNFbV6l6KJuV56XwZeSrvsUeVgfQ4rX75y1/OHq21/1mCwonrMXJijWkvfTxju5UjjjhiBiDSa2lT6FwAuER9cn39YrgECJaTRc8gywGWCSjmAC7StMrnFp1xqtxTHUpHB60ReQv5R39wQo4wxjgYlPvI5YMPPngjGYQuC4B+2223nenRpEceAHqnj9NP+eYlQrci6h5AAajUXvQc/YwxwOUS95BF6LkXXXTRBsACPVc6EmUNp1v+7UmLDoEuo2138+eRX4ynUB9obGrdyW+oj03V8ZyfJd1rEfMIyi3y96m/cw9AO+2/FvGyBcBFvsvQa7wOi+LZPHOCWpvwcjqvW/nS+o14X41KOmRNx4PX69YDNZEFAEad6KeUTzq4j7W0galzOM+7dD5Vt5hHl6npJMhW6VUOfHH+MadjMY/TWKd/yXbQp2sCoEfWMxd1kgzhmuS+3y+d06Y1vuYALtL7+O8yo2UMLbVBL9MQP0lLGflOPldRHlN51qeTKM++I31Dc/HSVtL+bEvfKNmjPM+pOonre96OyZPvSXQ39IecGNeZE7OYwsnlXkk+850WCeDi3dgvsEv5vIHrlJFI6QBM0UscwOXldPk8lX+8hzkX8xPNj5gXMY8Z25ZaxkkHJZXsbJSrREPjSA3ARV4t5VwkP+eRBYqinPMEmwTAote+9rUd24FDzFXQI4fetxZkueo7VQ7yXG3sk+N8ylyP/ATgKukQ3BcB2MKmBzHfQ++ApsiT9ID9W0aeln2cBgeCA8GB4MAKckDjUETgGmB6ALgGGBS3gwPBgeBAcGBFOYBxgAgEDOQ4a3DuOD3qUY/qcIxhUGOQZzJIiOyg+TiAwQpwB6seAR8BlsJYBgFsATCBMf+HP/zhRs7GqW/GOUCoc74jBjqc1kT2YtXvPDRvHSgL9dfqZIwMMjD4dmpeRvQojEQ4mTFiADAAsIHBb17CQEmd4BVRNCgXDlhtbTBv/lOep78BthCPcCrgYMCgnffRKfnW0gKegf+0O8AvOKcx1npktvxZntl1110TvzD8IxcwhIr4Vk94whNSe8YB7QAe+Et75D7GPvoA8mceaqnDPO/Ts8hQDHEAAVntSF3gRwvRrpUX7Zs2iAxAPqwEYXDEUEd/Qtb7N1vm+zHc817+kHuAx9jKxgE2pfdLVtJfkAW02RLYq/QsbZDn4DHgQeikk05KIC45EErPjb3mTk9tfQlIgy0eKTd1xFHb159b64ejgIhuyC7vk2PLvqWm21Ttv8Rvts9DRiIfkSs/+MEPZkCgUvrSNcZ+5Dr9Cv2N8SwHL5Wem0c+t+iMy5J7yGYcL/RvQJICwJbqzPjPeIZMIDoXwAFteUR6nLDoyuQFAE79Fl7R1/lWOELhMzKFNMikqUSZGfsZP/Xdx8o0f5fqA2+Rr2x1Tr3gQ19+AOy1bTSgYeRwjVrqPtTHlqnj1eqxmq9vKr2mhSetc4KhNlEqy1riS6n8U64tYw43RbdYhC6zWnQS5pw77rhjmuvAA8ZDwPt9uiYAOsZQwEWMx8umljF0mWVq4dkyy+N5L6NvkP8idRK+J/YedDraG0Av5nXLaEsAjJg7Em12Km2//fZp7Ien6Jvoi61lbOGfFvZhd2C76inUMh7Qrok46oDZKe9sSdtSTt6z0vws1Y12zFjAlom0Yew6bnvDfoQOTBtnfrtsapFL88jylvfBg6GxbxFzvWXzOvIPDgQHggPBgc2HAwHgGvktMUoFBQeCA8GB4EBwIDgQHNjSOEAUIBx+AJHOP//8jarPVoZEDYHYqnClQCsbFSQuBAeCA5sdB1hxDZgDuvjiixNAzSuJcZbtMZBRAJ+IrDcPlZye8+QXzwYHggPBgWVyAMct25oRVaQUjWWZ7468gwPBgdXJgS1ZlwHkBSgG4C5bWwYFB1YrBwBgETmYCNhE/gzq58ABBxzQPe95z+suueSSBOLsTx131zoHQpav9S8Y5Q8OBAeCA8GBRXAgAFwjuRgArpGMimTBgeBAcCA4EBwIDmxWHHjf+9432x7ozDPP3GDrRqIOsa0HVNr2bLNiRFQmOBAcWHEOAA4FJAr94he/SNuQKIoOq6L/5m/+Jm0byP2rr766IwrgPLQlOz3n4Vs8GxwIDqwsB4iARLTFAw88sNt3333Ty9mq07frXNkSxduCA8GB1cKBLVmXYYth7PdsX0wk1aDgwGrkANs1YmMBhH366aenCEmrsZyrpUy77757x5bNRCtj7qcozKulfFGOxXMgZPnieRo5BgeCA8GB4MDa40AAuEZ+swBwjWRUJAsOBAeCA8GB4EBwYLPiwP77798RBUdEiHzC8xNyneg3ENtInnPOOWmLR6WLY3AgOBAcmJcDWn3LEcJwz9akbC+GDMLxAREh8J//+Z9nW9qmiw3/tmSnZwO74pHgQHBgE3HgX/7lX9KbJQMB0b/zne+cbRO5iYoVrw0OBAdWAQe2ZF3mrW99a5qP3nDDDavgS0QRggNlDgDCPuGEExLomi0Bg/o5wDbRz33uc7tPfvKTacvr/tRxd3PgQMjyzeErRh2CA8GB4EBwYF4OBIBrJAcDwDWSUZEsOBAcCA4EB4IDwYHNjgNHHHFEt9tuu83AEl5BwFvnnXded8cdd/jlOA8OBAeCAwvhwKMe9ajk5Nhqq62K+S0KvEXmW7LTs8jcuBgcCA6sSg4A4BJ4i6iEF1xwQfed73xnVZY1ChUcCA6sLAdCl1lZfsfbggPBgeBAcCA4EBwIDgQHggPBgeDAojkQAK6RHA0A10hGRbLgQHAgOBAcCA4EBzZLDqAL/X/2zgRst6n8/4uUqMtPopA4SubMcVAyy0wqKpUQSmhWpyQNmjSojJE5UylDOKaSZJbhZCqhTCVDrlKh+p/Pcr7P/z7rrL2fvdezn+d9zzn3fV3vu/ez99pr+O617jXc332v9ddfP0CmgEhx3333xS+cf//73w/s9WaWBMwL5Qg4Ap0iwJatyy23XFhggQXCU089Fe68885IHH3yySc7SwfPXquttlqM76677gqPPvpoZ3F7RI6AI+AIdIXAxhtvHOaff/7w0EMPda4Hu8qjx+MIOAJjg4CPZcYGd0/VEXAEHAFHwBFwBBwBR8ARcAQcga4QcAJXQySdwNUQKA/mCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKNEXACV0OonMDVECgP5gg4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjRFwAldDqJzA1RAoD+YIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao0RcAJXQ6icwNUQKA/mCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKNEXACV0OonMDVECgP5gg4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjRFwAldDqJzA1RAoD+YIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao6AI+AIOAKOgCPgCDgCjoAj4Ag4Ao0RcAJXQ6icwNUQKA/mCDgCjoAj4Ag4Ao7ATIbAG97whvDSl760KNeXXXZZYEC94447hjnnnDPcdttt4aabbiqKq4uH5phjjrDllluGueaaK9x9993h1ltv7SLaMYnjhS98Ydh6663DIossEuaZZ55w2mmnhfvuu29M8tJVossuu2zgDznnnHPCf//7366inunjoQ3SFv/3v/+Fs88+e9yUR/kiQ1dffXV4+OGHp8vbKqusElZfffWoQx566KFw0kknhcUWWyysscYaMdz5558fnn766emeGcsf/cozlnmbndJGD6y22mqxyD/84Q9np6IHdPv2228f6K9+8YtfhAcffHCmKP9Y9K9bbbVVmG+++cIdd9wRbrzxxpkCp0EyudRSS4UVV1wx9o3nnntu7A8GiW9WeNbHDbPCW/QyDBuBddddNyy00ELh8ccfD5dffvmwkwtrrrlmQF899thj4cILLxx6ep7A8BB40YteFDbeeOM4JmGe+Zvf/GZ4ic0CMY/FWCgH28tf/vKw0UYbxVusx6Tzs9wzw7o2M4zpV1hhhfCa17wm/OMf/wgXX3zxsKCY7eJtOk4nHOtz11xzTWC9oAsZRpxd5MvjcAQcAUfAEegOASdwNcTSCVwNgfJgjoAj4Ag4Ao6AI+AIzGQIfOlLXwos3pbIiSeeGElSX//61+Pjv/3tb8P3v//9kqg6eeb5z39+UF7uuuuucPjhhw8c74QJEyI5DSPFE088MXB8TSJ4wQteEL74xS8GjpKTTz45XH/99fo5Ux4/8IEPhKWXXjrm/eMf/3h45plnhlKOhRdeOMw777zhqaeeGtMF7TaFe9e73hWJUJCdPvGJT7R5tJOwVZhBKtthhx1iGj/+8Y/DFVdc0Utvm222CRtuuGHvN5PrAw44IOy88849AtdBBx0UDYq9QCM6KSnPiLLmyUxFQPUdMD70oQ/NVphgQNp7771jmc8888xw5ZVXzhTlr+tfh9VPfutb34pG5TvvvDMcccQRMwVOg2TStgv1kf/3f/8XCbKQe++5555Bop8pnx3VuGGmBMczPW4QGOt2+uUvfzl+7PHPf/4zfOpTnxo6Lp/5zGfCggsuGMfZkyZNGnp6nsDwEIB8s9dee8UE7r///nDIIYf0Ehvret3LyDg6qRsLDSObTeYzfAhx7bXXDiP5RnHasct4HdN/+MMfDksssUR49tlnw8c+9rFG5fJA/RFoOk5XH8U49tBDD+0fcYMQw4izQbIexBFwBBwBR2CECDiBqyHYTuBqCJQHcwQcAUfAEXAEHAFHYCZD4POf/3z0cFGS7VmdwIW3HggpCF8kn3DCCSUwtX4Gj0YshiIYbf/+97+HU045JXohaR3ZOHpgVIZYSHwssP/tb38LBx544DhCoDorBx98cCSd3X777eGoo46qDjikO1WY1RG4vvCFLwTNE//1r3+FP/3pT+Gwww4bFwSukvIMCVqPNoPAzGDsyWS7k0uzGoFrmP1kU8NQJy9mHETylre8Jbz+9a+POZER9CMf+UhYfPHF41gAA+TsJqMaN8xuuHp5u0VgrNupDNlO4Or2vc4OsdURuMa6Xo9H/EdN4Goyn3ECV/+a4gSu/hiVhGg6Tn/zm98c1ltvvTiW3X///Tvxzj2MOEsw8GccAUfAEXAEhoeAE7gaYquF+YbBPZgj4Ag4Ao6AI+AIOAKOwEyCANs54a0olf322y/w5S1y7LHHhgceeCANEp588snoHYPFRWSsPXCxrcFOO+0Unve850Wy06Aeq4ZpmJ4BTHPBejCCNDarbGex8sorh9e+9rWxpCw2D2sLxarFbgPxuDq19ezoo4+OW5GOOoNVmLFFxyabbBKzw3ZvfJ0v0aIteuCzn/2sLge2qVh11VXj+z3jjDPi1869myM6KSnPiLLmyUxFwAlcM58Hrqr+1eqvronO0jGziweuzTffPGy22WbTkbVmdwP6qMYNrpgdgUEQGOt2yjiN8dqjjz4aLrjggkGK0uhZ98DVCKaZIpATuNq9pqqxULtYmoeums/YD2ycwNUfTydw9ceoJETTcTre/vEuT/th290utt4dRpwlGPgzjoAj4Ag4AsNDwAlcDbF1AldDoDyYI+AIOAKOgCPgCDgCswgCn/70p8NCCy0US/ONb3wjetfJFc1+iTrWBK5c/ga5NkzDdF2+3vve9waMlghejjDIuDRHoGqxu3kMow0pryv/+c9/wkc/+tHRJj4ttRLMvv3tb8en//jHP4ZvfvObY5LvqkRLylMVl1/vHgEncM18BK6qWjDMfrKpYagqbzPbdbwT4FHA9gVjTQyZ2TD0/DoCY4HA7NZOncA1FrVsOGk6gWs4uHYVa9V8xglc7RB2Alc7vJqGbjNOZ2vwV7ziFfEDUPvhV9O0cuGGEWcuHb/mCDgCjoAjMDYIOIGrIe5O4GoIlAdzBBwBR8ARcAQcAUdgFkGglMB1+umnh7XXXjt6WlpggQXCY489Fu69995w3nnnBbZZywmevjbaaKMwYcKEMP/888eFnTvuuCNcffXV4a9//Wvukey1OeecM7ztbW+L92666abeloNrrbVW9ApEXn7605+G5ZZbLkycODG88pWvjN668CqEty7r6QpPXi95yUvCMsssE+P7xz/+Ee6+++5AvDfeeON06a+xxhqRcMWiFMJWcrfccksMxxaIVvBMxB8TkTPPPDOe43GD8TZfIxIHaZI2Qp7+/e9/R/zYSlGy2mqrhTe+8Y0xHCQ6tk154oknwqWXXho9oSlceiTtddddNyyyyCJhrrnmiu+E8pMXns9Jm/Llntc10qZslOcnP/mJLscjedl+++0D24uBBd65ePe///3vY9lTHKd7eNqP9ddfP7zqVa+KdY8vPIljypQp4eGHHw7nn39+WHjhhcMWW2wRQ59zzjmx/LxniIrXXHNN4JqEurHllluGl73sZWGeeeaJrv55ZzfccEO4/PLLe97D5ptvvgABC/nLX/4S86o47HGDDTYISy65ZPSsglcq6pPkoIMOit7u/vCHP4TvfOc7sV7iTQEj/mmnnRbbhJ7HW94jjzwS6/ZFF12kKGY4brjhhhEH8g+2eMmCaMV7tu2wH2aLLrpo3PKABKif1JE3velNMU+0IYT4aBfE/+tf/zq2LxEQec+8bytt6yD1Yeutt4748bUt7xX8fve730VPExbLkvLYvFEXeO+0Q97tM888E/H+1a9+Fd+9Dcv5tttuGyCwoOMuu+yygEEFL3PUNcoNJnguQye0laZ1UPHivYd2jbfEyZMnh6222iqgJ+aee+6APreCrl1nnXXi1mzUqT//+c+xrdAObP2wz+DZkHrJuwUb2thTTz0VHnroofCzn/0stjMbvu48JXCBG3nFgwiYkx/wpB/ISWlellpqqfh+eWfgQptGP1xyySXxHebSol/ZeOONo26ifpA/6h59Bm22ShZbbLFA34NOIg7CXnvtteEFL3hB2Hvv9gSuUh25/PLLxzZM3UCX8c4o87nnnpv1rFlVnlz/2qafrIqX9k2ftPTSS0fdT39Ev3frrbdGYij1LOeBi35vu+22C9RlxhoIZGe2oaU+1klJn9am7pTqBdrAu9/97qg7jjvuuNhGwQVPqQjjCuof3jboH+rkPe95Txzf0GfRjt75zndGrOhX6WeslODBe9t0000DuKCXGYNQvxmDvP3tb4917brrrovvkbR4x/T/9OWULRXaJLgh9DUPPvhgPK8aNwy7fDHxin9t+qSKKHqXGcfRNukj6D9XWWWVsOaaa0bdDKb33XdfHLdq8br3oDlp8/7Ak7/cGPSkk04Kd911V9hll11i7OTn3ql9G309aVBu6tIVV1zR88ZJP0KeKQO6hfB4gKoaT7YZu6JzlJerrroqjsEZL1PnFlxwwfD444/30rPjAANNHFs1nV+UpAf5BQzatlM9R17fWcfnAABAAElEQVQZX4C7FfQAOp/2znuxQj/EmAih72KcQd9MG6Ld/PKXv4z3GEMMMpYED7Z0Ja+kCd4333xzHAt97GMfi++Adz5p0qSYnv3XdAyqsS3PMn+075HxAOMxhHKlHluod9QnxoToxKeffjqG7fevTZ8IpmCIMJbjfaDLVlxxxaj3GKug56ifVdJmjltSB6vSbXo9R+BS/Wxbr5VmP/1c2h+UzOeVJ46MNdFnrDWANXqKMTNzFVv37DPpOc/l1hoGbW9pOuv3mdOmBC7KwfoL/azG9bfddlvUEWnc+s34qe1cQM/a41iN6ckDc1LaI+NA5gaslTAOZL3n2Wef7WWzjsDFfIB3yrgSsWMQfqMz6JeZEzOfZoxKu6ffY85HmugHSb/6r3Btx7Fj1W5Kx+kqp46MPcAGOeSQQ3rjCN0vad/DiFP58aMj4Ag4Ao7A2COgOTB9Lf08a0OMxfjjXL+b5nSOqQPB6S00TZ8c5+HorF0cAUfAEXAEHAFHwBFwBGYfBEoIXCweQlxgYJ0KA29cp6dkDhbgWfjLPYOh70c/+lG48sor0+iyv1kI40tVBGPI4YcfHs8/8IEPROMKBk8MJRjMcoLR9+KLL4638DqWy9M999wTDj300BgGQzxxswCaEwylLFBZQgTGKMpM2SDgYIySYBgR8UXXdDz55JMjYYDf+tpQ99IjBnDStcLEZtddd40Ljfa6zsnP8ccfH40zulZSPj2bO+o9cI8yYAhBqDN4ntLCabxo/v3tb38LfOFZZRBU0P333z8upuq3jhh1PvGJT0TcwR9hcRbswQVhkZvtCxGMyBiVqoQ6TF2mTjNxBGuOYPipT31quvetOFSfCEM+ZWh68YtfHOMi3CmnnBIXhO2iPPnESE78qYjwZa+zaL/PPvtkcSAcbQAjGYZ2pB9mNi8//vGPo+H2q1/9aiS/xAjMP/A44IADgt0CFHIaRj+kpA5C6KPe6D2Z5OIpeB511FE9ok9JeRQnRoy3vvWtlWlhSD/iiCOme79f/vKXI1EB8h7tfPHFF1d0vSPGxe9973u1ZJ9e4Gknbeqgnv385z8fiVXgjYFhiSWW0K3woQ99qHcOUbJKB1IvIXegQ6xAICF+dEKVQLiyJMiqcFy3xh7IULznnEAoO/XUU6e7VZoXjAYs9OeEegTpLTUQY7RmO2HaaU4w4rDNbSoYvXfYYYdsXaLdQupCIFQ26d9KdSSknde97nVp9uJvyozRGUJpE8n1r9Jr6fO2n0zv2d8YPMEXokIqEAMpN20/JXDxHCQ4EZvSZ1ngO/LII2eox6V9Wtu6U6oXMJaj7zAkU/chQeQEPavFzNx9rslDIuMKjPEYKBHiFqGzFA/w33fffbN9NuRAiMP0Wfa9yXhKHqw+4jcCsX7PPfeM57RDtcWqccMwyxczUfGvbZ9UEU3v8uc+97lIJoAIw7tB56RCv42egcBnpeT91Y1B0bV8oKBxNO0YImpubIZxHKIA7y0Vxkhf+tKXYj9k77Udu1qdg64VMcHGyTm4kR4YWmk7vyhJb5tttilqpxDiGCsh9IGHHXZYL+v03bQXSeoFmb6FsRmibbelcyD9MQ5F7Pit7VgS3cq4WeTYGOG0f7xfxu/0iymBq+0YdLfdduvNSyDR8IGExJaTNpB6qKWckLzoy7jHWKuftO0TIWeQRwRSMUSR3HiUfp05pyWM8Mwo6iDpDCI5AldpvVY++unnQfuDNvN58sR8nn4vV5+5z3s75phjenMJrlWJ1RN2rWGQ9pZLq818BmIt7zEnkJmYJ6VSMhdI49DvsRjTo3+oR5Asc4KOYq7OGgKiOse7hoAqQddBQuXDFIS1m6985Su9tlw1/kPv3Hvvvb2Ps6zO7lf/SadkHKsy8HzJOKptu1E+S8bpPJuTr33ta3EumauXGu+1zecw4szl3a85Ao6AI+AIjB4BrXk4gasP9k7g6gOQ33YEHAFHwBFwBBwBR2AWQ6CEwCUIWHjB+MqCqQyw3EuNFJCV2C5QAkEHL1l8Wa+FNO5ZY57C5o5Vi6paELLPMAEgPdKyRmBtWYhxkjxwH8FgQXgW8PEmhuD+XYvBlJkvxCmzvB4RBlIH4UTYkfGMeyz+yRDAOV9v8pUnXwbL2MrX3QikGwwEeNXBGwxCmnioIm48BuG1S/FBDsPrlMRuy0ha4IwnJ4xzIicQH3nFGIaUlE/p5Y72PVgCFwusyoM8RYEjJAfhQF4x0NUJxh4M4OAPDhhzeA5CC8Z8DCngnwp44EEKAylpQoDiea5D2uF5vrqlLpAvxJLkIBKIgGJJgErHGoDsc9zHuwBf95OWDFDWCKA4WIimLjAv42t+CXURDwwSGYP5zTN49aDu8p41pyMtFjtpo/0ws3kRgWuvvfaKdQ3DGUL9AyPigwRYReBqWwd597RHkYYgSUESpc2hV1Q3KCf1CSkpD8+ttNJKkeDIOcICPu0ZYzrlFIEuJabIaPrcU8/951kw4TnVF35jDG0iJXWQeEXgStOgHbC9E2KNl9QDMMXgS17xzoRw/bvf/e50hDNreEc/oIvAn3ahtsuzGD0gcPQTa+xRWIwseFBK9X9KcirJC1/Ov+Md74hJqX+i7OgK2oYEIzr9FIJXIeqf3j26nDpOO8JgJF2b1gk8IOy4446KsucdEf1s+xoCpGXrPZSclOhIizHvlPoMvvQVtCHlH09ceE3qJ7n+tUk/WRUveYCIpHzwXmg71EPb//O8JQKl74W+HD0HtpRN+gK9h4HfGvVL+rSSulOqF/DmgNEWfUpfIq86IrhpPACh2RLDcxjLgJjeo+9HVyAleKT4gzNtiTYBecOKfW+DGh6J144bhlU+m//0vKRPSuNIf9s+W/cYa1KvqQ/qQ2jDtBeuS0reH2MgxkIIcar9cc6HApDEROBSOhi8eceMTdN3TBgW1umH7RiUsSnEZUnJ2NXqHMVDPuljaBP0Fcp/qodL5hcl6UHEKmmnvFfI/+Tfkq4oJ9uosp2qBI9aZ511ln5GIgpjE9u3S+fYuOz4TQ83HUvS99kxI3oD7G3fR5wpgcvW5yZj0BVWWCG8733vi9mzZBguiKAVb079lxLZ2Lqb/pm6efDBBytY5bGkT7Tjd0VMe6A/pT3YvgqyrPUuOKo6qHyVHnMErtJ6rTz0089d9AdKq998nnC77757JN9xTp+F1zrqLiQaO4+nzjFerxOrJ2ydHaS95dJrM5/R85QN/ci43BKbUnJk6VxA6aRH27Z0b5hjetKwcx6N69F/EyZM6JGOGfOiyxDVOUvgYtzIepd0HWMs5sfEh9j+kt/EzxyIOqO5AdfRjcQv6Vf/03FU03GsykA6JQQu5Y9jk3ZTOk636aTnqitgzJjOjtHtOpGea5LPYcSp9P3oCDgCjoAjMLYIOIGrIf4azDQM7sEcAUfAEXAEHAFHwBFwBGZyBEoJXCx+sajOIiKCRxoWnDBSWKIF91hwlzEoNWTjRp3tmXiuKfmhalHVLgixyIaXLUtuskYCyDAiaLH4icEMST2t4GFFW+dh2MCwIOIT+SBOLQpbAlq6GAhZAOIQC1QSS3SB1KOFRO7rK0OuYbxTmtwDaxE1WJzmPSAY1QgLlpQfkgLGNQlfoorEcPbZZ4ef//zncduUkvIpztzRvgcZYjHAfeYzn4nBU5IWi6MHHnhgj7CEYVv1Khe/rmF85B2weMzzkpTABVkM7wXW2xALohOmLv4ifE3P4ryEhV55IbJ10noOScvAs5bgJS9bipOyg4EldqVGAIzgfBlO+0FY1CcMAqFJxlbaDFtXIUx0IbxZI78WOLlvjfj8rsLM5kUELsIjWqC2Rnqu5whcJXUQo6i2z2E7VbaTtAIhCgIRwnuhTJK25QErFtQRtQHFxfuhPcuQfuyxx/a2BZPRlLAYBTDe/fa3v42PYkCh/lEXkabkppI6SPzWmEE7px2je2SMQp9R39AD5BVPgnZrR7ZsYjsixOoPDJTEjUCwASvil1hPDVbX6X7uaOsi9/ECxXZdEhsnbf6Tn/xkTLM0L+hFeUezJC3Ss3rBGuOsvkoNsxAI0Zsi1lAPRK6x/RqkXDzVSNiiBU9vkrTf03V7LNGRkDwxpPOu6St417xTCV65MOAhGE/o52xfonD2WNW/1vWT9vn0HCKoPEZQD8kj9RJZffXVox4h/4jVMXvssUckOnMdrxOQcyW0Uetlgy1u0LlIaZ9dUndK9QL5h8CJ/hYRUunT5qyBUGWuOko/c586AA6MY9R2S/GwHnMYv9BPKs6UvGjfWxeGR40bKNOwykfcVTJIn1QVpyW8gCNjSTsmsWNx+xFE6fvrNwa17Zw8M8YAa40/bH65T79M/4ywbozXTcZujGk1tuNeydg1zQtjOjxEYpRH7NjLGuW5Z/Vwqmer5heDpFfSTu27xQOMPIjZOQlloW9Bp0ggcKIb7fhPOqeOwNV0LMk2XXgjRcgTdVKY0/fRlkWgtwSukjEo5SB+6oyNi7S5rnEXvy2RjW00P/jBD3I5fsSgeVu8kPlX2iemBK6UKMhHN/RJCO0XsrWwGnUdzBS70aUcgUsPltRrnu2nn7voD8C76XxecwL6QsaT0mfk1epEvH7jqbtOrJ6wY0Y7X+L5pu2tLi3uKe/pnDZNL50nsbUgf4idY5bOBWJEFf9GPaa3Ooo5Dmscmu9C4qcdiszP+JYxleqc+grWn9C7+giFDzTAWoQiS16irlGn+agLYb7I+ozWebhP/JJ+9b90HKsykE4pgatNuykdpwuH3BHMeF8IxGRt+ctvO+9qk89hxEl+XBwBR8ARcATGHgEncDV8B07gagiUB3MEHAFHwBFwBBwBR2AWQcAaFlhEt4Z+W0S7kMliS+ohgLBaxLYLXGzhhft+BK9WkCJSsQtHePbBS1Kd2LzYRVW7IJRLyxodLVGrzjANGQNCD2XCoMXCqhUWBik3AlFIi1V2oZjFQrajS6WOwEV4DBpTpkwJYJKKDB4QLVjAROyXx2zXhTHLit2uRV4TSstn403P7XuQIdYaRyxpRM9CKMFoikBikmFE93PHqsVuS9RgARcjmV3EJy6MFZCCeJ/6atemoS0tbF3mvoyTXMc4YIlmeicYDiB9EAbBAAbOGLDs19F2UZ4Jq0iE8aGp/1iUJj3ELuhbjwlVZCFLVKKNi7RRhZnNyyAErpI6uN1220XSBXhhFLYkR8rO/fXXX5/T6OnDkhLblAeyCMYHxBo54oVp/yZOnBgJpfzE+xRGZERGU87tNqf8RtgSTFtMQe6CCNRPSuugJXClxCHStETCXF4JY3W16jGGUnQxgnFLxvp4Yeo/yJ/ajgQSVpMt+ayxB8Lb97//fUXXO/LO5W0OY8i9U7cqKc2LjSvXNsAGIw51iHpOujyD5Nog160Bib4JfWzrUpV+t3lJiQXEm0qJjrQkm9Qbo+K3dZOtL9kCs06q+te6frIqPvpHdBG6D70IGZOjFbttjYhA5AHdx3PobvqRVOiXeccIehidjXCtpM+276tJ3SGtLvVCFwZ09BV6y0oJHhZ/+lDem4ydihtvOnjVQfTeOO/C8KhxA/FZA2lX5SPeOhmkT6qK1xKi2D6OsYAV9BK6HYILbQSjMVLy/niu3xjUtnP6XkhYGifwvNXduTGb5g48aw3aJWPXNC+Mo7WIT14QO6aRQbt0flGaHvkoaaeWqGz7Ao0ZwVA6Uu9d26ySpvX4Kp1TReDK9WNNxpKQxURaIE3EepaypKvSMajG1cTNuAMiBp6RVGbhYIlsfKwAYQxBR+O1sE5K+0Tb/6LzqN92fE+atk2IZDYWdbCu/HX3hk3gyunnLvqDNvN5kR4h+/AO7dwPT2ravpdxEMTvOrF6wq412PlSm/ZWlxb3msxn7DqD4mOOqTUGO1csnQso3tzRtoFRjOmtV9pc/bLbQ+qjHNU52jEf10De0oc7zP3Qu+gaidUZuY9TLCk07e/qxifUn9JxrMpAHtXfKb8cLanZ5tmuvzRtN6XjdJufqnPWNZg7sKZj11pK8qk0hhGn4vajI+AIOAKOwNghoLkfa8HYQZgbMS/mj3P9bprDOaYu7P7/3r7pUzNBOCdwzQQvybPoCDgCjoAj4Ag4Ao5AhwjICEOULGo1IXClXn2UHYxubG2EYOTA4GcXxmScV3gd7dZFN9xwQzjppJN0K3usWlS1C0K5stgv96xHjyrDNJMFebeq27pDC4x2Yc8az6oIHXUErlzBySfGjvWnklkmTPMeZQlc9l3mFjqJc9VVV43GbRYx+aq/tHy5/OmafQ8yxFpjO+EwwkACgSSiyZqeb3qsWuy2BC5L1OsXLwZU8IVMxrYiTBLtO+V5SzawZIi67WE22GCDsO2228a4MITLoGCNADmDLumJMGaNZzJQ2IV6wlrBYwEkGMR6kqrCzOZlEAJX2zpYpW8wSKBLMGJC7JN3BrZqKiVwWa9IP/jBD+IWUhYzzlnkx1CMWHxlNLWG9Rho2j/7Bbw10NowTc6b1EERuKib6FmOVmRcRf9yPyeWfGHrRxoWHcj2Xni12mKLLXrbj5QQuDAwQXZKZfPNNw+bbbZZvJzWPRu2SV4sWYny33777YG2hfEtJ7RzPMkheGOAuJaKNY6x5RlEDOvFrAo/68WnSZ0o0ZFqb9QBq1tsGegvIKQgN954YzjxxBPt7RnOq/rXqn5yhgjMBWsUr0rbelsTEQjvVGzbiNTpcOoU70d6epA+u23dIW9d6oUSYgh5kAHR6iuuI6V4LLnkkmG//faLcVThb8dtem880IXhUeMG4htG+Yi3rTTtk6ritQSuKiKKDJPEQXvGAF06Rus3BrXtPDWukj5beLMdIgLZgfGOFevRNWdgtmH7jV1tXvBIB2ktFbUPrnM+yPyiND2lTX8onZPmM/fb6k7NPSxxiXEw5HFEcxe2AqaNIeh5keukc6oIXF2PJbscg9pxkrxxahtJ3ufdd98dtym34yzIfMzd7Bg4glLxr7RPtH0VRDbG2amwhTqkGESEntI57iB1MM1X09/DJHDl+h/y1UV/oDZhy1k1n7dETwh4119/fbjiiit6ni5tHP3O7TvS++YZO19q0976pddkfnbttdeGH/7whzNExTateIq17aTLuYAStASuUYzpNd+tms9ATuWDCgRCGQQ31TmegSQKQQmhjtIPp3Mm6QzC0O/KmzG/JSKfpnq/bnxSOo4lTZWB81z/2oTA1bTdWN3XZpxO3vqJbSvUR32cZ9eJmuZTaXUZJ/MuvFAyVk7lqquuqpw7pmH9tyPgCDgCjsDgCMgm4ASuPlg6gasPQH7bEXAEHAFHwBFwBByBWQwBu3CVW0RRce1CpvVMo/scMfhh+EP4oppFeLtFSLpoFgNO+wdZBrFbx0y7NcPB5sUuqtoFIX3dbR+2BnJrcLTGFWustFsVEk9V/pV3wrDIx6KxNZ5hgGBRMZV+BC4W/9hGb6GFFuptEZDGYQlcMmYTRgauNLz9PUj5bDzpuX0P1hBrt9yzz2CI4r3zRbsl59gwufOqxW5L4LKeC9I4+KqWr/shw2Gcte9RYdPFWsgs8vRivQTYMqdbt2nhN9120S5Csv0a3pRSkfcNLcpbglFdW2FLQnk0s9uFVmFm85KSaLRAbdsM+bTvUwbptnVQ5cXwwJfUK620UiRR5RZTCTsIgcuS2nL6QXmRIQTjuTxOyWhqPf0oPEfrhaEJWUfPltRBEbgwMmBsSAU9LsJbE51lvcIRFx6nNtpoo+idSvGkaZQQuDBI5PJjjQdpvG3zgq4EE/oIK6RLe8U4gZ7RFiwyHitsLn/ck27gOeoO/RsGeMQa1+OFaf/slktN64RtUzauKh2p9qZ82Wd0DibkEbFbcel+eqzqX6v6yfR5+9uS8yzh1YbhXEY66RiRXrmX1k+uSSyJBF2JMY++T9LvfRJOfXbbusOzXeoFEVTIM22lqUg/3zvVc53O9WxpHw+ZRJ5TL7jggjB58mRF2TvmiHfc7MLwaMcNKlOX5Us96/QKZU5K+yQTxXSnGI7xAFP3fu12S/R16PjS+txvDGrbeW5Mv+GGGwY8RyEi29gC2baXGpjbjl1tXhgDUvZUrDeZQecXpemRp9J2Kl0NEQt9LM8xzJMg7smTrzw7qb6kHn6kc6oIXE3Hklaf596/8E8JVIOMQbE3yAsL5Oqjjjoq9qds10j/TNm1pSPjGPorjvS/OW8yyqM9Cue2faIdh1SRZNAJjA8Rzb1K57iD1EFb3jbnwyRw5fQzeeuiP8iN16vm82wbDflRYzbhw3geT4KQJXm/TcS+I7vWYOdLTdtbk/SazM+qxlE5ouWgc4Fcni2Ba9hj+ioP57l82Wu2ztnr9L3oHz5CtKJ5tiWO2vuc66OXtP+uG5+UjmMhMdsypP0r+WlC4GrabkrH6eSjnzCXp14zl7QfaNo1k6b5VFpdxknfQ/o5YU2G9u3iCDgCjoAjMBoEnMDVEGcncDUEyoM5Ao6AI+AIOAKOgCMwiyBQQuCqcpufI3DZr2GbQFZlvLHPVi2q2gWh3IJX1YKvNWRYApf1MGHTrzuXMd8az3Qtfa6OwKXFwvQZFg/xXsXXx4iMCJzLCF5F7CCMlUHKZ+NJz+17sIZYwr3+9a8Pm266acD4mxO7SJ67b69VLXZbAlfVYjcLqxgn00V+8IUsRV1hwTFdrCV963ENsggetUT6SQk+xME90rHbHBCPNQLwRXXOqKCFZRG48KoFEQmp+lKWezbuSy+9NJx77rlcbrRFxyAErrZ1kDxBhEF3YBxLRYZ2CHbIIAQutohacMEFs+/UpisDqV3M1zW+CCOeVEoIXKV1UASutK6RJ+oZ76CNaNsRyC/oKkhlqWD8ouwQEJCUaJWG1+8mxh67VZTiHSQvL3nJS8KOO+4YPXnkCGi810MPPTQa86r0rPKfHoW5jOvcz/U1XLceVpoSuHiujY5Ue6vyfkF8tr8cNYHLvv9TTz01XHPNNWRpBpGhUQQu6y2vysMZkdi+Bl0JYQnjbRux/XObukMaXeqFUmKIDIg5EkZpH49Bj3qIVBHoaFu8N0TvjfN+hke8gOLJErF9on2XdtwwjPLJm1HMRObfIH1SJrp4STpDOiQXzm4Xd/TRR8ftXkvrc78xqNULuTG9JXCddtppM2ypW0XgqtKpdWPXfnkBqxyBq3R+UZoe+Shtp9bDH+NG2gkfBOhDAJWF35AxNG60Rm/Sl86pInA1HUtar7G33XZboL7lRASlrsagKqeIbNL9kLfOP//8nvc1fjMf3HXXXWO22H6ZetpPSvtES+CqmjeQtuKXJ2qVp1++dF9z3EHqoOJqexwmgSvX/5C/LvqD3Biraj5PmnhKg5wPOSOd43GfsSxEHs0vuJYT+47s3NTOr5q2t1z86bWqOW2T9FIC1yBzgTRf9rcd0/Fu0eupdDWmtx6s7rnnnjhuT9PK/bZ1jvvM0XmXiPRt/DHtn3RQ1fyOYNLf6ZpA3fikdBzbhMDVZBzVtN3Yd9pmnG4xrDsXMZ33wNgOseO9pvm0aXQVpxO4LKp+7gg4Ao7A2CLgBK6G+DuBqyFQHswRcAQcAUfAEXAEHIFZBAGMl3ieQFjEqtrSzC5k5ow9PJ8jcGnxn/ssdEEEqBMWxTEU1InNi11ULV0QqiJwLbHEEnHxmbxg+GZhq04oGwtvSD/jGWGqCFzW6MpiIcZRtoFg6xUWGBEtOFoClxZwyYc8B8XAFf8GKV9FlPGyfQ/WEGuf4St+ti/DaEI+tLhKGLYxYzuzflK12N2PwMU2dSzea2Gf7SQhQ91888099/4Y2BZddNEs2cd67eHrTLbXZDERSY1ta621VvTyxT1LFOB3k0X5lMBlPRjUeeCSdwfSYcs0yodUYWbzMgiBq20dJE8ixHGOpzowJL+0ObbAgGiHMRkZhMBl9VPui9+YwNR/alvkBe8TiIymVQv8bQlcg9TBOgIXeWXLLb5QhsjJVqr95MEHH4w4W8M7C+20B/QOhhMMXdb7nIhW/eK2hgHqHu80FYgib3nLW+Lls846K3rh6CovGPLYXgWDkkivJCRjNB4OqfsIBuMciTLenPaP9882ipa8cOCBB8atWWw4zu32jG0IXIqniY5Ue6vzNmKJZLfccktg+9A6qepfq/rJurjYdhPCLlLlDdEaGkUEstu3VRGIiJN2jAEGwWgHgYsj0rbPjg+Zf/3qDkG71AulxJA6A2JpH48Hvq233jqiUdXW7TaLem88YI2nOcOc3dJzUAJXafliwWr+DdInVUUrAlfVVlA8Z8dO6EtIzaX1ud8Y1Lbz3Ji+hMBVOnbtlxewyRG4SucXpemRj9J2aselkD4gGUOClJcP9XkQjNlGnveHpNuhS+cMSuCCzC4yepX3JNJnrMy4U33moGNQygUWCERqxmUIhDXG4iJEQbRgy2XC1rWZ+LD5V9onWgJX1XbKEMuJHyFvePsaizpoitvqdDwSuJr0B7l+pI7AJVDQn3j1hewCGYgxlcSuHehaerR6woa386XxSuCiLKVzgRQH+3uUY3o+JsG7M8J4jj6xidgxyE033RTXbtAr+kgoHVNqnl1Hrq76AKhu/FU6jhVJjPENkqv/Xbab0nF6k3dBmMUWW6y3JnXCCSfEbdHtWCdXvn7tu6s46YOZ2+aEjyX7kf1zz/k1R8ARcAQcgTIEnMDVEDcncDUEyoM5Ao6AI+AIOAKOgCMwiyAwbAKXjBLAxXYZbJuRCoYEGXmvu+66uJ1eGsb+rlpULV0QqjJMWw8T+tra5kPnGDtf/vKXBwwqLAwi1kiREnf0XBWBy271ljP8WwOKJXCxDQtlQXLbNrIYihERgzkGm+9+97s9Dxpty6cy5I72PYjABYlimWWWiaQSCFqpWCwgqh155JFpkBl+V5GRrKEs9yW9Xfi0hmebgLZh0UKqvWfxx4gD0YeyIRjXMDxJWJicMHWLRkgfYG+l1AggDwBVZCLSsEYljNHUE6QKM5uXQQhcbevgMcccEwlc5A0DIYvkGOus2PYwCIFrp512CmwNhpxyyikBXZOKJQRgTMSoiMhoWoV5WwLXIHWwH4FL5APqLltNpXhSHoyV/CG0Ecpl6zxpsHhtxW4HUkXqsOE5t8aetF4prL5s5zcGXQhjJXnBOMMX7wgEQPSIFbyHoRe1PSckTXSFPNxUbZuEvmRbJ+JnCx6Mu7ZcIp3ZtDgnXggNSE6PxxvmX4mOVHvjXdPP5MjPlqR20UUXRS8nJtkZTqv616p+coYIzAVrFK8inFqjsvQx5CkIG8iUKVMCeiIV3gvtk35aX/aX9tkldQeCZJd6oZQYUmdALMXD4l9F7rAkL7033pE1nk6aNGmGOinPCYQdlMBVWj7SrhLItehQpKRPqoqX/l8eDKtInyLLiLAySPn6jUFtO++KwGX76pzOs2MnO3btlxcwzRG4SucXpemRj9J2Sr8DORy9xdgCAzTCmAxyAvqffgXh9yte8Yq4DT19uBXpnEEJXMSpsaSNy6YFThCWyLMIXPa5qvEQYarGoFbfCwfr7XS33XaLYxOuYTzHWy8kcxGnbP5y56V9ou2rqnSefUdXXnll7NfHog7myt3kWoq9xrc8W1qv6/of4u2iP2hK8KBNrbfeeiQbxznMu6zY92frsw1jz62emBkJXKVzAYtBem7HvsMe05O2dFTVRwpss65tVzWeUJ2zH7PZcOgW5gIi52gdjHE0W2oztkuFtsI4MV0TqKv/dhzVZhxL2ioD56XjqKbtxuq+NuN08tZUNG+VLrfrRE3zmaY1jDjTNPy3I+AIOAKOwOgQcAJXQ6ydwNUQKA/mCDgCjoAj4Ag4Ao7ALIKAFq4ozjA8cFmigiVEWPisN5PDDjts3BC4yKOIBJzL0MK5xHo3sQapfsYznrekJYw0LCoiLCDqi0AWBzEmWGGxksVIxKZpF1ZTT1CEzRlOS8tHfFViF+ZE4GLLJL6CRnJlmnfeecPBBx8c71ctIMab5l8VGakfgWvbbbcNkFEQtvRKPavZRdd0sVbJWyMRxlYMrdZjk8LJmxPefSB6WLGkqTZfcWvRkrgOP/zwgGHBCkQLiFAY3dIviqsws3lJF+W1QG2N9KS38847Ry9DnPOVNISftnXw1ltv7W2p8Oijj0ZvD8QnwZiO4ZuFc2QQApfdciKXFvFbI7G2FuS6jKZVBsu2BK5B6qDef/puySdijegXX3xx9Hz03J3n/lNX+eIc4xR1F/3LUXWVOp8Sv6hLGBHkrbGEwMWiDMQF0pLwlTXtHuM2JBxIVaRfkhdrIMcrnvSJ0uJosaHvw6MGeUJs+vHCtH9bbrll2GSTTeKvq6++OrCdmPU0k9u+EIwxOHNEcmSGeMP8K9GRtg9R3kyUEVeMeOhXBIIIdb9OqoyWJQQuPF6QPvWH94pRH0OcFetFSzqGLVOpozxHfaFP5P1YwVCLN0RE21JxXtKnldQdjH9d6gUZ0ClDzqDF9ZxIP1dtYVWCB/jzHMK4hHpjjeHghV7WVsh6b4S3xEU8CTEWkaA/0CO8V0QGV85z4wauD6N8xFslEGe0zU+un+jXJ1XFC14icNmtuhXebmlnt3kqeX/EuYvxdISuk7Fa6dl23hWBq3Ts2i8v5Nn2zRovl84vStMjH6XtlGctqYnftC0RtBjjMD5T2+D+fffdN8OWyNI5lnRlx29txpIieJAWnhnx0GjFenK1hBeNQQjbdgzKM+rfOUfkzYpzO07jN8LWipCPm0hpn2hJDLwXsElJ7IwV2GYXgVjC+xmLOtgEh1yYJgQunuuy/+miP8jlJ+ehZ+WVV47zaspQ5UVN+rSKEMSzEqsnZkYClx3vtpkLqPy5o53jDXtMT/q238xtoyqPfYTV+FbkJ0vg4j5jj5e97GWcxo8x8FCGiDTKuYiZnEvsxz/pmkDd+GSQcewo203pOF34NDm+6U1vCvyBH+sTzHfwjIw0bd9pOsOIM03DfzsCjoAj4AiMDgEncDXE2glcDYHyYI6AI+AIOAKOgCPgCMwiCAybwAVMdoGNLbnwfsMiDsZtFttXXHHFiCYGJvLTT6oWVa0BsM2CkCUPySiLsQLD8TrrrNPz6gJhAgKJtplk8Wn33XePJADyfMYZZwSIDUg/4xlhrKFBBimuW0M+hjy2UCFfGNAhfrAlhMSSBywZgvsYRzF6qBwQv2RI5+tTJkml5VP6uaN9DyJw2e3EMARj/CDvCIuH4AVxCqnaaiveNP9kyKIuYfhi+0rK2o/AZRf5+dKWRVyepR5gCGNRUEY04sZgx9GK9T6i6+kCuTUIYfDCy5mVUqObLR+GJjzbicRFHYEEw8Ixknogq8LM5mUQAlfbOgghCvyFt4yI6Abq+dvf/vZe+6I8xx57bID0JWlbHohmeKJDIAri6Q0MISRst912va/3U3KU0umKwDVIHZTxNM2jMGGrQAiGauuQh6SXIBDss88+PU99eESk/iAWG8hAPAc2bJX2zne+M+ApUYKHK7bm7CfW2ENYPFjhgY8jW9+RF21rc+mll4Zzzz03RlmaF3mwIRLiIk4J7RGdy7um3dNOEOsBDOIX9VEkI7ZAgcAlLMFeRl29B+K4dyrJ9ogjjoiEScoDKU6GXu43IXCV6EhreCGdCy64IEyePJnTQFvEgCdPL1UEnxjY/KvqX+v6SfP4DKe77rprr8+iX8V4DzGG98D74L1ILBHI9oP0Veh4SLIIuoFnpTcwtsvLYGmfVlJ3utQLtt9ka14M0Bgg+0mdAZFnS/HYa6+9AgZ/BNyPPvro6DmIPgZ9wJZUEvveLAEDHYWnT/pXrkMIEZmQZ7sgcJWWT3lPj9SpQfqkND79toZorl1yySXxD11Df0B9p00g1qtfafn6jUFtO++KwGXbbJuxa7+8gEmOwMX1kvnFIOmVtlPyutVWWwX6FIm8kei37fe4ZuuBwkjndEHgsh6JGDsfd9xxvfHV+uuvH+cb0rGWwDXIGJRy2I92+H3ZZZfFsSrnjP3wdqN0uYbBn7FXEyntE+14nXToc/D8CEmL8fT73//+MGGqN13EYs/vUddB27YtduSlTuoIXKX1ul//00V/0HQ+DyEfghZ1h76TsS3jfASCJJ6+5e2bMZvyXoWZ1ROjIHCpbadzWjs/qyJo8sEA5bfttHQuUIUH10c9pl9++eXjx2ekzXyENsm8hXfMOIRxM2LXj6oIXMxhWGOSbjn55JPjNvEQ0dG9un7FFVfEtRMI53wsh56U8G6IX6I6VDW+tn1im3HsKNsNZSkdp/OsvKSlhDnuSWzbBF+81g9K4BpGnMqvHx0BR8ARcARGjwD9JMKYn/kA/TJzY/441++mOZtj6tYL06/gN31ynIdzAtc4f0GePUfAEXAEHAFHwBFwBDpGYBQELhaNMZJrcYwFMLxpsPgi4RqGXbx09ZOqRVW7AN10wVdpaeFUv60XKOtVgPssIiJMLCTp4p1dYAdjFhdTqSJw4QECA4fw4jnSVHpgxUIZOCBgSf4hCOEpBsKBhLCIjcsaTblXUj6eqxL7HkTgIqw1YJIvjLuUSeUgDIvPkE+EMdeqxKZDGBlVrHEpJTARjjTxziOSE9fAU16e+E3edJ+8EM/ll1/OrSjkWVvLcIHypN5lMPiwtaJdWH7u6ef+N1mUxwsNRAy7KM/TqQEM4xt5UB0hTM57SBVmNi+DELhIt20d3HfffacjA4C3JurEx3udZ555OI1lvO222wJfYiNtywNJkGdse0jTA0dIppBNJdIPXRG4BqmDIg5VEbjIs/WcwW/KZHUG1yAO0Nbk2SglJfIMIqwIRxvR79y2oPEB8y819ugWcSserkEQwYCidl+aF0uMI17iI9+0ZZseW92qPdO+IO9YPZSWlbhSQljarxHG6hF73oTAxfMlOnKbbbYJG264IY9HAVv0gdUFlAdd0s/7FhGAA2QpxBot+a12wDli+8nnrsz4H3yps7a/t23O1gVLBOKd8V7S53iPIrqQWm7ry5I+raTuCI8u9ILVwUKxauyg+xz7GRAJU4IH7w1inO0XqVcWe+JG7HvjOQgNtv49F+q5/7x73bNjEavL7bhhWOWzeUrPB+mT0rj027ZtXeNo6z+/rTc5fiMl76/fGNS2864IXKVjVxm2KWsuL1yvInClehg80XdWb3DNzi/6lb0uvdJ2SpyQeuXxkd+WtMzvtL/kvdPPW5HO0XiXezZPVQSPqrEkROZFF120lwRYIeovVT+7GoMSNyQ2yGwS8gbJU2LbSpVuVdjcsaRPTAlcilflt78hmLHNpWTUddB6U2Ycw3imiZBPiLlI6hHb1iHF1UX/00V/0GY+n7YhyPq8Q/SBrdPM3/D8VidWT9ixkMWqbXurS8/2gYRTG2+SXo7ARRwlcwGeq5IUX4VL20lXY3rit7qf36SF2PfJOAGyJVJF4OIe26xDikboJyCHomM333zzsNlmm8XruX8qH8c2BK7Sceyo2w3plYzTwUpjNMZ28iiZw3C//faLHwRRr/kQclACF2kMI85c3v2aI+AIOAKOwPARcAJXQ4ydwNUQKA/mCDgCjoAj4Ag4Ao7ALIKA3dIDA4e8S6XFw4jHojVy8803xy+10zBaSOG69SjFb7YahNBivS9wHWGhj0XQO+6447kLff7bvNhFVesxos2CL8lBOmEBT8bJe+65Jxx66KG9nOAJaK211ur91gmLeWw7csIJJ0Sjua7bry7ZIkoeXXSfYxWBi3t89bnDDjv08sM1hIUvyCtssYhHLQnvRuQ3vtxkezsZSRWGxTW8RGE0TaVt+dLn7W/7HqwhFs8dGEblAck+wzkeVPhi+pFHHklvZX9DxsEDmohW8qqDdxa+JkVyHgy4Tn1ksVzEIK4hYHTeeeeFKVOmREKW6sMvfvGLgFcUK9Rx4kFyW91oq44cwYBneMd6h9QftlVKpcroRji7xUL6XFUbrcLM5qUpgcumDxFIk27y0qYO8v7wcsYXuVZoW7wHvHJBJJCnJrvdTkl5qt49aVOH2BLo3qlf51uB8IfuQlfhvS6Vtlso8nxVPvrVQXnpkHEnzYt+r7322rF+qQ7rOkeMj+gRuy0a16uMM7RJFul552z1hfRbrCeMttlUmTDapnoJb1zoWsJYKc0LaUAAk3HHxgkBBY8VtHErvFv0gbxV2XvUCfQmf6nghYh+DUOfFQxCePGhf0WaErhKdWTdu0av8oV8jkRs86zzqv6V+/36ScWRHtl2jjaORwgrtPHjjz8+GtVo35YIRDjeC17atKVw+iw6WUQ8e4/zkj6tbd3pUi9gQINMYTFK9WpaRn7L+0E/Ml0JHnimwIubth1S+rw3PI7uuOOO8VL63nhfGDktiYaAeN1kW0XiRCyBq2rcMMzyxUxk/g3SJ2Wii5dESkGfQEKGMJjqKN4hYyDIn6m0fX/9xqC2nefGC5BCIcIg8lRi82RJ5Ha8XTJ2hbzTb35hjfiDzi/6lZ1yVqVX2k6FnXQGv1PikiXYWO+6epajnrf9vx2/tR1LUgcZL1tPiEoPL4D0Scstt9wMHxEQxo4B9YyOuTqle+h6SJ4IdZ26ZMXG29Tbp32e87Z9oiVwMWanfWpuobhpu8xVb7rpJl3qHRnPtZnjDlIHmffIczWe/NLxTC9TyQnjZeo1ks5ZSut1E/08aH9g9YuKRH5pP4jtfxhf0pdQ1pwwNuMDjXS70FxY+47sWsMg7S2Xjq41mc9Ute8qAhdx17WFqrmA8pQex2JMTx5YD4HIlgrjWtokxF8J9WXCVG95Od2CvkOHau5v9RTevvDqpbkm8dHmr7vuujgG4v0w9rEErib1v3QcO8p2Q1lLx+kicOXwJl4JhC3mWQhzbN4R0qZ9xwfMv2HEaaL3U0fAEXAEHIERIqC1ZPfA1Qd0J3D1AchvOwKOgCPgCDgCjoAj4AgUI8DC2ZJTt+NikQwjJR6jMGThvWo8CAvCLAaxYMfiUkq6YqwMeQGjPQtVbBFz1VVXxfDDyD+LiKuttlokFbDVCgQfCCQSFvfweMCCNIuXkBOsLL744hFriBEQ8/DsQL6rZFTlAz8IPngjID8Y7+6+++7eVhdV+au6jnEJkgoENm3LWBXWXucZ8KVO4l0AQ5H9Ihs8Xv3qV0dyBtv2pfXBEvAwurKtw6gFAzvlJ598Tcw7xphBeeqkFLO6OHP32tRBFsfJF0Z/jCsYs1ksRyDIYNxk0Z26knoSalse3j2Lx8TJNhnUHXATCTJXlmFcG7QO9ssTWOIFjnKCIbhhjEi387TxsLUhxky2G8FoRd1HJ0qIi4V+4oLo2lZo/3jJo91jPKkjFpXmRbqTtDDCsu0hOhTDa7/06J8gcrF4RD+AXqgTDIY8QxtE11J35Qmg7rm6eyU6knLSf/FHO+HdoJPq3nVdHqru9esnq57j+hJLLNEjrqCrIG2nxL3c89QD2jj6hHpHvUQP9Hu2pE8rrTu5fJdcgwjAVqcQ72x/VBJX+kwJHsShdgg2jNl4d+hmjJeINaDHC9P+QcqlbTD2430N2i5s3Lnz0vLl4uLaIH1SGqclcEHUgxwASYT2yliSMRzHOum6fHVpDXJPbQg9WjJ2LUl71POLYbbTkvIP+gxELY3LmS/Qj/UbR5Jm6Rh00Pw2eb5Nn2gJXNrGHf215pprxrET/SnjlXSeZfMx6jpo0+7qfJj1epT9AePX173udZH8zTib7bEZ3zOP1ryiK8y6jqftfKZJ+iVzgSbxEmYUY3rSYazNfIaxNmshtEk+AOlaSIe+izkK6yvIpEmTIomLuZC2X2+bbuk4dpTthjKVjtPb4uHhHQFHwBFwBBwBi4ATuCwaNecsCLg4Ao6AI+AIOAKOgCPgCDgCjoAj4AhUI4Chhm3GMMKmW8tUP+V3HAFHwBFwBByBmR8B+sB+BK6Zv5TdlSAlcHUXs8fkCDgCgyKQI3ANGqc/7wg4AuMfATz24qUMwdskH25YgfzG1rUQuyDW4xHaxRFwBBwBR8ARcAS6RcAJXA3xdAJXQ6A8mCPgCDgCjoAj4Ag4Ao6AI+AIzHYI4H0GT1zbbrtt2GCDDWL57TZQsx0gXmBHwBFwBByB2Q4BJ3C1e+VO4GqHl4d2BEaJgBO4Rom2p+UIjB8EJk6cGHbaaaeYITy1sTWnPO3hve3Tn/503EqWAJMnTw4XXHDB+Mm858QRcAQcAUfAEZhFEHACV8MX6QSuhkB5MEfAEXAEHAFHwBFwBBwBR8ARmO0QkMcRjNcI2xbuv//+vcXe2Q4QL7Aj4Ag4Ao7AbIeAE7javXIncLXDy0M7AqNEwAlco0Tb03IExg8CbC9/8MEHx61SydWzzz4bnnjiibj9OtvFa77/wAMPhEMOOWTcb8E5fpD1nDgCjoAj4Ag4As0RcAJXQ6ycwNUQKA/mCDgCjoAj4Ag4Ao6AI+AIOAKzHQIQuLSYyxe6xx9/fLjllltmOxy8wI6AI+AIOAKzLwJO4Gr37p3A1Q4vD+0IjBIBJ3CNEm1PyxEYXwgsvPDCYb/99gvzzDNPNmNO3srC4hcdAUfAEXAEHIHOEHACV0MoncDVECgP5gg4Ao6AI+AIOAKOgCPgCDgCsx0CG2+8cZh//vnDQw89FG699dbw5JNPznYYeIEdAUfAEXAEHIFVV101Gjzvv//+8Mc//tEBqUFgqaWWCgsttFB46qmnws0331wT0m85Ao7AqBGYe+65w2qrrRaTveuuu8Kjjz466ix4eo6AIzDGCDCmWW655cICCywQ++o777zT5/pj/E48eUfAEXAEHIHZAwEncDV8z07gagiUB3MEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcgcYIOIGrIVRO4GoIlAdzBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHIHGCDiBqyFUTuBqCJQHcwQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBByBxgg4gashVE7gagiUB3MEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcgcYIOIGrIVRO4GoIlAdzBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHIHGCDiBqyFUTuBqCJQHcwQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBBwBR8ARcAQcAUfAEXAEHAFHwBFwBByBxgg4gashVE7gagiUB3MEHAFHwBFwBBwBR2BICCy66KJhtdVWCxMmTAgLL7xweMELXhD++c9/hsceeyzcfffd4fzzzw///e9/h5R6+2hXWWWVsPrqq4eXvvSl4aGHHgonnXRSWGyxxcIaa6wRIyO/Tz/9dPuIZ9Inll122cAfcs4554yrd0We5phjjrDllluGueaaK9anW2+9lcsjlxe+8IVh++23j/n5xS9+ER588MGR52EsEtxkk03CQgstFB555JFw8cUXj0UWxjTNl7/85WGjjTaKebjsssvCww8/PCb5Ge/tFFA233zzMPfcc4c//OEP4ZZbbhkTnDzR7hB4xSteETbYYINAG0APH3LIIQE9uOmmm4Y555wzXH/99eH+++/vLsGKmNr02V4HK0D0y+MOgVxdrdLzO+ywQ9Stt99+e/jNb34z7sqSZmi8jNvSfPnvsUVgzTXXDEsttVScH1544YVDz0zT9NZee+3YzzFn7WqOMYw4uwJs4sSJcb5OfGeffXb43//+1yjqddddN84HHn/88XD55Zc3eqYq0HgZW1flr+R613pv5ZVXDiussELMyg9/+MOSLPkzY4SAt5UxAn7EyebGcSPOgifnCDgCjoAjMEYIOIGrIfBO4GoIlAdzBBwBR8ARcAQcAUdgCAhss802YcMNN6yN+d///nc47bTTxoXRKc0vg+4DDjgg7Lzzzj0C10EHHRRYnJ7VBIIdRneIdU888USveB/4wAfC0ksvHX9//OMfD88880zv3ng4ef7znx++/vWvx6zcdddd4fDDDx+TbL3mNa8Je++9d0z7zDPPDFdeeeWY5GMYiUK6gayBPPDAA4E2K/nyl78c5plnnvD3v/89fOYzn9Hl2eb4hje8IWA8RzCgXHvttWNS9vHeTgHl29/+dsQGAtd3vvOdoeJUpc+GmmgSeV27SYLOdD8hkuy5556RuKXMf+hDHwrLL7982GOPPeKlCy64IEyePFm3h3Js22ePsg4OpcAe6dAQGA86wxYuV1er9Py3vvWt2BYhcB111FE2mnF5Pl7GbeMSnNk4U4whF1xwwfDUU0+FSZMmDR2Jpump3TH23X///TvJ1zDi7CRjUyP5whe+EGRL+ehHPxr+85//NIpa8wE+0vrUpz7V6JmqQIOMrflYbN555431aKw+qsiVq2u9pzpEWoy/2sh46+/a5H1WCFvVVkrq7iBtZVbAcjyXITeOG8/59bw5Ao6AI+AIdIeAE7gaYqlJR8PgHswRcAQcAUfAEXAEHAFHoAME8LK17777Rs9Vio4veFmUh+jxohe9KP7xNark+OOPDzfddJN+jsnRLlr/61//Cn/605/CYYcdNssTuPA2BlENwXvDCSec0MPfLhA7gasHywwnszKB601velPgDznuuOPCzTff3Cu/FqGdwOUErl6lqDgZ1UJ2nT6ryNpQLte1m6EkOMJId9ttt/Da1742pvjss8/Gvv2zn/3syAlcbfvsUdXBEb4KT6oDBMaLzrBFydXVqvGYE7gscn4+syLQlFDVVfmapgfZRQSZY489thMvXMOIsytcbL86MxK4+KgHstTf/va3cOCBB3YFy8DxjBcC13js7wYGdyaLQHPnlOxYUnedwDV+X35uHDd+c+s5cwQcAUfAEegSASdwNUTTCVwNgfJgjoAj4Ag4Ao6AI+AIdIjA+973vp5bf6K97rrroncauw3DAgssEDACy7MPX/h+/vOfjwuuHWalVVQygj355JMBY7SELQpWXXXVuH3gGWecETBYz0pSt5jLFg0y1ONhaDxtd8k7gAS40047hec973nhjjvuiNt2jcW7mV0IXD/4wQ+m2/5Oi9BO4BpbAtd4b6e0yVEtZNfps1HqBkvgStvNKPMxjLQwir7kJS+JWyt94hOf6HlmBHu2DEHYSnbYWyi27bNHVQeHgbnHOTwExovOsCXM1dUqPa92MLN44Bov4zaLt5+PPQJNCVVd5bRNel/60pfih0d//OMfwze/+c1OsjCMOLvIWCmBiy3V2frw0UcfDXjgHEQGIaWUkGAGyWvTZ7vWe5bQK4Jhk7yMx/6uSb5npTBVbaWk7g7SVmYlTMdjWXLjuPGYT8+TI+AIOAKOQPcIOIGrIaZO4GoIlAdzBBwBR8ARcAQcAUegIwRYGGRRXN618OaEV6ecEIawPINcffXVcTvFXNhRXNNCS5cL9KPI96Bp+GLuoAiG4AQu30JxLLdQHLwGDz8G6ddhb6E4XvTZrEzggmw933zzReIWnhnHSlSnmvbZCj/sOjhWeHi6ZQiMF51hc9+mrs5sBC5bTj93BIRAG0KVnhnk2Ca9rbfeOmy00UaRtMz2gHhpHlSGEeegeeL5UgJXF2krjkFIKSUkGKU7Mx2dwDUzva1meS2pu4O0lWa58lClCLQZx5Wm4c85Ao6AI+AIjE8EnMDV8L04gashUB7MEXAEHAFHwBFwBByBjhD48Ic/HJZYYokYG943DjnkkNqYX/WqV8XtFgmU2+5goYUWCltuuWX01CWD8SOPPBJ+9atfhRtuuGGGuLfddttICLv33nvDZZddFljYwoPUwgsvv8949QAAQABJREFUHP79738HDL14BmF7RAmG9vnnnz9MnDgxXmJhnu0cCfvrX/86LLfccgHPB8hPfvKTGE/8Me3f8ssvH9Zcc82w6KKLBrx3QVjD69giiywS08Y7EWkibC/5zne+MxLcbrnllqzHqE033TRuP/mXv/wlnHfeefE5/r3nPe+JnqYoN96miIdtOP7617+G73znO71wjIExDCy55JLxi3G8Zv3jH/8Iv/vd7+JX0ZxL8F6FJ5VlllkmXuLe3XffHct/4403Rs9j3AM7yp4KZV9vvfViWeeZZ564ldbDDz8czj333PDAAw9MF3zOOecMu+yyS7x21VVXxXQ222yzsNRSS4UFF1wwPP7444H3xpfbNo/TRZL8IM63ve1t8SrvDFysEDf1BwPt3HPPHZhIkb9LLrkkpmXDNjlfbLHFwlprrRWot6QNEeDaa6+N73XvvfeOUZx55pnhyiuvnC66UdRjmyB523jjjSOxjLSfeeaZ+P6vv/76mGcbtur8xS9+ccSWekwcyEMPPRRof+eff37E0XrgOuigg2I7oK3gWY8tU2lDl156aXjwwQenSwYvPcRLHZk8eXLYaqutwmqrrRbf0ac//enpwm644YaxHb3sZS8Lc801V2xjxAvOqRGNOr/BBhvE5y+88MIZ0uXGu971rri9SlX7oz7jdQ8vgZT3mmuuCbfddltsC7Rx0uZZJF04pzxrr712bE/zzjtv+POf/xyfpb61lTZ1Fw+BaTu17Q09RttCx62xxhoBHUFbueKKK3pektZZZ52wyiqrxPfCuyM8bfGJJ57oZZ12xJfjCDqNerXuuuuGFVdcMeoayovuo32n0m8hm3yp7vAsOhqc0UPWe2Mar/3dT5/ZsE3TQ5eq/uPhJi0b3v/e8Y53RJ2Od0ZwoU+pazc2H1XntKEtttgitiXq0tNPPx09W5x99tmVbbhNW3nlK18Z3yXeL0877bTYB9J2aEOkRzunjlx00UW9LK6++upRp3BkOyDeC+2D+nLOOefENkP/hVDnaCtWXvjCF/b0Euf0NYRjrIC+4hr9Z9p32Dg4L+2z+9VB+lPaweKLLx4xoD5PmTIlljHVNcpTm3aqZ6qO6Lftt98+Ykwbpe+mf//9738fxwJV7aBNP1yVNtetzijto7vKi83n//3f/0XyBO+HsRrjLOompH/wkdA/q/6B3amnnjrDeI26y5iQtkr9RKf3GwOhXxk70OdwzriF93PSSSdNN+agD3vjG98Y46N9sD0S+pM+8Le//a2y2eiYq6s5PU9klsD1ox/9KOoNsGK7csZVjFPOOuusQFu30rQfbjOmJH7yWYfZXXfdVTtuI46m+pmwEvpj+jjqCHUZ7NEljF2bjikVF0d0JONHxh6Mb9HB1APG4JdffvkMHmmb4kncJeXjuVQYjzJeeeyxx8JPf/rTOF8BA/JO34RuZdxX9SEN8ZXmpURf8l5e//rXh2WXXTb2q9RPtuUG04997GNxLkB/MmnSpLSoYdTp2QzQNzHe5cMj5pb0d1ZK+tNhxGnzVHpeSuBibMgYkfH+L3/5yxmSH/bYev31149zM/Q774k+gP6bOR+6mLENcvrpp0+nD/AaRjtHyDtzByuMi9HtxMeHGugBSdO+ifDU/br5KvVB8zbO+42PUgIXeoC2/OpXvzqOzdB9jM/stvdtxsgqo47SNbRPxoy0R3QuHzCha5kLkx7zAs0VwBzdyTtAB6XjZ8XNsUQPte2bStqpzWNX4720rdTVXebbdTLoPJR+gvww/2KdjbbDO2b++7Of/Sy+u1z69LNvfetb41yHsQZjZNat6G+oB03k3e9+d5zXU2cYT+WE9RbwQZgv2HWeNu2P5+vW0cg7cx/aOR+f5sbb9FvMERDWIRnL1EluHGfDN52zaW7Gs13qL5sXP3cEHAFHwBHoFgEncDXEk8GciyPgCDgCjoAj4Ag4Ao7A6BD4xje+EY0GpPjVr341LgD1S52FI8g1LORb4guLJCwOsZiUk/vuuy8cccQR05E4RChhIYbFJAyxqbA4873vfa9nBCefpJ8Kg+4DDjgg7LzzznFhkfuQVMinhMUgDFWpsPADAYLFS84htiEYCtl+CsGg9/3vfz+e239s3wh5hEVitqeSaCGIxVgWkZRnDFMivbCQyqJuFWbk5aijjuotgNn3pXQ43nPPPeHQQw+NcS299NLxFp5WWGSTQCB73etep5/THUmHhVq2nJRg0OTrUoSFXAgnLJalQnnYWoQFxH5i42Qh7fDDD+89UvVuCED+IA6lC/W9hzMnGJ122GGHLLYYSFlkRFIC16jqsbIM2WS//fYLELByAvYsTvYTDNu2/tnwEAshJam98a4wiLP4mwpGY+quJU3Kgw9tCWO8SJ88q61AqBv77LNPXBxO4+Q38bKQidFAgmFdW7hVecSSsRvjvd0Kh/RopyLrKE6OlI/2yIK1JZrahXMWlWmXOWm7tVXbumsNOWqntm3QniE3cC0VjL60RYiqqUDcpC3yjhAMY2x9i9x6662RuJXTNbQH2qLdblb6i3uWcAqplfxjDMoJBA2IwFUEGvtMP31G2Lbp0T/JuIfeAA9LGoGUioEPgaxLn1TllUrtJgau+cdi/TbbbJPVNTyWeqssaSu27qIHIb5gYEzFvi/0ioygaTjaLeSdPfbYI96C/IeOldAXf/CDH4z465qO9OUY1ki/iX4q7bOr6iD5gDgF+SYntH3qbLolZNt2motb1yCZf/SjH822UcKgd9BdllDJ9bb9MM9UidUZJX10l3lRHmlbkG4xMqZCe4SwpHEjBDjap8ZG6ZgAYx/jL9VzCF4Y1HNxawyk9k1akBnpYyQ8L0MlbV5bguu+PTb5oMGGz9XVnJ7nGfVpbF1GH5UrDwSGww47bLo63KQfbjumJD/9MIOUq7Fg+o7a6mfSo7yMgRk354R+6JhjjumNe3Nh0mt8DCIyeHqP3/SNX/ziFyOhS/eb4FlSPsWfO6pOMB7CoFylwzDEX3zxxdNFMUheSvQlpBTGlLn3BJ7MLxi3Mt5KCVyjTm86oKb9+MhHPhLnlHbOpXAl/SnPDiNO5an0WErg0nwAXYOXMsmoxtb7779/dr5A/33nnXfG8St5gtAJAVPCvI73h9CO6IetUBZIXvQB3GMNAWnTNxHe9q+p3isZH6ntEzfrCRC4cmLL22SMnIuDazY95l18uJYKbZctw9///vdn+6Gf//zngY8QrJTqoZK+qbSdkt8ux3tpW6mru1XzcGFoy9R2HopOpt/iHVRJjrDab12D+n3kkUf22kpV3AcffHBvHYjwlpylZ6Qj+X3iiSfGj3o4b9v+eEbjqtw6GnVX8/9TTjklfozEM1bsezruuOOmI0facDpXenYexb22czbm3cy/Edue+V2qv3jWxRFwBBwBR2B4CDiBqyG2TuBqCJQHcwQcAUfAEXAEHAFHoAMEMJ7J4xYL8VVG7CZJrbTSSmHXXXftBcVgzpexLK6wkCoDnIxsCqhFMf3myLMs4PKcDFv81qLYXnvtFYlV3Ee4B7GErw+PP/74SgKXjFTxoan/WLRmYR/jhPLHPRZ9uyRwKT0d+fKeBTiMliy8ayEOEhtf4JIfjMMyatp3s++++0bSDR6wEIwoGIghZ0A2sAu2IoYQDoMqniwQyse7wXiIAZP0ROrAExfeJxC7eB0vTP3Hs3yZS90hD3oufa8Knx5tnHZBnIVlvOIgLMjzLsEDTwoQWSQYNPFM1k/wqrTjjjv2gvGuwQlCHgugViyBa5T1mDzwFSx1QPUP0gtlZ17EF9Ft8KVstA0MasSLQCIgTuoGdSRtb7xPPPcQBg9EvB+E35/85CfjOf9k6OxdmHaCYYTFWuRzn/tcNEZzTp2FAEb95P1pnkd6X/va13pE0UEIXGmeIC2RH4z+wo28VBG4uIeQR+o0uIG5JF101fX0WFJ3c+3Utg2lgSGbdoCRnwXkVFhoAGvevcqM9x8Ir4glcOlZ4qT9E6cl8LFAzgK3pGohW4RVwtFWiQs9LY9rXKf+EA7dXCf99BnPlqQnAx7Po+doYwhf40NKQlQXyWu/dhMfqPgHIRhDkQRSGOQP3on6KO5B/pVXn5K2Yo0+Sot3j9cp2hf1XiLiGZ7yqAP0w+gYykx9op1AqqoicNEWwEx1iufot9Cd0i1KqwmBq7TPrqqDlnikMqHjwRvvEQjXv/vd7/aI3yXtVGXMHSGDgBOC7oFgSjuAGKy+G90KQUlS0g/r2dwxpzMod5M+uuu8kD88Qrz3ve/tZZU+l3rDWMHqGgiIImNj1JU3TB48+uijoxdEzq3xTXqtn86w4zywsHX45JNPjp6DaBd4TkHQYcSNrmI8ZHUpBLB+3jRiJFP/5epqTs8TXgQuPasxD3ll7Kdxb0qMSfs8Pa9+uGRMSRz9MMOzYhWBq0Q/77777pFMTNr0v7QdygopVGQhMEGP9+tDiIM2B3kc/HjnEEyZE0Bqp+4Jz5SU1w9P4i4pH89Via0TCkOfQVshr3Z8ig6m/5KU5qVEX5Im6duxG20ZfO3YlHBpPR11euQhJxBk3ve+98VbtE8+0pGU9Kc8O4w4lafSo31PEJZoO01E84GUwJW2i2GNrSEy8MERY0faLnqMPpO2i7dZvTs7V6RcdnzHb0hO9oMTPvRgvMNYB8IJUtI32f7V5qF0fJRr+8xPKC9jNDu2YoyITujX38XCVfzLpUcbZtwozO2j4I++YSypdQnus3UpOkpSoodK+6bSdtr1eC9tK3V1F2JTneTK1HQeaonnrF9BNAJb+jqNR0n7K1/5Ss8TFzqcNq3+kXdMeyE83t+0/gCxnvWQOuEjTbw4I7kPnehrWVckLeaaeGlEStofz2lcxbkV6jGkTojCCLo9Dcs6FeOWNC82nvRccaQErrZzNttPWN1BeiX6K82n/3YEHAFHwBHoHgEncDXEVJPDhsE9mCPgCDgCjoAj4Ag4Ao7AAAhg9GBxEGGBlkW5UsFIqcVHvtbkq00JRgkWLGREOfbYYyOZhPtaFOOcxR4IBDJys7iE9wWRSuyCFOG10MKXunhRkeQ8cGEUwjuXFrB4FiMPQr5Z9JbhCANF1wQuFtT5QhBjN/Ej1lNM6qGF+xDWWJRDWHxjwQrBeEJZkNR4bhdsReBiMZbFJ8pOPvDUhcFMglcuDC4IC7jEzcKgXbzmHgvNeDWRQQkPQHvuuSe3pluoixcq/tk47aKW/WIzJWnx1SYGRsQ+Ey9U/LNfibJFGl6LJHjwkEt9rlkC16jrsX1fKYEGQy6LnyxCIrQVyBr9hO3K+EP4slrbB/LbtjcMoxg6MPYjGA7xWCfyAVhgTEGsQYf6S/vGAC/jKlt1vP3tb49hmfzyLMQYiSULUI+JDyklcLGAzEIyQjrUb6XHl/EYc6U36ghcabuz2KXG3phY5l9J3bXvXe3Utg2SgcyJnsLYglBGSFcStkQh/wjzaN4dC/EYWjC4ICmBKyVaWgIP7xVSitq39KtdyMar3Vve8pYYN+8Rwxm6AiH/6HnpUUvSiAEq/tXps9L0IIvQd6jPwbsU3kyolyL48BsvJxL77tN2ozC5o9U1eLmDPCWxRBERUErbSmr0od/DS43qh/2qm7ojwgV5UftNDe32/VsPXHhjwMsbgmGEPkP9VurZMO2D4kMV/1SnmvTZRKHwtg5SX6jfMsqQN2u8ZYsXtk9CrNe+knYaI8n8Y0yjNpaStGiD1D0R6iAhYZwr7YczyfcupTqjaR89jLyQKdsWbL/KPeo920Hx3ug3RMjnnu0fwArvTOh4GecID97qb+p0hiUjETeEbzxYWgM0JGIM1IyHGPdKhxGe/kOkZFt/uFcnubqa0/PEYQlc6Fv6YeUBHY8eVT9sPTGpHRNHrh8uHVP2w8zWMzsGK9XP6CbiBH+I4tJhlMvmBW9teKnqJ3gTnDDNIySeJMmjhHENuPG+03rXD8/S8int3NHWCd4h/ZAlCVrjru1PSvNSqi/tGIt+g75eYwPGpsyRVEdtvzLq9HIY22vy/ogeYG4hGaQ/HUacylfJsUsCl33voxpbSx/YsTr9BHWO/tTWL/DhusZ2/Gb7R7acRSxJ37afkr6pSu+Vjo/Stk+/dNNNN8V888+OUSx5uK6/6z2cOUnT4yMCtpdH0rkBJDLqteZRtk+23pFL9VBp31TaTi2WXawpaO6ckh1zdTfzKqa7lJap6TyUeQ19FsKHjsxnNC7nGl6A2eIPsfMvO7dhS0Prad2SEdN2FiNK/tk8pP0pQa3+sGsaJe2P+DSu4jxdR7MELdauWMezeLAF7Jvf/GYejVuzMl/qJ0rPzjlK5mxd669++fb7joAj4Ag4AoMj4ASuhhg6gashUB7MEXAEHAFHwBFwBByBDhCwC3FNyQq5ZPHsxGIfUhXPxIkTo/GOMHZhRItiXMc7wvXXX89pTyAIaauw1P25FlqaGIOtO3O7sKWErEGTBaCuCVwQnyi3le222y7wDkgP8oU1MBKO++uvv358BI86EACQusVcu2ArYogtu10UjpFN+2dxPueccwIu+O3iNXnE0KmJjZ5lAVHEPW2lp3u5o43TGgIpvwzeKVGPePDQwUIjGPz4xz/ORd27Zusj3qxYlE7FpidDs31uFPWY8pIPBFxFyrN5tYuhLPQfP9XDXD+xi7UpEcW2N+KyxgPitYYJ+6w1dKaEOJ6zBqTc+yOMrSsY6DFYlxK4lB71EuO3JXCQlvXAZo1CduE8R1rFIKn6Yp8jziqxdSlX9lzdzbVT2zYoF4QFGfVJ2xpUcsQCMOUrfp6V/rJGGgiyhIEgYcXGaw1h0q9WX1M+DOKkAaEMjKzgJYyFeiSHrw2r8zp9Nkh6tu2QX/oJbZmZkm7IS127UV7ToyW/5uLE+IiBB2MjuEPmUd0lrlx94Xqurdi6m9MXEBQgpiBp3VX7TQ00OQKXrYfkmXpoCRbEb/uLURO4aE94bUJyYwauy1jEe4cgQjlK2ilx5cS2q1xbhECGwRLBaATxobQfzqWva/ZdUdamffQw8sJWcCJc4e0Ron4qeGJTG7T9D8YuyKMaS0C2sJ4h0q166nSGJQBV9f/oeNrklClTsn2qCAIYSclXE8npy5yeJy4RuKr0qPVWYb3IqB0TR64fLh1T9sPM1jM7bivVzyo/BmD6JKtfILDpwwDGoddddx3FrRWM9XzsgN5Dv6YiT27grb6RMP3wLC1fmr79betErp3YsYvVraV5KdWXtp/ifemDF5XF1lHbr4w6PeWn6ghplPkn754PIiABIIP0p8OIsyr/Ta7bdzWoBy7FBV6jGltXkWDUbsGAfhxyEV76KCNCHuk7+LCFuQ3ChyQQLhD6fIhJpX1TTu/Za23HR7bt5+Zzdsxq235dfxcLWvHPpsc2uGxnZ0V6mGvgZz8Qsh/4se0w2w8jpXqotG8qbaddjvcot+bOXRO4cvOkqnko5ETGUAjkZn3AEy9M/Ye3a3m8skQt+1FDbv2LcRue8Jhvynuz4swdmRPwEQNi16b4bT2EUVf4QKy0/RGfxlWc59bRLHnbbtdIePsRJGO6dJ2AMKkoPTvvlU4krMqUPpebs3Wlv9K0/Lcj4Ag4Ao7AcBCQnQObDOsEjDFZy+OPc/1umvocUycgz3223/SJmSScE7hmkhfl2XQEHAFHwBFwBByBWQIB60EpZ4RsWkjr0ciSPuzzGOZY4ECscVmLYiyqa1HWPmeN6iLa6L4WWpoQuERu4FkWdeTJQXFx1GKLNfKwnQ7eNBA8g/EFayoYTfE6k36NqPzZ8qbPpr9ZuGMbHxbTMADrK2O7SFa3mGsXbEXgUtkpF2W3xjKlD1GMBVZEC712oZpFOBauUrFfuXLOV5B1YuO0hkBLCCAOXPPjHt96UqiL196zi5XW25sNY78GVr0adT1eY4014naf5AsvEyzIpmIXctlOA9JMP7FtJm2Pam+2jtv4rMcg4cJ9GTp5jvfM0YoMAXV1na3rWIBG9F5KCVwYlnJf5ts8yQhv82SNAddee23gy/JU2P6BL3utUTINY3+X1N1cO7VtA8IHi8ZW2PKL94Ng2IZoaYWFe203KjKlJZpggOU9pWK3MLNtUvpLC9ngDe6IJRWk8Wlruao6loav0mddpIexfokllpguSfoajDsYTqzUtRsbzp5bnVG1/QiGMLb2ox7i/aC0rdi6i25ki89U5Fkorbtqv+n1HIHLGu7SvlXp0UfRvyDW0Kj7VUfVqTTenNdM4lB41UGuyZhCP4EuyglbL7GFCSJdU9JOc3FzDRKj7Q8xEmMww6Cmxb/02dJ+OI3H/rY6o00fPYy8WFIY7w3PbanYbY1uuOGGcNJJJ/WCYJjnfbI4aiUNx70qncE9S0ZKSf/crxLiJA+MhSZM8+Y0bAIX28+KdJnmS6QGazBWO67qh9M4+N1kTNkPM1vP1EcMop+tsRMSBB9usGWavIHmytH2GoR/3idkSsZa1Ku0T6rDc5Dy1eXV9v05ozJzCeYUyB133BEgLw6Sl1J92aSfyvU3o06vDmvuWY8xdov4QfrTruPEcw5zzVTQofT1/USYE25QAtdYjK2l6+xYnbLYMRleg+hj8aiDZx36/7vvvjuSue36gebjdqxT2jfl9N4g4yPb9o866qg4z6WcEjzbaet6tX3u1fV3ejZ3tOmlngkJr/Zr8VM8fAzCOAHRVnmD6CHFa49N+qbSdtrleI88a+5s+2OuV9Vd7lWJLdOg81DeySKLLBI9h26xxRa9LW8tgcuSgskTRCY+1oFEmFsPqsq3rltCFiR4ebaivVCn6GstMa20/ZGe5gCpblBeLJHYzhOsdy6rC/Rc1VHp2bia9IW59Y2u9FdVXv26I+AIOAKOQLcIaA3HCVx9cHUCVx+A/LYj4Ag4Ao6AI+AIOAIdImCNkHaxpW0SduFCX8nm4hAxAk8w+kpQi2IYcSBQpWIXiiyhhHBaaGliDMbYSnlzC5VKU0Zfa+TpgsDFIrzyqrR0ZJGJrx9XWmml6H2CxbicDELgwtsEC6V8vazF4TQNu1ir7bfs4rW2/0qfs1/bNzEc2DhlCCRO0odcxn0rvAu+CoZUxoKjtnewYdJz8oHhDmEB2noxUlhLXFC9GnU9liFCeaKsOZFBu+792efsomEVgauqvVmSkHAhbhk6U5Ii9yw5M92mhvuSLbfcsueVRlublBC4rN6q8rBCmiIS2YVfu3AuT3PKn44yajRd9C2pu9awIqKlbRt28Vj5YmsMDH2IjFm6x7EfgavKUIAOQjcjlrAgnaW82K3FCNuvvhIGnU5dq5Mq41QX6VFXqAeUUXL66aeHq666Sj97x7p20wuUnFidwXY4kJnqZJC2YusuW8LigScV9XNp3VX7Ta9bPagtFG2fy1apbImcijXkjZrAJWImeWpSByG6YQQvaadpue1vSzqz1zHwoQfpr+Q1k/ul/bCNOz23OqNNHz2MvNit36reC/lXf5brK+w2oISl78bbQxpflc7gGUtGwpifEjUJg2CIJz3qBd7rcmL1Ye6+vZbqS+7l9DzXZRDMeWDiPiIPIpRdXqPUjnP98HNPhajr2o4p+2Fm65nGbYPoZzwXvuMd7+jVBeWduQEfk0CEpL9qI3g0xfsO5DvGu6pnNg6LJdfr8BykfDbN9NzWidx8yY5vNLcZJC8l+hKjvDzCqv9Py8HvHFlm1Onl8pVek26yH0EM0p8Sf5dxam6c5rtufGvDdkXgsnWvLu2ux9ZVJBjsQ5QNEYmINgPRiXkhfay2UqfeMXflSNu3ulXvinjSvoRrEukM9U05vTfI+Mi2/dzc1JZXbZ+81fV3ynvu2C89jQPScSFx0S+mBK5B9BBxlqx3lLbTrsd7WqvqmsBVMg/FU9tGG20UvZbrIz/wtWIJXISZNGlSrEc2DOfMj9H3fBSkLXLTMOlv4kNn0V7sWATyuz4EnDx5cmBOgZS2P57VuKpuHU36067v2R0G0i0qibdKlJ76vUHmbLY9D6K/qvLq1x0BR8ARcAS6RcAJXA3xdAJXQ6A8mCPgCDgCjoAj4Ag4Ah0hoMV2S6rqFzULWSz0IpAGWJzBnXpqHEnj0QKYXWTRNb50wFCXil0stYQSwmmhxS50ct0aVrWFgspZlQ7P6YtNW44mBK4qI5Dyp4Ug0rACyWi//fabjlyg+yI9YIhCBiFwyVhoiSxKR0e7UJ0jcFV5H+uKwEU+wHrHHXeMX1TnFiWpN4ceemg08CnfuePnpnqpYhseRJ6I0nBgL49vqlfaFsC+//Q5fqvODlqPRRjMpZG7VkW6SsPWEVGU96p20I/AlcsDXrUgsiDy3pbmid92If7SSy8NeENoQuBSO5KXQGtEqKqXpCcvH7be2zzgfStnJG5L4CKttnXXGlZyBK5cuSyB67TTTpth+4x+BK4qQwH5l4547LHHolGba8Jd+st6z+F+E8kZqdLnqoxTXaSHgYF6wNaOktw2HNyrazd6Nj1SPgxFCPqDdlUng7SVJnW3CwKXJZZWtZGxInDxPqmrbQQCGkQ0pG077ZcORqJNN900ennJhRXZhXtqY1Yfpc/k+uE0jP1tw+d0BmFzffQw8iJ9a/NXd54jnK2zzjoBr3aSKq9iVTqD5ywZqUr/VPW99P14U8MLEjJsAtfNN98c8BKWEwg0lNOORzTWzPXDxFE6puyHma1nqtOD6mc8P6JrIGOIOGFxQJdC3tBY2N5LzzfYYINIbk7jATvICcxXGFNaLImjDs9By5fmUb9t358bn1oSjeY2pXmh7CX6Eg+btBEEr5FHH320sj/dUYZ5EUBK9XNpetNlpuYHW+pB7kMYA0P+GaQ/JZ4u4xQBgXit1JGobLiuCFxjNbauInBRRvUrkHnR55rLQ946//zze54wRZjeddddIzR4y6ZPtHHEHw3+qW/K6b1Bxkf92r4lfKjtk926/q6uOP3SE4FL2Nq4cgSuUj1E/KV90yDttMvxnubOXRO4qsbYuXkoRHPaAGTlVFjDo8/U2oclcBFW5Dm8UWpdKY3DeihM76W/991330iC5zpzRQhW6g/oZyFaqu9WG07jqPqt9qe4OWoemnvGjue0NTfzajwFI1oHzD2bXkvnvYPM2YhbZVcbK9FfaR79tyPgCDgCjsBwEHACV0NcncDVECgP5gg4Ao6AI+AIOAKOQEcIyIBBdCeffHLcyqQuajxFaYFUJBBISEsuuWR8LPdFueLTwoX19qVFMcWlsDp2ReCSYbvK8EV6OQIPC4D9tlDUIiiu6LWtFPGlC0Fcs6KvmLkGJmwVBAEGAhXbQ+BtB9IGMgiBSwuBdR6cLKHplltuCXhusovXbYzDMcMV/2ycMgTmgmLcW3311eNWkjKmEk6GotwzumaJLLw7DOap2O0LReAadT223kYwPuTIRDbftBE8CPSTOiJKv/ZWQuCyRgd9uZ7LI15BaM/IiSeeGOt6PwKXNWaKwGWv2YXeNM0cSaGJMUDtpUldS9Pkd5O6aw0royJwVW3TiVGAMiPWYJjqL7YilCcYdNSpp54an6n6h0GhyZZYVcapLtKzZF7lEwMMuh4CppW6dmPD2XMM8BOmbbdGH9OvvIO0lSZ1V/1cWnfVz6fX+3ngqvL0ZbfdHLUHLm3xxJf/kPH6CdvU0Z+m0qSdps9U/eYrffoUtiyl3tLPSdgal7YnvdK2H1Y8uaPtT9v00cPIi4x35BPdQfuvE8ii1EcJbQPydUretgQ8ha3SGdzvR0ayRmgMjRjK2cKPLavoYxGNVYdN4LL6NiZs/h188MGReGoNxmrHVePY0jFlP8xsPdO4rQv9THExLDOvWHXVVaNBmLYkUVr6nTuyVSIEFpG37r///ji2gBwnjyJ4d1100UVbEbi6Kl+aZ9v3NyVwDZKXEn3JmJs+EsEwr7FAWhYRh2y/Mur00jxV/VabFmlykP5UaXQVJyRG6yVU8WP0h1DaT/QeCNfEE7Li03xAOmasxtZ1BC6rm/iAh3kaAumNti6CBKQ89Okqq6wS+3u7vXJp35TTe3ZNou34qF/bt+PD8UjgGkQPlfZNXbRT6sug4720rRAnUld3nwsx4/8mZdIYzepWS1RivQldxtjlnnvuiWQptlKUJ/uUwGVzwZbueMFk/M8z6jsJA3FcBmz7THpu5w7kAyIa8w/i0lxdz5S2P55X31NH4LL1kjWBI444ouchzHpdVH7qjml6tk22Xd8gnS70V11+/Z4j4Ag4Ao5Adwio/2M9gPUI+jQ+WuSPc/1umuIcEydOzO+v0TSGcRqOztHFEXAEHAFHwBFwBBwBR2B0CLznPe+JhhNStMSqqhxYcoy2KNhpp53C1PFpfOSUU04J11133QyP2wUWFl1ZfEW0KDZsApc8pWCwY4EL428q+grafqXP14wYFZEcWYQvGSFwIW0IXBidWNBEWKDDWJIame32XIMQuKwnB3CwRtOYgan/LJnooosuil8228XrNsZhxZk72jhlnMNwIa8bENgwpFrhHbBVC5MnBGNc7v3pmXe9612R/MXvs846K26zoXs6sn0PhlxEBK5R12NrSK7a3o6JIluEgBGLohAB+kkdEaVfeyshcJEfkaWq2jFh7CIudR/jON5rtthiC26HnDHEejmwi8IyElZ5s8EgBpkUsWFKF85jRJl/pXXXGnJGReCqMsRClKTNIFdeeWVsD5ynC9ksZGCwRKynrnjB/GNrDwwEGAXZvq6fVJExBk3PkozQ6ZAeMFAi6rts3urajQ1nz/HqQR1FtFWfvc+5DFciXZS2lSZ1twsCF2RsGUmryFlbbbVVQFcgVWHizeSf6pQ1ThLEEu3s1/IKb403wpN3irE67TeJDyIVfwie5yBNddnH0GaWWWaZ2A9B0Erlve99b1h55ZXjZfqzI488MhrGqOvku00/nMZtf9v+tE0fXTomsGmn59a4eNRRR8Utr9IweGpF5yOMEzGISbQ1Fr/R2fIyAV4QlyyRoUpn8Kw1mIEzJAgrdlylvt/etwa7YRO4RJyw6XMOiYl6Tv//yCOPRJIC1+sIXIOMKfthZuuZxm2l+nmxxRYL6623HsWJ48yUlG77I2u8jg9k/tltm1K9ouD6yIK6JBIy9+rwLC2f0qw62r6/KYFrkLyU6EvGceqnquoodQKSAXXUvqdRp1eFc3pdOlmecwfpTxX3MOJU3G2OXRG4SHMsxtZ1JJhll1027LXXXhEO1g7QH3qHXNxtt91iX881dP18880XIG1TNyWlfVNO7w0yPurX9m3fY3VZXX+nMuaO/dKTXpR3IBtHzgNXqR4apG8qaael8zJb/vRcc+dUH9bV3TQO/W5SphyBS+8rNyYibjxRbrvttjEZS+DafPPNo/dZPrxhO3ErvFPWVzTeOuOMMwLPNhHlkbkNXrU1l+cDQD4ElJS2P57PzQEUrz3q/aAHWPfR1qoXXnhh4K+p5NJTX9h2fYM0u9BfTfPu4RwBR8ARcAQGQ8AJXA3xcwJXQ6A8mCPgCDgCjoAj4Ag4Ah0hwEIXX/rjhQXR12ssgqTCgsi6664bL7OAxEI9X7jz5TxEMITfLCanYrfxsR4VtOhStTBiv3ZNjW1aaLELnaSbMwZrkZf7lqTAb8SSd1IjjxZvGNRj+LSyxx57xK8YudaGwIVrd3nrymHGgufnphLHeD/IIAQuGRmI5+qrrw5sv2YFYhTvUtuM8f7Ik128bmMctnGn5zZOGQJJH2IIhiC2UqE+pmKNrjmjrA1viVGWvKMwLFiy8MgRUb0adT223t2oOxDTqHtWttxyy7DJJpvES7l3Z8Pq3BJR2J6Jr2Ml/dpbKYFLhlDSOfzwwwPv1goGCEiKvGMRWbhvFzdTUg31gnhpC4glcFmDlW0bMeDUfyIo8NvWgdKFc8WbHkvrrjWsjIrAhU6nnVsiBOXBSx11EUHX3XffffFc+tWSZ2RAIACGCwwCVqwnv6bkhzrjVGl66E08M2irkMmTJ4cpU6YEvDJQBxG2hWJ7KEldu1GY9LjCCiv0tprKfemdIyCWtpUmdbcLApfV0dQZiDXoJwmGHtqX+qZRE7hsX3DxxReHn/3sZ8paPKLXwYFyQO6CdI502cdY4jvtBHKkFfpS9WP6ar+0H7bxpuf2XbXpo4eRF0uksSR9m2f7AcBhhx3WI3DRx9HXIZCzJ02aFPbZZ5/ozYxr6RipTmfssssu0QsLz+XGChD48TqB5N6dHec21WGKi6PVlzk9TxiNKTnPbYdriaEi1BNWusP2oVxHBhlT9sPM1jON20izRD9DbKT+IVVeIRVvnbe6GMHUfxisMVwj11xzzQyeIVMib1MCF/EpH5x31d/ZOtGUwDVIXkr0JXqTsYLGXqlBnvxYj6qWwDXq9MhLE4FUzkcECN5DmfPy4QpStY1ZVX8aH5r6bxhxKu42RzseHsQDF2nauEY1tu5HgpGnM2FiPRfaeZvus7UielNS2jfl9J691nZ81K/tj3cCF3iW6MRB+qaScW/pvEz1JXfU3HksCVxqB6wTpB8vMK9h3KQt3S2Bi48iGLej13kuXWewxK+qD1FymNiP5eir+UCG8RtrGVZK2x9x5OahNm6d2zU8xkfM/Sgn8xd+N5Vcehp3EUeb9Q2lqfem3231l57zoyPgCDgCjsBwEXACV0N8ncDVECgP5gg4Ao6AI+AIOAKOQIcIWCM00UKmgiyC5wg8reDJgu2BMM5LLrnkknDeeefpZ9ACERcwWOJxgsVNFtK222673hf3qeFJi2LDJnDxVS55lPH+iiuuiAu8LHZBSuOrfwmLPjkjD/chf+HRA8ID2xvKixP32hC4yAdfOis/MpBgfGY7GQx4ItUR97HHHhu9xnBujcN8OQuGGFBYnLMLxCKGWG8OPH/BBRcECA0IC24YXPiqGbHGR7tQ3cY4HCOq+GfjtIZAazQ499xz49ecioL6h8GPupRbHFQ4e7QLbhjXcalP3QMLjMgirPCMCFycj7oe77nnnmG55ZYj6Uheo06wEIpApsKoTR2hTlKmlHwTAyb/7II3GFO3FGe/9lZK4GLbEozACO0eDywicWFsZ0FXZBraz2WXXRbDUt9Z3EQoI4Y0thJdaqmlomGWhX+JJXBZLx2khxcciCRgiQco1WeeHSaBi/hL6m6undq2kWtv6Bu2VUVyRn9LjpBhmLYDeVXCwsQxxxwTSVq8j/e///1hwoQJ8XZqmMgtZK+zzjo9T0a0Jwx8f/rTn+LzSy+9dNh99917eqvpl9x1+qw0PUustcQqu8iPLoFUyBGpazcxQMU/Sxak70NX094WX3zxqFulx08//fRw1VVXRXJJSVux+WtrcJY+tIZ2imO3QbH9Atsk0w8h9DHHH3989GrBOAAPSnabs1ETuNjeC8ylF9Hf8hiAt0aIP+gc5Pbbb4+6iPOSdspzObFb8FK/IOSgZxCw4f1CHEEgmEE0K+2HYyQV//rpDB77f+ydCbhvU/nHl4hu9SCZh1xcswgZK/OQ4aKUhOaJBjSHRCUaSCpzGTJHMoZKLpFCkkskEuoqyvQUadD/fhbv7/+eddb+7b3Xb7jnnPt9n+ec3x7WXsN3rfWutdf73e/y5Hkz7g8iL6Rl21lxzNY+eGRFr6PnGcNXW201bsU2BbkKoa6M3Mu5kY7JI+2WZxE/5+ymM8Ce8QjJEbg8+Y5tt9iGkzZOPiADWbvneT92cN5Ncvoyp+eJwxO4wAePEXjhZA4HkZQxDGEbSsZOriPWj9N5NPd6mVPWYebbmZ+3lehn9CEkAPJL+ZgrmCc2SKHoF/PSxtzNcKWMOfGEMHQ5cyi2sqWNoDPBk7QQsIbEyy/SDU/ul5SP57qJbxM2Tvvwfhs7/3FKaV5K9aWfY9H+6JeQ7BEM8vQVw9WPK8NOj/z4ebttrcf1VPgohjGCfg95sFcCF/EPIs4033XnfmxjPLb2XfUc7+6829v7gJ/7+Xof1tza8kG+IXPRf03nUQY/t+Wc9wfeIxDGB+rc2iLXGE9YU/BSMjZV6b3S+VFd368icHUb73wZ0+O69IyM1dQDF/GX6CHqpnS9o3Te6/tEP9YUrI36vgIedr2q7RImlSZlMu9WXrd6PccaHXNf+ihe4XbfffeAh1MTvJl/5zvfiaf+A0bmxCeeeGKnf0FCZc5sZF30WeoV0+JMf72XNrvn07Vr/Jb0P56z8d+vTXE9FcYdPIl5qfqQwIdJj3Ppla5vWNz90F8Wl36FgBAQAkJgcAiIwNUQWxG4GgKlYEJACAgBISAEhIAQ6DMC3mtVXdS57d4wVrJY6BdRWViCdGPXWODCoIdhz8QWvwZN4CI93MhvvfXWlvSoX/JHXvn1BC7vkWHUQzMv2HNtCFzEs/fee4dll122E2WKFwuFkyZNivdJA28xLLwhhls8mfnPvHz4BVsjcBEG4gcEEBPiY4HcjKNcJ/98cY63C8QvXucIJYTJGYe5XiU+Tm8I9EY4ngUL8gPJxNoP15t+HYpnJ8hR/lmMheY1xh97Atew2zHGOhaawcWEcpNPn/d0AdrC5n4xfLNY6sW2krR2U9XfSglcpJUuUtK+aGe+jaWeVHgOI7t9Ncy5F+tbXPMELs4xfk5+jnzEeZV4I3zpwnlV3Fwvabu5fur7Rq6/9YPAZeXwuHKNc4xg3qNWbiGbsN6DDef0VcTXc91ie3zA/bN2aZdMn3HeNj0IGBjYEMqFIYQvnhH6FJ5F6COIT6dbv4mBK/7RBtHljHUmtH1/jlczSBsmJX2lSdut8hhiRAVvDCIvVQQuSBboJSNdWr7tlzq3+h42gYs8eO8vnFPP6HSvRyHRQfRCnyIl/TQ+WPHPDPfcJn1INWDi8wDe5MH6SMk4XJF8vFynMwhUNUb3Oy+klY674AL+RmIkDNcg7WJcQzC6YXxDUr3hjcU8h8EZ4z5SpTPqyEiQgul/fnz17Zl0fFsi/6QF4aGb5PRlTs8ThydwdYsTYrLfNtn6cY7ARTylc8o6zHw78/M20myrn3nGe+7gHOIVuNNOrF4497qbcDmhz+HtzusqP7/jGfCy+9Q15A+2kqrDk2dLysdzVeLbRBsCF/GV5qVEX5Ie5MHFF1+cwyjUCeLriON0XBl2ep4cgG4xUnnMrPsHMXLq1KnxCsRa8/rXlhDtooxky37H6eNvcuzJKk3C84EVH2iZDk1JKcOeW/s+Qf7T/Pj3Eu4zz7FxgHM/Fle925SMTVV6r3R+5MuZ6/tVBC7KaHXFMeLnrs9eGf2/Lr0SAheplOih0rGpdN7b7/me4Z+2TY8x2KT3uZZKkzLlCFxefxFnqo/TtQP7eAVyFmOdvZPwHGMuY6etiRBfkzZFOC+eVMZ13q/wXJpKSf8jjty8Ko3bzr0e4Nqpp54aP+yy+01+q9IreWez9Pqhvywu/QoBISAEhMDgEBCBqyG2InA1BErBhIAQEAJCQAgIASEwAAQgr+CdwL7ES5NgYQpvPizy5AQPXSxmGenIh2GxCNfjfFHvBcMLX5c+8cQTo76eI1zJFoq77bZbxzMWxlObjBMfBmu+VDQDPtfI24033hgWXnjh6DWDxS1P4CKM99rAOUI4PKpAPgE7Fs9sW0Tum5GualEMgxLeAPgC0gvxst0XWHuygXe7zpZDENJsQe7ee+8NRx11VNhzzz2jEZX4PIGL8w022CBARrNnuGbCghv55UtcExb2IHUgbMPHF/ipVBmH03B27uNMDYHbb799NIaYccie4RdCBF9de69v/n7uGHIcHoa8QZ1wGPP4Gti2U/EELu4Pux3T/uk33msU+UBom3hv4a+NeM9ePAdueDCp629+odHjYou0dYvTvu+l+a1qQxhDMBT6r4d5lv5EHl7/+tdHT3EpgYswEDJZ1DaCAH2HPGL43mWXXeJzfhssvO3RB5Cqxd3cwnl8oMu/tm03109938hh5Qlcp59++ggiLFnzC8xmHPIeuCDeYtQwQ7YVhzaGAfOWW26xS/G3m/7yW3z5h8D/1ltvjdjSZ5tKlT6z59ukZ/XHs7ltR5deeukR+t1vpVjVbywfVb/ocAxUflwhLHjgteS0007rEIksjrZ9pUnbbUvgwqhCW0RSkijtEa9JeLWzMYPyYBRibMKbE+K3aokXuvwz44j3LENwj4Ufs7u1wW7jGUREyM6pJ4G2/bRLUaLHJuqcrWlygt7Bu9DDDz884na3fOfG4REPJyd1OoPg3cbofubFssb4ybjLuJYK8zx0DeQBBE9L2267bTyGWEPdQwbx4vWaJwBX6Qw/V2P7HPM+6eOkL+F9x9q13WPsoN2wxaKNE9zr5tXHns211ZyeJ7yFRSfjPSMlMKM78XqHLvVSNw6XzinrMPPtLJ23kb82+pnwGI7BhnlzTpij8bFHWv5cWK5VzdloU8x9mE8z37P6njZtWrjgggs6npvq5jVty1eVT677NmHjtA9f5YHLwpTmpVtfr9KXzMUhQzOPSAViIV7rGB9SAhdhh5me3+4xJff4fNPuzFsT72XW/qrmglXj6aDj9PE3OTYiYpOwhDECl70P5Nr/MOfW1APeW21uypyUdwITT66HnMmY4MXPH6o8ABG+zdhE+G56r2R+VNf3uxG4qsY78lkldekZgYs1EsZeL967Urq9PeHa6qHSsamXeW8/53tVfaWu7XpM7bhJmew9JtWtKfnZ4mSeyfyavoBXfYTxz+bp6HDGedptTpiT4Z2Z/tVG/IeRRhirer5t/yMemytVraP5tHxeUh3iw3U77pae1zNpHLl3dgvTL/1l8elXCAgBISAEBoOA2Yz4GIB3Bt6DeHfkj2M7b5r6HOuvv/6zn740fWKchBOBa5xUlLIpBISAEBACQkAITGgEIFKwKAWhhMUcvIdgcPXknioAmOBOnumRBMMwC4B4WeBZ87ZQ9dywrzMpp3x8xYuxCNl///0jiatq4QciEOXiObbgaIpJXdnAGiMIJBQMVixUYShHLE1Icffcc0/HOxb3MPSwbRn5hRiXM1QSzguLqDzDH3FC/MKlfp13CR/HII9Z6FprrbWiZzLyypaBM2bMiOSSJu0vzRv1DGlvueWWi4uZ4Et7rpNZ0Y4XXXTRmFfaFy+O1CnGiFLBuIa3EV5GKXMbQk1pmjzHdqW0ZzCHhHX33XfHvpIa5tM0II6y8Mz2lpC16F8sQDcR+gIETPSMlROSHvWYej9qEl9JmH633ZI8pM94Apdt5QbZiK1f0S30fxaeDbP0+W7nvLtTXxAlGSfQiRBa0UclUqfP+p1eVR576Tc8yzZx5BUvILT9bnqrtK9U5X1Q1/GQxPhkXi8Y44184LdeHFT6VfGSpxVXXDGOy7RnCD6QsbuNZ/3up7T/NddcM+ot+gEYMVYzjlfJWBqHB5EXFjghJjH20naoDzyI4mGrn1KnM7qlZe2A8ZY5Bp7kIJiZQOJi/GR+iEfEEh1pcdX9Ug4IL+gNyEa94lQ6p6zLZ7f7JfqZd4111lknEuYYqx966KE4hjfZBi7NC88zd6TdMddg7mSeFwlL/piToCcgJDSZL/s0Ssrnn+/ncWleSvQl+WZcMx3HuMY8um4+x3PDTo80Jf1FYJhzayOLM4+3LYn7W5pnvcf1e2wa1vyol/Gu3zgSX4keGvbYZOM887R+rClU4TiMtmtps1bAux3jJ4RqxjP/3sU6Fe/TzId5xzNhPYTnmJdxn3UG5qvMb3h/G4YMcm7IGAVJDcntFNCP8o2Xd7Z+lFVxCAEhIARmNwRE4GpY40wAJUJACAgBISAEhIAQEAJCoN8I4KUHAxmC9xrIMV4wNOAinwUu77HHh9GxEBACYwMBPANhXIcghkeFlOiFkQDPM0jOC9PYKMXgc5EjcA0+VaUw3hHAWw1jIZ5Zcp4Xd9111zDzo8JYzKOPProrWWm8Y6H8CwEhIASEgBCYHRDQ3Lq+ljU/qsdIIYTAsBHw3lq9J99h50PpCQEhIASEwPhEQASuhvUmAldDoBRMCAgBISAEhIAQEAJCoBUCGJsxOiN85c/WGOZRgS/3DzjggPiVOfevuOKKgFcRiRAQAmMTAb/VXbqFG94iMLDYVhEQM4f1dfFYQ0sErrFWI+MjP1/4whc620HadkuWc/+Ve7ptsIXRrxAQAkJACAgBITC+ENDcur6+ND+qx0ghhMAwEMArHd40/UdbePDGA7dECAgBISAEhEAbBETgaoiWCFwNgVIwISAEhIAQEAJCQAgIgVYIsG3KoYceGrdP4UG2OnrssceiS31cyePWHcHjCB59bAvDeFH/hIAQGFMITJkyJXrYsn7Ldj5PPfVU3Bb0hS98YSev06ZNCxdccEHnfHY7EIFrdqvx/pR36tSpAa+VJmxnhJc7xkq8VSKMkSeeeGLcGs/C6VcICAEhIASEgBAYnwhobl1fb5of1WOkEEJgGAgcfPDBYb755uus4ZHmEUccEbewH0b6SkMICAEhIAQmDgIicDWsSxG4GgKlYEJACAgBISAEhIAQEAKtEVh00UXDPvvsE0keuYdF3sqhomtCYGwiwJaob3jDG+JWb7kcXnPNNeH888/P3ZptronANdtUdd8Luscee4S11157hGHEEoG8ddJJJ4Xp06fbJf0KASEgBISAEBAC4xwBza3rK1Dzo3qMFEIIDBoBCFzzzz9/Jxm993eg0IEQEAJCQAi0REAEroaAicDVECgFEwJCQAgIASEgBISAEChGgC2gVl555bDAAgsEPPf89re/jYboJ554ojhOPSgEhMDwEZhzzjnDxhtvHJZaaqm4iMv2qHfccUf0CvSvf/1r+BkaYynOM888Ya211oq5uuuuu8Lf/va3MZZDZWcsI8D6zCabbBIgP0+aNCncd999sW/dfffd8lI5litOeRMCQkAICAEhUIiA5tb1wGl+VI+RQgiBQSLAeh5eAx999NH4bjJjxoxBJqe4hYAQEAJCYAIjIAJXw8oVgashUAomBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEQGMEROBqCJUIXA2BUjAhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEGiMgAldDqETgagiUggkBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASHQGAERuBpCJQJXQ6AUTAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICIHGCIjA1RAqEbgaAqVgQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACDRGQASuhlCJwNUQKAUTAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCoDECInA1hEoEroZAKZgQEAJCQAgIASEgBAaIwCte8Yqw9tprh5e+9KXhwQcfDKeddtoAUxs7Uc8xxxxhu+22C3PNNVe45557wvTp08dO5mZxTtZdd90wZcqU8Mgjj4TLL7+8dW6e97znhfXWWy8sv/zyYckllwzzzTdfeOaZZ8ITTzwRHnroofCjH/0o3H///dl4eWbVVVfN3rOLjz76aPj9738f/vjHP4b//e9/drnzu9FGG4UFFlggpnnRRRd1rucOaAfbbLNNmHvuucNf//rXcO211+aC6ZoQmGUIrLHGGp0+ceaZZ86yfCjh8YkA6y6bbbZZQNfdcMMNYcaMGeOzIAPO9ViaE4ylvAwYdkUvBCY0Ajanffzxx8NVV13Vl7IOIs6qjPGOuOGGG8Z3xBe/+MXhP//5T5zL//nPfw6XXHJJ+Nvf/lb1aF+vSyf2Fc7ZKjLeZ1dbbbVY5uuvvw/9HugAAEAASURBVD785S9/aVT+V73qVWGhhRYKvHNeffXVjZ4pCVQ1R1t22WXD6quvHqOkr9H3hi3qdyMR72ebGA/YjoU2OLIGdCYEhIAQEAJCoHcEROBqiKEIXA2BUjAhIASEgBAQAkJACAwIgR122CEadi16JrIHHnignQ71d/LkyQHiD6Shxx57bOBpP//5zw9f+cpXYjp33XVXOOaYY3pOc9FFFw0vfOELw5NPPhkwboxX+fSnPx0WXHDBWI7999+/VTEgAn74wx8OGHq6yZ133hlOOeWU8M9//nNEsPe+971hlVVWGXGt6uTpp58OZ511VrjllltGBPnSl74U5plnnnht3333HXEvPSGfhxxySLw8K9t/mi+dCwFD4P3vf39YYYUV4mlde7Zn9DscBMaSzq/KCwb4t7/97REQCK0/+clPhgPOOEul25xgIsxPxll1FGeXsX+JJZaIz//pT38KzBMkQmBWIfCa17wm7LzzzjH5z3/+830hPA0izhSf+eefP+y1115hkUUWSW+NOOdjim9+85vxg4kRN/p80k0/9zkpRTfBEHjXu94VXv7yl8dSnXvuueG6665rVMLDDjssTJo0KTz11FNhv/326zxTNdfqBGh5UDVHY97GPeSAAw4I//jHP1rG3Htw9buRGFa1iZGhmp2NB2zHQhtshmZ9KD4mZI2KD//uvffe+gcUQggIASEgBCYsAiJwNaxaEbgaAqVgQkAICAEhIASEgBAYEAIYE2xOBpHmgQceCEcfffSAUquOlgUVI4796le/Cqeeemp14D7dGcTCGYQw4uVL+4MOOqhPOR1+NKUErle+8pVht912i0Q8yzVfDEPI40vTeeedN+Jj9yDrfe5zn7PT+NuGwGUPnnDCCeE3v/mNnQYRuDpQ6GACICAC19itxLGk86vyUmUcHLuozpqcVc0JJsr8ZNagOvxUX/va1wb+kJNPPjn8+te/Hn4mlKIQeA4BPkxBN88555zhl7/8ZV+8HA8izrTCDj744ACJywTyCB+nQGjhQxXyYIIx/KijjrLTgfxW6eeBJKZIJxQC/SZwVc21SkGrmqONBfKM+t3IWhWBa/gkwpE1UH72kY98JLzsZS+LBC4+NJQIASEgBITA7IuACFwN696MhQ2DK5gQEAJCQAgIASEgBIRAnxE48sgjI7GGre0+85nP9Dn25tHNCgMphKJdd901GlXwBnXTTTc1z3BFyH4v6lYkM/DLJQQuyFmf/exnY3sigxACIQNCCvSy8cYbhx133LFj/EkNWp7Add5554Ubb7zRPx4WX3zxuL3j5ptvHl7wghfEe//617/CJz7xiU44Ebg6UOhgAiAgAtfYrcSxpPOr8lJlHBy7qM6anFXNCSbK/GTWoDr8VD2B66STTgq33nrr8DOhFIWAQ8Dmtf/+97/Dxz/+cXen/HAQcVpu2FZ86623jqeQtpjL483Oyxvf+MbAdmImeMP9xS9+Yad9/63Sz31PSBFOOARKCVxbbrll9EDHNqGXXXZZB5equVYnQMuDqjnaWCBwqd+NrMyqNjEyVLOz8YDtWGiDzdCsDyUCVz1GCiEEhIAQmF0QEIGrYU2LwNUQKAUTAkJACAgBISAEhMCAEPja174WY77//vvDV7/61QGlUh/trDCQ1ueqfYh+L+q2z0F/nighcH3gAx8Iyy+/fMwAi91f/OIXA8aqnKy++urhne98Z7yFK3sW1fhFzCjF8dlnnx1+/vOfczhKXvSiF0XvXXg1QPDkhUcvRASuCIP+TRAEROAauxU5lnR+VV6qjINjF9WxlbOJMj8ZW6gOLjcicA0OW8VchgBbehpx6zvf+U64+eabyyJyTw0iToueDyL4WAJhLl+1Jbw37kOUhDApEQJjDYFSAldVOarmWlXh665XzdF8/5pVWyjW5V33JzYCE6kNisA1sduqSicEhIAQaIOACFwN0RKBqyFQCiYEhIAQEAJCYIIhsN5664VVV101bsUAQWPy5MnhNa95TSR/sNXaDTfcEL/ihfyB4Yyv3ZZZZpm4bQOLyGyxd/3111eiwjZua6yxRmBxG8EDEAvLLJgbScQeHmRecDu/0047xfItsMACMUmILXfccUe49NJLLQud3zXXXDPwx2Ty3HPPjcd8Ac2cadq0aWGppZaKYadPnz7KK5FFgkcptrb4+9//Hr773e/a5VG/GLjYGmP99deP9/CWdMsttwSIXD/72c864duWwR5s8xx5fslLXhJWXHHF+DjbdNxzzz0xP9QZHpsWW2yxWI/kjYXOddddN7pBf+qpp8J9990XLrjggoibpc/voosuGrbddtt46aKLLgpzzTVX9Li10EILxfZ1ySWXhF122SXep+x44UKsTUAGIt6VV1454gT+kIX++Mc/Rm9dtEOTTTbZJCy77LLh5S9/efRA9cwzz4TbbrstGj1+8IMfWLCuv8RNW6ft4s2KLzP58v3BBx+M7SU1oJAfwv/3v/+NRCfqc9NNN419hTbw8MMPxzL98Ic/zKbL9ievfvWrw0orrRTA5NFHH41bDeER62Mf+1hYcMEFY/r7779/9nl/kfr55Cc/2bl0+OGHR5w6FzIHLEaTLuK9ZDQlcPHcpz71qVjPHJ955plRd3A8aALX3HPPHXbfffdYR+iWnPe2rbbaKiy55JLhoYceCrQ1L9QV3gvADSIa/Y9wxFPlwQA9MHXq1Fi/PEMbo6/87ne/i1+Gc5wTvJRtscUWUb9yTN+iH9GOuc412nLqXYG42ujSXNq5a03jfOtb3xr7LOPAaaedlosq9jn6HkKZrA9zTvu2ctPOiAeswPj3v/89QUYIXuEYb/7whz+En/zkJ3FMoj+jR55++umoG9HDqUe5EZE8d0JdUb8IeF999dXP3Xn2B/1rxtGLL7449lUf4C1veUvcapRtQSEwpgQuxk9wXG655eKYRt3RbrptFdYUd/JRogN9/rsd095e//rXx3Fxvvnmi/qLMe+aa64J1113XfbRVVZZJWy00Uaxv7B9E3oRfQh2abul3jE4IMwTwJ9xdMqUKVGnoeeoY7wp5PpM075Ju2ui84eh1+vykhoH6QNveMMbIkY5/RRvzPxn4wnzJuYTObwsrMedvgjGzC9od/QH+uZPf/rTzriw4YYbxrEcHUh9Wp0wB8zJMNoAZUjnBHXzE8trmzkPz3Sb76Hv7rrrrlF54ble+2Yv4wHp54T5O3VN3wFD6pB++f3vf7/TZpjT9NLm6L/bbbdd1NHzzDNPnO+hA3784x/HtkO+XvziF0fMaFM2t2D+xFyIeZifQ5HPNuOD4U5bHcR7S4prXfvwYx3vUPQntgZi7veXv/wlzj8ZE5hb5KQJnulzbds43pmY16M/2MoyFcZbxl3k8ssvDzNmzIjHbcrOWLraaqsF3rHQtbz78I7F3J3tu3PSZiy055u0cQtb9ctHBvQDyvnlL395RDCvP9uMW4OIk4zZHJp5JkbvKvEksr/+9a/hkEMOGRV0s802i+9GCy+8cJzT4fGZd03ec6va56hIZl7I6WfCWd9s886Wiz93rWm9l74T+Hrv17g5iDg9NmuttVZ8L+e9HZ3Aezg6/8orrwy33367DxqP3/a2t3W2D0Vv8e6EzqK9MP9AZ9HOTj311FFrNETAOyr6Dbn22mvj2BhPWvwrJXDxjo2eos8yR92k4byPrLXRy+kcjXcQhLks9xAjcA26fmNi7l9Vv7MgJWOJPZv7ZW2DsR59wXwfD9u8I7A2wbsUbaWp2HsdW7zy/gaWzFV4/+IdmD7HmhrCB2isiSy99NIxDe5T53ffffeI5NI2wc3S9ZgqbE2nDWK+0XYtIdcGRwBScdL0XYrxgb6CXHHFFaPe6bhe9U7Nut7rXve6WHeEoW2gV6gz1l1szdd0yAorrNDx3M7aDesCrB2xjmbCeyke3skTZWC8Qm/xLk7cXobdF33aOhYCQkAICIHeEBCBqyF+DLASISAEhIAQEAJCYPZDwBujIWtBxkmFRQtIHXvttVdc+EvvX3XVVeHCCy8ccZkFVOK2hYARN2ee8OINscQvGA8qLyzm4JEIY1lOMDIcd9xxHWMmYWyRhAUHSDe2fQX3MFJhbEJ4Fg9JqUAWgXiDPP744+Gggw5Kg3TObXG+c+G5AyayBx54YDwrKQMPtn3uiCOOyNYxC25HHXVUOPjgg+MiCm0CA7IZ5p7Lcvxh8YXFXxZkTPyCKMYh8IQUhUCMwKDEV7QIxtJjjjkmHlubIE4WiyGQ5QQS3o9+9KN4C/ISRsNU0q390vt2TjvBEEMbrhIWdCGimWBY2HnnneMp5YMwxGJSKpBVvv71r4+4THp8YW/EQn8TsgqLWhhDwbwJgWuHHXYILMIhTb25sUgKQQbBYAW5EWlD4KK90+6RM844o0Ns9O173333jfer/lFOMzr59l8VnusYLqx/YbA48cQTRwVnS1LwTdsAhgjIPdYW0wdpi+gGv0jNwjLtsuoZdMbxxx8/gsBEvBh0P/jBD2bbFcRH+iptBgIX/cekRJfas1W/beM89NBDo2GH+MDDG6wtDfualnPv1QIdsc8++8Q2bGH9b1pe7h122GHRWIBBiTEC7FKhTr75zW9mCWA+LGWlDVJfxAXR0AveDq2vYiA4//zzO7fpFx/+8IfjOduH0q5NJ3GR9gaBKycQJlKyWFvcidfSa6MDc/lJr0F42nPPPbPtkbAQLb7whS+MeAxj3zrrrDPimp3Q7tEdnqyMQdH0OvUMgQADYSqMJaSFjjNp0zeb6Pxh6fW6vPixkDFk2kxDFnMh2iAY7rfffiPmRYaHjc2EIQ10WZV43Bm70ctcSwViBXUCMToVxh7qBIOJl2G1AV8GmxMYBj4/HNv8hOO2cx6e6TbfYysyyOvWji0vPNdL3ywdD0g3JxBmMCrn5hGEh0DzrW99K+pu2lppm4MAAKknJ7RNjH7MgTDK+q2UfXiMecyjkZLxwXDn+X6+txBfTurahxG9MV5WzVHpr8z9MER7aYqnf6akjTOOMZ4huXkYOuB973tfvE/98Yc0KTvzNuKHXJET5rDM63gXMikZC9u0cUun6tc8w9FmeYfzhFive9qMW4OIk/wbMYzjb3/72x2SA+ep4FEXbCEOeoI74+6HPvSh7LsRcTC/OOecczofXqTxpuceo37pxDQNO29b76XvBL5M/Ro3BxGn4YIXOfs4zq75X3QNet6LefrmAwOIFBBwEdo/8y97p/fvcP55P7/hvb3bhwr+OX9cSuCy9wJIasyTfF58/Ol7Xlu9nM7RuhG4Blm/vkx27NPz/Y77JWOJxZv7hXDFhwNVwhwRvW5G16pwdt3qj/A8y4dpqfBhD2tCRib299HVJ5xwQiQF23WL09oE10vXY6qwHdR8o2QtwcZjymkkQo67SZt3Kb8VLx8+MmdMxa8x2fs7H1d99KMfzb5n8Dxj/5FHHhnJpf75NG7WPK090Q/5gAr9nwpt4bzzzhvxoZGvv37p7zRdnQsBISAEhMBgEDDdj20Nvc+6LesV/HFs501Tn2MmQ/x/TQOPp3AicI2n2lJehYAQEAJCQAj0DwG/MGGx8uUsC+58ccdkyQtGc8gdfBXFIrEJC+BMuEyMMME5C8N8MclkzL765TrGdMKZIXIQecE7zuc///mOgZ48sqiNMZeFTysDi0ksCBpRwy+SsFBgOHB8+umnR9IHcSDEb4SXeGHmPxYd1l577XgKsSjn5cvCYkRnwXmRRRaJl8ADzyR4KzjllFOiV6CSMpSUfe+9945fpdvCGrjwNS9fRWJ0MwKX5Z1f7oMrpClbaAEnFmKsTfgFUf8s4fC4xVd33Qyk/hniJE3yaHXAfasHiFR82WftlzqFjACmkE/qxC+Ks6gN6YpFbgySGKpM/FYmfsHQ7tOHMKIwz6a/mHjjJdfIt83FwYP+xy/GMGt3hGtK4MIIZwZ5b4gjjrbSlMCFcYhyWP3TTqgjZKwSuMAcwxgYgzd9GN1AHeNNiRdGBE9EeCdAaAeU0/QGBCO8m2A0ZwHTjCHUvW3Rw3PEyXNWn1bPtF/6qRdbELVrJbrUnq36bRunX9TFowYENS/UO4YiygdRwMirqQ5C56PXwN63b0/AIF5blPdpQPpFN6InrZ2lhhof3h97veUXvSFReI8W9FfSNvHlxvD1h5mejHLjFIvT6BeMX74+fT8gzra480wuvTodyHPdJG2P4AjZky+oMfJbO4WYAIEF8WMa7ZcxnT7DOEr7t2fwxIX3B8QvqMcLM//xLN53SAsdbs/5NtC2bzbR+cPS63V58WMhBC6Mg37LW09GNswg12L0RHJGWQtnvznc6ZfoK75iz5HoWDhDbzEXsTrhy3lIkibDbAO+DGaorJufpPqGftKP+R5k9H7NT8Ay7X9txgOri/T33e9+d/R+xHXmbfRn5gyQfYzUxVyceS79vaTN8YHHbrvtFpMmLnQ5bYq5lpG3uXn00UfHjzSY21JW04noScYA5kDMKdP6ajo+5HRiP95bYsEy/+reB/BG4omN1Ce4YFRmvMJzCcL1b3zjGx3ScRs8MWwjKWZN23ivBC7SJv+mGzjmXYiye4KRtQvKPnny5I5Bl7GCOZBJyVjYto1bWrlf9AuetygP3gi/973vdYJ53WMXKW/duDWIOEmffgTZBiEf6EPI4czFOG8ifg6Enkcvoifot/79A0zo13XiMTL9zDO5vtnrfKVtvfeDwGXl73Xc9Dj1K07i2X777TsfktHnGKvR68zH/BjOB2je87QRuCwv9ov+pE1BdkKY66ZhmbMxDtJn/Dzf4mj62y8CV91ci/yU6OXcHI24/Dhg7xGDql/Sy4lPz/e7krEkF79d4wMPCJ/UNTqGj4x4x2EdhHm7vYM1mY9anOl7HfEyTtKucuRf2jNzBuYvlh46C+KeicVZReCycE3WY6qwzem0XucbpWsJuTZoZcz9tn2XYq3qs5/9bIwK/HMkfD+W2DoYRD7meoh5daTOaEe2LmIfBeF10zynUfcI794IJC/mgXi/f8c73hGv8Y/1JDCn7eE508Svcfn6s/u96m+LR79CQAgIASEwWARE4GqIr720NQyuYEJACAgBISAEhMAEQcAvTLCYgvcavn5DvNGQcxZvIGPwco14Y57fMg2357Y9Cy/ceE2wr5t5wcaAZAYl//I9iLx4AgpeYzyBh8UF77HAPKxQNr9IwjnGE7zisBCNYMQyb2We4BFvzvxni0pgSnkNM7uf+7XF0t/+9rfh2GOP7QQpLUPpcyykmeevlEziF24oG3XrPQqAp33BC2YY8hC/IMo5Czx8SWnP+oUXvyiZtgnIcH4xGmyN+IZHBwyDJiw0Ey8LgOahye5V/bIwhDEKgTCCBxLKaeK/HPRtNyVwUYd8uciiIcJCM2EQCD9mDGZbG0giCMZW8DQyIIQgjG62+NWUwOXrAAIgBLlS8W2IrYpwW2/Cwi5GJRbj8EJkZDpewKz9EHasErjMWwJ5ZOsG7znIG9g97iw6sp0EAhZg4oXFTha4EdoR+g/Be6FtS4phBG921q7Ql7QP8ER8nyvVpTGiin8lcfp+kVvU9e2YL/Jtiybff/11skb7huhlC7joTFvENf1JOBZgic+2g6Fu6M/0bcQWkONJxT+2YrNtX/hqF29+CNsHsh2gCYYwviI2gZjMgrE3VvkyUYeMC76PeU9k3nhWgjv5SNNrowOtHOmvNwh7fUQ4CB+GgRlLWNRH99NGwYj2C0HEBK9cGMoQCLP0f8Z8r9e5hy7GC43pOO/1xWNc0jeJv0rn+/Y7DL3eLS9+LDQCl8fBjBzEYeLJNlWeMSwsvynu1DHzCxuP/DhOeK/bWRfCgAOBlfmOeRgddhvwZfBzgm7zEz9e9XO+V5WX0r5ZOh5QV1VibZ/+iZdBq2vC+/ms6b+SNud1G3M7IxWRhm/Xvr58X/bbM/OMx6/N+OCfQwf3472F/FSJx48w6fsAbZJ+gn5Ej6EfIciYsAUxW50h3itqCZ6lbbwfBC7yn5bdj/3MDSBm2fsORHcMu0Z4Z1xgflg6FrZt4+S3m0AIxcDsjf+E9/2d86bjFmEHESdzEN51jMRAOghtn/dyyBV4S6EP0f5SYQuwN7/5zfEy+PNeY3XERf8uz5zV3oHSePy5x8j397Rv9mO+0rbe+0Xg6se46XECv37ESTwQ7ehX6Hv6nK2xcM9/mOD1DfdsnYFjnmU+wTsHbckTtJjHMQ/kuglzZebMSJV3Hgvb7bdfBC5Lw9pH+q5fqpf9WGZzNNLy40AVgatf9WtlS399e/L9rmQsSeP253hpnDyTgIvgEZ20THjXR0fQ/nLvgxYu/fXvdbxb0IbtXcDrIJ6D4I23QRM/Z+W90fScxel1eOl6TBW2qU7rx3yjdC0h1wYNo9yvn381Xefw5GrmMnxcY8L7gRGx7f2A8cneE9L3F94jeF+3jxgh31H3iLVZdAzzEy/e6zcf0LHOasJ4xnbqzLd8+/P1R9hB90XLj36FgBAQAkKgdwRE4GqIoQhcDYFSMCEgBISAEBACEwwBvzDBNjFsfeWFr6GMWOAN7IRh4ZvFasR76sCgzgIPL+UsurCo5gXvD7ycIxB5WCxA+p0XXubtC2uMWd4jTkxw5j/ySX4RM1Zz7BdJ+BoZEooXb0T0ZSCM3zqGL7Ytfv987tgWVj2Bq7QMpc+RL182Tybhnl9E42tdtgnzArmChT0WbVgcNiKAXxBl4Q1jhDcw+oUXvyjp20S6oEe6G2ywQXjTm94Us5DmtWpR1+c3PZ4yZUr82p3rGDo9YYlrfK1u3oX8YphfMEwJTDzHQidtEfGLzCyE2TycvoYhxov/CtETiXyY9NgvfGE8g7BQKt5Y2DSOdLF3rBK4vFHVk/GsnHyJjic3DCPmhWannXaKxkd0GyQHI3TaM9zfZJNN4inP8FW8b9voGBY6fdsnsPea5ttxqS6NGaj4Vxon+WahFrGyWRLeuxHxo/f8l7y5PsGz3vgLCQrCIWKL8hzj5eOmm27isCMerybbuOCVwPS/J3YYAZT6tHEOEiXGd87ZXpFf743A6ySfZ8ucL1M/6tKn11YHWp78r9dF6Gi2ZbUFdQsHAQSCHUK9o2Nti1VPSrPw/Po6MaOXb/tgzFhvCzT2LMZk89BjW3uV9E3iq9L5w9br3fLix0LDifBmkAUn8Pd1QpvEeE99Mf4QppukuFOH3sDrjWWpgZd4jQRMOmZY8UbXYbQBXwY/J6ianxB+UPO9qryU9E0fV9vxoFud21wdgxb158cYvK7RPxE8vvGxAtK2zTHmmSHO9HyM6Ll/EA2ZAzLumUcjb0D0BK5exgePe7/eW3w50uO69wFPsMyNV8Rn8zLfv9vi2Usb7weBK/cu5D1vQM7FY60Xv33ZhRdeGK666qr4TlTyjljSxn1e0mO2sKLuED9v9X2U+mo6bhHPIOIkXojVeKKy/se1VMgrH2CxDbQRIwjj3zNy/ZYwfhw2cgrXq8Rj5PWz75v9mK+Qftt67weBCyz7MW56nPoVJ5jwXsWcACKVzZu5bmJzBt7/6KMmts7Aea6/euKO3wqd8P4DFZsnc72t+LlESs7oFpe9F3iyDuGr5n2lerlqjubHAesjg6rfKhx8er7ftR1LquK365BrWMtivcIIO3aPX9u+kjZtc0R/P3ds9cc9Pxfg3H8wyhyG+InbxHtD5kNM3uMQi9O3idL1mCpsvU7r13yjZC2B8ubaINerpORdypPL8D5LXZn4jxhtDdDXXe59AvI6cSJ82GhjUxWBi22ozRNgbgwhHv8REvqPd3Fff/3UtaQnEQJCQAgIgcEiYOuDrK0zv2X9FXsSfxzbedNcaAvFpkgpnBAQAkJACAgBITAuEPALE34B2zJvBh5PxrF7eFpiEQuxbbWYZGH0RnCNbkSteMH9s0V/v/jT77x4gpk3pLtsxEMjmPi8+EWSKoKAXzDjmC+hEW8cNW8H8UbNP1tY9QSu0jKUPkcWqwyk3PMELl9m7pnwlT1xICz4shjnF0RzdeEXXvyipG8TuQVjPLkZAdATM0i7alGXe22ENs32kHzVvO2223YIV1UELlvUStOwvuSJWGaY8KSuJs+lYfw5eJiHOxZebbHMh2l63IbART2zOEf9ebH+xTUjaPj7/hjjrxkbqkg/PjzHpcYaT/4jHkg711xzTVwI9MZv7nUTPKRBEILsxSKleUkwkpPvi75v+zg9wcj6Ry+61Mftj3uJ0y+q+q/v6btGnPBkVjyz7bHHHjF5PF6hC1MBO9oHwhYJ6BfEFuVz4w73PSmgqQEo1//MyAWhAS9SCG0AA6j3UOP1uNdJbCXJ2OcF4hMEHMR0Ui+4+/Ta6kCfLzv27TH9WtrCQNTDEI0XhhtuuCGSvBjvGSPR6bn+sclM4iJGCcSMHF6vV5GZbRGf5zgmzdK+2UbnD1KvU5aqvPix0BO43va2t4U111yTR4O/jnfD97znPfG6HxvjhYp/Hnf0f2qA22KLLeIWTDwOoYf0vEASsy3xTGcz1xtmG/Bl8OWump/4dm061JfJjm08ajPfq8pLSd/0+WwzHlj+q349AQNiGKRXtoaj31VJ2zbnSZr0U3Qf8510zPfpeV3tjba9jA8e9368t/j85o7r3geMIAMm6LCc0IfpywieRTBMtsXTt522bbwfBK7cu5DNYavKDmHYtpPHiyYL86XviCVtPFcX/prNNfwWvr6/txm3LN5BxGlxM7+AJM78hLmvzTftPr/UBfMd6/tWR93eMz74wQ8GiM6Itc94UvHPY+T1s++b/ZivkHzbei99J/Bl6te4OYg4K6okvndD9GMuNvk570lVBK6qtuA/GoKMCckL8d65/DtsVV66XR8WgatUL1fN0fw4kCNw9avNdMPOtyff79qOJd3SqLrH+zntC0IO4zfGVD+PqnrOrptezD3jPTvliMJ4vjYS0DnnnBOuv/76GK3FWUXgarMeU4Wt12mDnG/UrSVQ4FwbNHxzvyXvUozZzJOpX+aSfstKvwYIwZXx3H8ISx5YC2V9jA8gzSCfy5u9+6XtwesH1kX5gCoVv2Uo2zifdtppIwhcw+iLaZ50LgSEgBAQAuUI2HghAlcNhvblf00w3RYCQkAICAEhIAQmGAJ+YcIWpHwRzdiVW7DLEbi8637i4cU8JywMmJhL7X7nZdNNNw077rhjTAZPUSzk5MQbK+3rZL9IAiEGYkIq22+/fcAQipjRn2P70p5FdOLmt4nkCFylZWBxv7TsVQZSymCLN+mCiy+fJ/0YicUviF566aWBbTW8NFk4gxTht/3geb9wlBpDqwzoPt2qYwwkm2++efzaPWcg4bkqAtcFF1wQpk2bNipq2hb5tb7kcfYL5emDRsiy59L76bnvR/ZlYhqm6bmvSxaLbctL/zwLZffcc0/HWOTvcWwLrBwbGYDjnPhFXE/oyYW1a6XGGuoV70NGNrT4+MW4AUkJcgPl84Ihg69DV1999eg5CCJITqzte+IT3ifwQpGKJ/iYYbYXXZrGb+e9xAlehx9+eFzUhawHkQfBWGTEnSuuuCJcdtll8Xq6PWHdWEDfNuKTtZl08ThGPPOfx7QpgcsbKBlzFl544Y6HQAzKfNGLd0i2cSR9I+KSb8JTZsT3r9yY6duw6aRecPfptdWBMcPJP19f6bZlSdDOqc0DfB11bj534OcDbF1h+pdfBK889IlUvJcEPDZC2ivtm5ZmlXFyGHrdyleVFz8WeqIWJGEzlFgbJC5f/+m2dZZW+uvH09zYstlmmwW+pEfYOpaxzIufE5nOHnYb8GXwhko/bpquJO+lcyVIDnXzvaq8+Lpp2je97mozHvj6yR1DQGVrbz+3JhweT/GKgDENMqaXtm2OPo7eBw8v6EjaLMRN5sJ+nlZF4OplfPC453SwtdXcnMnrKfvwxJcld1zXPowIzLN14xxh7H2kLZ69tPF+ELjSd6Eqb8qUsUp6GQtL2nhVPuy6bW/ux3nf39uMW4OM0+JOf/GSs/7668f5KJ72TPC4SN/AuyUEKMRva2/h7NcTJNLt6C2M//UYef3s+2ZTnejjzR23rffSdwJfpn6Nm4OI0zCC0En7RY9AushJFYELUoStOaTPMc/nPcd/QOG3Pc1tH5/G0e3cEzSazt+Jz94LPFmH61VzrVK9XDVH8+OAjTuDrF/KlopPz/e7tmNJGm/uHG9/bL06eSYZEHJROrfgGfRmWw9cufc6r6fswxefJ6+f2hC4mq7HkFYVtl6nWb37vJXON9quJZBmrg36vKTHpe9S/h2A92PmkP4Du1Sv8LEWpL5U6KuMO8wLGUu9VBG4zDM2YavmU9yz9mjjmq+/fulv0pEIASEgBITA4BEQgashxiJwNQRKwYSAEBACQkAITDAE/MKEGep8EW1hwhaD/b2cIcR/FeXDdju2BZF+52WXXXYJG264YUy62xfFPt0cgcvyl5YB4wUL4ywimOcZbxDLLSCkcfhzW0w1oz/3Ssuw0UYbFZe9ykBKfozAlVuA4z7Cgt96660Xj0844YS4pUfVgmgMNPOfX3jxi5K+bnLts98ELhbBqe/cNiUYQvkqxIwkVQSuM888c5ShlHKmBC7vWYVtT8AqJ7aYlTNG5sL7bfyaGGKIAyIiC6QIRnHbStUTuM4+++xRW0rGB2r+eU91thBY9YjHJPcVbu65JsYatvWcd955IwnHiEfEZQuoLDyyQJ2Tiy++OFx55ZXxFl8f77PPPvG5NCx9ArF4jMDlDdVVbSNH4OpFl6Z5s/Ne42TLXIxGiH0Za+2ThVYMdoaD9zhi6Xf79TrFDDX0N77yTcWTIJoagDBy7rrrrjGqs846K3rVg8xkRir7gt3Ord2mRMI6nZQjcPWCe1163XRgihvnfky55JJLAjqiTpp48PA6PEfgwvvKiSeeOCqpHIGLQG37Js9UGfKGqdfJB1KVl25jId4HMZLQl8zTmRlTff94NoXq/74ucrh7AldOr3vjjY27w24Dvgx+TlA1P/Htut/zvaq8lPTN0vGgurb//w5e84gfLz1m3Pr/uyHOX/BKYjqae23bHOMtW6qSFsbBVNCfRx11VDT4ca+KwNXL+FCHe9v3lrQM6Xk3oyk40zfaiG0lyDNt8OyljdcRuPD+h0c2xG8p3a3s3iOY92AVI6n418tYSJQlbbwiK/EyxAH6APVo5fb9Pac/ebBq3OJeP+PkPdM+MoBsmvN+SZoIW1m9853v7PR93mV4J4W8jphnzHiS/PPbjjHfZd7bTTxGXj/X9c228xXLQ5t6L30n8GXK1XvJuDmIOMGkSn8yd8ADjnlhTokWts7QbX3Cx20fAflt0qu8b1td1f0Og8DVi16umqPldOGg6rcKQ5+e73eEbzOWVMVv1yELQ/JP5xG0L9Yi6MeM/5y3JXDl3us8gStHrC4lcFW9c6frMZS7Cts6nVYy3yhZSyCPuTbI9W5S8i7lx2m2J0QP+I9W/ccnljYkz6222iqutdg1/5u21yoCl/e46J+vOjaSta+/funvqjR1XQgIASEgBPqLgAhcDfEUgashUAomBISAEBACQmCCIdDvhYmll166s5iDERdDeTeBFGPbPPQ7L36bIPviPZcXSAcYvBAWoliQarpIYuQFnuXr9KlTp3a24sJYzSJCU7GFVU/gKi0D3qNYbEHalr3KQEpcRuCq2i6FML4eMWLTDqoWRAmP+IUXv8jj4zJD8rNPPPu/mzGgyoDun0+P/cI1hhI81LAVEcYpjJ6eoNcrgYttyoyc0u1raNsGoimByxvieBliS8s6MbIK4fyCZz8IXHiVATckt11XvPHcP2/ctkU5fz933MRYYwus1CmGiJwsssgise+ussoqMb9+4RoMwdKM3TwPaRPX/RjFaOP0CRa8MfIgRuDyZKOqr4ExTmEQRMyrTC+6NEaU+ddrnGBDm0DoG7QVFsLBii90bWsk7ptnC475+jb1/sJ1LyzqQ5ZCBkHgYhEbnUBe0csYuGiXRhT0C9ZsG/OhD30ohr3uuusCJDGTOp2UI3D1gntdet10oOXZ/7INiZE1mxI8bfvJbh64MEjgQQu59dZbA9uleb2eW1AnbDdDOPeRJn2TcFU6f5h6nXwgVXnpNhZ6/YeuYBtq62+2TcizsXf/X4d7iSF62G3Al8HPCarmJ6VzpSbzvaq8lPTN0vGge42PvIuuw0MkcwFINhhHTTyWXOulzTFusT0e2wcbYYA4/VylisDVy/hQh7uN900/PCHP3aTufYBxDxI2Xhpty7Fu8c2YMSPrmbcOz17aeB2By3tmNCITZehWdj50YO6IGGk3nnT518tY6KNt08b9c7ljyLJ4srIPcXx/Lx23+hWnn1M2Ga95T5o802MOcvrppwfeJ3mHQMxTSTxJ/uFVFt2E8AEH89pu4jHyOqWub7adr6R5aFLvpe8Evky5ei8ZNwcRp5+rMn5Rx7yn4rmIeTRi3qdKCFy+j/Ieduyxx3a876YfNKT10+R8GAQu8lGql6vmaDldOIj67YahT8/3u/SZurEkDe/P+YgAnWHvwHjeRh/wzmceqU2/icAV4naDfLjVZr5RspZAHeXaoK+7uuOm71LUPR+PQNKzj0fMGzx1bh+Y5NJjvslHcRCK0SW0WZPzzjsvXHvttfG0isDl11VZF2WduJs88sgjcc7p+0a/9He3dHVPCAgBISAE+oeACFwNsRSBqyFQCiYEhIAQEAJCYIIhULfY2tYQwss+C4cIL9V4vskJBCMWEnCvDcEI6XdePCmC7dC+9a1vjcqKX6Tw5I6miyT+q2W2w8HzFF9M+7hGJVpxIUfgKi1D6XNkrcpAyj0jcHF80EEHxa3mOPZihCNP8qpaELXn/MKLX5SsaxPdjAFVBnRLM/dr7Z0FKtouXzJ78VvY9ErgIl7zapJuCWFpgguGc9qpN4ra/dyv//qf+6eeemokBeXCcs2T0tKFuX4QuHbfffcOqTEl+aR5sgU9rjfxAkA4PKLRLpEc6YuFVeoV8f1ym222iV8sY3hMt1dFj7FYaZ7Y2GYMUgqLrgh1Afku3R7Vb9NnBK5lllkmeu3iOSNncezFf9lqYXrRpT5uf9yPOI3IwaIudbTtttvGJCDsgJGJNzJB3oLslQrt+o1vfGP0tkTbsIXdQRC4SBsiHvoNI9ekSZPi4rRtY+Z1EG1iiSWWiNmlvBjcTep0Uo7A1Qvudel104GWZ/8L2YI4kSqDrnkjIwxtfq+99oq4oR/w6kH7T8UTMn74wx+GH/zgB8UErrZ907YArNL5w9brYFOVl25joW87EAshFKy44ooRavoE29Q1Ed+W+2XIsL4zrDbgy+DnBFXzk17mPHXzvaq8lPTN0vGgW70vueSSAa+rCP3OiLD2DCQrtoRF0nlE0zYHcQIPUAjGXMgCXhiHGTNtS2EMfBCaqghcvYwPdbhbf29jUPVlSY/r2ocZY+kbkFjTeQHxYczkD8F7BWTYtnhiDDWid9t3Gk/gYuvoVIf7uV5TAhdlsTlsFbmXbWsZ4xHiZfv0knfEXtp4TLzLP//RA3ob4iy/SE5/cr2OeNyvOCFi4lULIV+HHnpoPK765+v5+OOPD3izsTrKeb6xeLzRnPYM8aeb9FMndkunpN5L3wl8mXL1PlYIXP49I+eB1uv0EgIX9WFzcDwqnn/++SP6MP24FxkWgatEL9NHquZouXFgEG2mG7Y+PZsXlY7NVel4Mq//oNCHtzGWMQ+d00SsTeX00OzkgctvRdhmLQGMc22wG/al71LE6edZxx13XNhzzz1jUuk6DvNL3lOY70HQSuUd73hHWGONNeJlvz2mrfekbch/cGNjWBonH0Di7Qu58cYb47us7xv90t9pujoXAkJACAiBwSAgAldDXJnkS4SAEBACQkAICIHZDwH/gp7zcGSLNG0MIfYMaLIIjjHci/fU4RcX+50XiBvmHQaDCp6AIHB4weiFBwLEkz+aLpJgmOcrNYgIEBps+zS+VDz55JN9UrXHOQJXaRlKnyOTVQZS7kGUsS0EjWjCdRO/BR7GZhbskKoFUXvOL7zYoiT36tpEN/JClQHd0sz92lfLOUMcdYzhi61DkX4QuGyBmfhSAgzX/FfxqeGV+1XijaYswLOlFNs0psIiGB6prN36xTXCeqNebqutNL7cue/v4Eq/8EQfe8Z7JuFaFUHQwvtfM1DlPI75MngCl3kdQzdgdCVvXjxZD5Ip+sG8d/EVsnk1sGdYlKV/sJiOGIHLt23qAo9/Xg9BEoMcYc/5flWqSy1Pud9e44QIwIItgtGWPsjCLQZ7L94LAuXlfoqx3xbj5z//eaCNId0W+rnv20rOgEWYnHiSkd33BC3batPu5cpVp5O88cwbP0pxr0uvmw60cvhfthMkL+gz6sNve0k48g8O3Lfy+wV4X08WL6QNdBnkZYS+QR/xbT+3oE7YnCG8bd80AmaVzh+2XqdcVXmpGws9UQrdxBzDPMMQbxOpw73EED3sNuDL4OcEVfOTXuY8dfO9qryU9E0fV9vxoKruMY5RPwgk2JwRzfRPjmjTpM3Rx+lH6IUqIoknFtjW434uwtjP3BjpZXyow93K2ua9JWaq4l9d+/DlhqB06aWXjoiJPsy7CHVPn2aLUqQtnniiKH2n2W233QKkOeS0006L3kPjycx/zGmZ21K3SBsCl38nyHkd9tsh2bhg9UNaTd8Re23jpNVNTF8zT6adco60GbfS+PsRpx/fGa/BODeXJ23moIzdKYnSz2uOOeaYgD71gk7lgwTq37yt+Pu5Y6/HvH6u65u+PH5+lEuDa6X1XvJO4MuUq/eScXMQcXqvxqwb/OEPfxgBH4RJiJOIX2Ph3NYZum2hSDi2G2fbccTWNnLzxRig5b9hEbhK9DL6uWqOlhsHBlG/3eD06Vm/Kx2bq9LZcccdA++/yC9+8YtRnvQ9WZ42IQLXl+IaStP5Bh8HlawlUB+5Nsj1Kil9lyI+X8+mA7iefhTI1ssQlpGcPuK90IjH/qMhI3DxnF9/9gRCvL+xxpqK3+b96KOPFoErBUjnQkAICIFxhoAIXA0rTASuhkApmBAQAkJACAiBCYZA3WKrLbQ3XZgAng033LDzZTkv/RAZHnjggYgc3j/e/e53B4zICJ5tzHvGIPLiFxaYGLKgjjEU4ctmjF5mtMAAbV8dt1kk2WeffQJeFbxgZLGtIf31bse2sJouapeWofQ5v9hCvUOmgDzEwqY31lAWtvTgD6MgC+2kacYDvtpl6zSkakE03pz5L7coyb26NtHNGGAkEBYYqXfqgzJ0E1vsIgxEBcghGFmpXzxJQXgywQsFW40g3hOb34LQwvJLmyC/nojlPWOQNwxH06dPj4+xiMVCqrVP/5yPt+rYjLJ2n3gxSkDSYsvQ1VZbLWywwQaRIEAYyBoYXT25yJOfSglcxO3bDfXBtnS33HJL3L4O0h95oT+a2LZ2dl736w1UxI2HCwzEGFzMaEkcnsDljQh4KcAwZu0D74BsoYdBDCH/jz/+eNySw+rDCHcYZ8n7m9/85o5e4xlIc1aXeFCw8tGnTjnllOjViW0G+IrUb3HlCVylupT0q6TXODH20k68+L7gr3tPThj92dIEXYGwHRQELvCkTVCH5vHO+m7uS22eLSVw0e4hLJnQt23bP655ncm5X2zmHKnTSVUErlLc69LrpgOfzfHo/7RVvEUijIuMPRCuaIcspBtJlX7BF9CpV7/LLrssXHHFFfF50sdQhqcMxBsGvV7PGUQJnyNwte2b5nHI2k2q84et1ylXVV7qxkK8k7IVs5ccKcTfT4/rcC8xRA+7DfgymKGScnabn/j+28/5XlVeSvtm6XiQ1rOde1ImJB/6LLoLgRjMGGPeCjD221zTnm/a5sy7Ks9dfPHF0QOjxYF3KebTzP+M+Mk9PzeiHhk3bQwoHR/qcC95b7Fy5H7r3gfYPpL5lo1lzBvtvYYPHphLQJJBTKdyXIJnaRv3HqF4L/vGN74R58Rc50MBI9+SrzYELr+tMuMpno4pI1gwZ2aOg/h3yJKxsLSN+7rrtoW3EdMpA96oIHYjbcat+ID71684/XhI9Mxxr7/++oBxG6FtrbPOOrGP2/vXfffdFz1vcd+POZQP/UBfRHgWcr19xMHcGZzqpN86sSq90noveSfwZcrVe8m4OYg4vQ7gQym2baV/UZe8M9q7Bpjy3sLHMCam+/08ze75X3QaHhW9pGQK34/xUmt9xj+TO/btmTh5N+gmbA8JadHmVKnHaruezvtK9bLvL74/eF1iBOVB1G83LHx6fl5UMpZUpeNJk4zlvLexfsIYwXgOKdveg8EcIg6/dWL1lHuvY34J2RfxY6TF6T/2Oeecc6L+457F6duEn3O0WY+pwrbf8w2wA1PDsM1aQq4NGka5X9/XwLVuncPepSwum0vZOfNLI6DbNcb4PfbYI57yPORZ9A5CvZJnyGAI5HbeZxCPK1vG8/GBbZfoid/0/zPOOCO2MdZbmGeyZoT4eYWvv37p75iI/gkBISAEhMDAERCBqyHEInA1BErBhIAQEAJCQAhMMAT8C7T/AsqKaS/v/iXZ7nlDfrrg4r8QJTyLxggv3ybpAuIg8sKiNAtbRhgjbfLCwoktdHMt3eKLBQcW8RBbqIsnmX8YrlgkMWnrLcOes4XVlMBVWobS58iPLYpZ3ozI4Ik4do9fFu9sMYpz782M86oFUe4hfuHFL0rWtYlu5AX/LGn4BT7Oc5IaMm1R0soGAQiDqJ2zWAUmpQuG5AHjyeKLL97JTpqmYduWwAXZjLh92+8kkhwQ97HHHtshWtrtfhG4yAtGMd//LY30lwVjjC8s8jYV/8V57hnD0BO4Um8FhCFt8mjesIjL2j7He++9d1h22WU5jIIuQY9Ye6CNsTUfQnwYHVgwpQ7QQ2YkiwHcP+IxbDyBiyAlutRFnT3sNU5PiCEBT371CdI/KTf92yTtQ1xPyQCmf3IL/YQvJXDxLEYm6xN4hzRPG9zzHgQ5zy3+e72SGzOrCFzEV4J7XXrddCBp5oT6oA690R7yoh8TMfJjwOMX2WGHHSIh0uKjffOMtVuuU7cQVSGDIV6v5xbUCZMjcJX0TeLyWHFuOn9W6PWqvDQZC2mjplPAGd1ppBfKVSd1uJcYoklzmG3Al8HPCciH6QeOEdPRpXOeuvleVV58Hed0QVXf7GU8eLbEo/8bWcTuMJbRdkjLtyXaFgRpL5SvSZvzhl2eZ9yiz4O7pcF1PFaaVzxvmOUeYuT+0vGhDvfS95Znczf6f1374AnvLZVzsMcYCbYm9GGIXmCGlOBZ2sbBGqOo19eWL379HKQNgYtnvQ7nnLIj1iY45/0GUpFJyVhY0sb9HJY2SdvMCeQX6gbBmLzTTjvF4zbjVnzA/etXnIzLzKPoS15SnO0e76CM79SpifdYwjXGbp737SHnWdaeT3/7rRPT+P15Sb2XvBP4MuXqvWTcHEScePChPq1/gZXvv9Sr1z3oG8bMRx55pEPeTddfPN52nL7vp553/LwKgg9zvybiSSVNwpsXKBv3bV5nz/rxgGv+folerpqj5caBQdSvlSv369Pz86KSsSQXP9fQCXhM8u+rtCf/Xsx7gd2n7UF0szG/Kl6rv9x7nZ8npOuJxDeRCFyUp3QtIdcGia9KSt+lLL5U91btLuB1BfqH9kE7or2asM7EGGvjkl83szC23rrSSisFCP6m44gTPWbv7oTnGl5Mjcjs+0a/9LflS79CQAgIASEwWARE4GqIrwhcDYFSMCEgBISAEBACEwyBPffcM/CijOQMUGYIYVJli9sGgSdw4WkGjzNevJcPf52XbrZQYzHQPN5wf1B5wUjNF/CLLbaYz0Y8Ji8s1qcLT/4LVzy21BlPbasMIvXeSUYl2OVCFYGLR0rK0MtzW265Zdhmm206Bv177703HHXUUZGshEcBjIOQU1g0tAUWKxrGVL7wti/puM4XwXidQMx4F0+e+8fCoLlJ94uSdW2iykBKtHzxh7c3W2QkzxCa6iRdsLLwDz/8cFz8ZisaiB4IC1F48GG7CgwGSLrIHS/O/JfzwMU98AMbiICp8EUiRqCVV155hOeuNFzVObjSD9daa61R9cQzVX3R4gM/+9KxFw9cxMdWgSy0492qSvDUB5GMhb624vusPUv58FSArqI9sABoWxcQBsx5zi9M27P84q0MjxLWlmlLfG2cloF0brvttuhdBDKTGdq8JzHS4MtR6tKIMjwHCZAvcM0TlN+a0/LSVpfac91+e4kT3bD11lvH6I3EWJUWuot6Nw9NPhx9kq9x7Ytcu4fxgOeqyLC9ELi8sQfvgZdccoklG/uIfRlN3WAks8VmC1Snk7oRuIijLe516XXTgZbn3C+L3YyLL3vZy0bdxpsCbdI8UloAPPah56z92nV+CcuX15C9Tbxer1r498Z/+oDh3bZvkmY3nT9svV6Vl7qxkHKAA1vPIt6TSrzQ4F8d7t4Qffrppwe+cPfijf3pvHBYbcCXwc8JyGfV/IR7JXMlP3bk5ntVeemlbxJn6XhAOVPBWEZ+aHc5waCGF4Pc9sWEb9rmtt9++4DhPp33EQfzebz3eJ3Kde9pi3Puo3uRkvGhDvde3ltippJ/de3DgnfrG5CFIXMzXnopwbOkjZMm70Bsd+WNoFzHWwbbKuJJEfEErqZlz21PTFyMBxChMaim0nYsLGnjfg6bjvdpfnjHZb6NVyOb47UdtwYVJ2Vn7MVzZq7vkS5jJ+9lzNX9OGx58tto2jX7rSqn3U9/B6ET0zTsvKTeeda3XYur2zuBL1MOj5JxcxBxUhbeOelz6VwM8hJ6hr5u76SE5/0aooNtLWmkZ+5ViZ/n596fPYHLv+tUxWfXGfdYO2gqRuCy9wJP0CKOqrmWxd9WL1fN0Xx7snnCoOrX8p7++vTSeVHJWJLGb+fMP3lXsg+S7Do6hvGb910+LLD2N23atLiWZuFyv1Z/ufc6T+DKrSd6Apefs1qcvk2UrsdUYTuI+UbpWkKuDeaw9tdK3qXsecZB6tkEEjhrYakwbkJKY50nJ7wfsi7on+XdlTU5POWZMAabEZ82uNdee8U5ot23X9oQ8wrWaEx8/fVLf1vc+hUCQkAICIHBImC6H5I3837edZhj8MexnTfNxRwz9wGv9w3aNLYxFE4ErjFUGcqKEBACQkAICIEJhABzDMgueK2BBMHCOIQKFgOHLWyfBXkCgzVfGrP4dc8993QMxr3kx7ZrYHGYBQn7wr6XOHPPlpah5DkWV9jykrr6w8xtdyCxHTzT25QRuCgnCyYsxBKOusVIw+9YEiPMsHhtbt3r8gdeLHrhOYp2woKib7OQHvmykXYEua0fwgIY29mw9R9EJgytJUSmXF4gayy33HJxK0j6JFvVUa7777+/4y0h99wgrs0777xxWx2wZdGWFzW20Ljxxht7Li/lpG4gC9EO8WaXM2T5cvGSSF2zFRB1Sn74mrxbW8ZgQLvCEEo9YQyh7yOWBxa+0S/mjcinyYIlz5IOMnny5A6BtooAOghdOog4fTn9MX0KjKkbMEansPXi7CjDxL0OX8gAjNEY/GiPVW3W4sHwgL7njzaO/uOLebw79FtK+iZ5qNL5s0KvV+WlG1bewInRA3zHkgyzDVSVOzc/8WFL5jz++WEel4wHVfljXGU7Nfozi56Mrcx98OxoY1Tu2TZtjnEbUjjzetoC8wlIn2yLXDXeMr/BcwwLtJAS/ccb5GeijA+M6yuuuGKchzAXYPxnbtNNP5bgaZihX9q+02CUZSxmMRx97z1j5dpG02voa8rOXBMDK2MDc8xuUjIWlrbxbvkYL/cYrxl7MW7TZyBUYBSHIEj/S/tVWi7m37QZ6oj3VLwlM0/u17tGml4/z0vq3ebjbd4J+pnnQcZleoOyoX/R8fQ7E8YAdC41DLmOAABAAElEQVTkXd5n6tqGPWe/ftvV1Eu5hUHf4b0xR5iwMMP67TbXKtHLw8p3P9OxNtFmbK5Kn/kD4/wyyywT9QPva957J7obPUIfY42k7kPHqnRm5+u9riU0xa70XYp+AyGeuQJzGNY6uwntztaxWPO1d0rWSKqEsYx1RUhevn0RnnRpf8xXmCeTB0jKeBCUCAEhIASEwMRBQASuhnXJ5EsiBISAEBACQkAICAEh0B4Btr8zz058DXbccce1j2ScPJESuMZJtpVNITBLEbDtIzGynXzyyaPysuuuu4aZHwrF60cffXQkhI0KpAtCQAhMeAQwWODRE3I0RvX9999/wpd5divgWBsP1OZmtxao8goBISAEuiPgPXF6zzj+KdtWGe+OEFUlQkAITBwE8Oq2xRZbxAKde+654brrrps4hVNJhIAQEAJCYMwgIAJXw6oQgashUAomBISAEBACQkAICIGZCGBcxejFF2aQt/jSFWGbPPOqEy9MsH8icE2wClVxhoIA2w7wZTQCwdO7/fdfuadbPA4lc0pECAiBWY4AHqXwYLDjjjuGTTfdNObHb2U2yzOoDPQNgbEyHqjN9a1KFZEQEAJCYNwjYGMCnoHY3hrBix5bi6fCdp5sg8o6yAEHHBA9faVhdC4EhMD4QgAdgNc+vD0edNBB0Vt4bgvV8VUq5VYICAEhIATGMgIicDWsHRG4GgKlYEJACAgBISAEhIAQmIkAW669613vilvSQORC2Kbm8MMPj8cT9Z8IXBO1ZlWuQSIwderUsPnmm3eSYCtPtr5hy0a2KEDY3urEE0+M2wN0AupACAiB2QKBI488MpbT5hOQOT/5yU+23vZotgBrnBdyrIwHanPjvCEp+0JACAiBPiLAO/58880XP1CzaI844ojwwAMP2Gnnd4011ggbb7xx+O53vzuhP1zrFFgHQmA2QOAtb3lL3DrT3kUo8o9//ONwySWXzAalVxGFgBAQAkJgViAgAldD1EXgagiUggkBISAEhIAQEAJCYCYCRuAyMJ566qm47dEjjzxilybkrwhcE7JaVaghILDHHnuEtddee4RhxJKFvHXSSSeF6dOn2yX9CgEhMBshAJnGDCbPPPNMOOWUU8Ktt946GyEwexV1LIwHanOzV5tTaYWAEBAC3RCwd3wLc80114Tzzz/fTvUrBITABEcAAhdrFSazw8epVlb9CgEhIASEwKxBQASuhriLwNUQKAUTAkJACAgBISAEhMBMBNgODY86c845Z7jvvvsi8QKPGRNdpkyZEhZaaKHw5JNPhl//+tcTvbgqnxDoKwK8c22yySZh0UUXDZMmTYq64ze/+U24++67oweuviamyISAEBg3CGyxxRZh/vnnDw8++GCcTzzxxBPjJu/KaBkCs3o8UJsrqzc9JQSEgBCYiAiwpTvv+Y8++mj0BjxjxoyJWEyVSQgIgQoEllpqqbDOOusEtk1kbeLOO++sCKnLQkAICAEhIAT6g4AIXA1xFIGrIVAKJgSEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhEBjBETgagiVCFwNgVIwISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBBojIAJXQ6hE4GoIlIIJASEgBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEh0BgBEbgaQiUCV0OgFEwICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACAiBxgg0IXBNnjy5cXxzrL/++v9rHHocBRSBaxxVlrIqBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIATGCQJtCFzPe97zakslAlctRAogBISAEBACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBB4FoGmBC7IWy94wQtqYROBqxYiBRACQkAICAEhIASEgBAQAkJACAgBISAEhIAQEAJCQAgIASEgBISAEBACQkAICAEhIASEgBAQAkJACDyLgAhcDVuCtlBsCJSCCQEhIASEgBAQAkJACAgBISAEhIAQEAJ9Q2CllVYK/CEXXXRReOaZZ/oW97Aj2n777cO8884b7rzzznDzzTcPO/nwqle9Kiy00ELh0UcfDVdfffXQ0x90gksvvXQsI+mcddZZ4X//+1+jJPlic+rUqWGxxRYLkyZNCmeffXa47777Gj07kQPtvPPOYZ555gl33HFH+NWvfjXLi0o9ve51rwtzzDFHmDZtWpgxY0ajPJU+1yjyikCzuq9XZEuXxykC6667bpgyZUp45JFHwuWXXz5OS6FsC4HBIcC4sN1224W55por3HPPPWH69OmDS6zHmNuOD/PNN1/YZJNN4tj3u9/9Ltx+++2NcjCR5s+NCvxcoFmtL2dX3OvqaI011girrrpqDHbmmWfWBdd9ISAEhIAQmM0REIGrYQMQgashUAomBISAEBACQkAICAEhIASEgBAQAkJACPQNgfe///1hhRVWiPF9/OMfD//+97/jMcSSJZZYIh7/6U9/Ck8//XTf0hxUREceeWQ0wP32t78Nxx577KCSqYz3sMMOiwSlp556Kuy3336V4cbrjV133TWsv/76Mfsf/ehHw3//+9/aosw999zhkEMOCfyanH766eGmm26y09n219orBK7jjz9+luOw/PLLhw984AMxH+eee2647rrrGuWp9LlGkVcEMuxmVV+vyFbry+NRz7Yu5Dh44NOf/nRYcMEFw5NPPhn233//cZBjZXF2R2Dy5MmBLW0gHT722GMDh+P5z39++MpXvhLTueuuu8Ixxxwz8DRLE2g7Przyla8Me+yxR0yuzZhSNX8uzfd4eW5W68vZFfe69uFx2XfffeuC674QEAJCQAjM5giIwNWwAYjA1RAoBRMCQkAICAEhIASEgBAQAkJACAgBISAE+oaAX/D3BK7Xvva1gT/k5JNPDr/+9a/7luagImprtOt3PkTgGo3o2muvHd7ylrfEG3js+vvf/x7OOOOM6CVtdOjZ64q1VxG42te7YdfG2N4+lcE/MR717OBRGX4Ks5qQMPwSK8XxjMBLX/rScOCBB8Yi4L3x1FNPHXhxROAaDXHV/Hl0yIl1ZVbry9kV97pW5HERgasOLd0XAkJACAgBEbgatgERuBoCpWBCQAgIASEgBISAEBACQkAICAEhIASEQN8QYMuNl7/85TE+ttywLRQ9seCkk04Kt956a9/SHFREs5rUseWWW4ZFFlkk/O1vfwuXXXbZoIo5y+It8cCFVwu8WyAYmcfCVoGzDMAkYWuvInAlwDQ4NewmEoFrvOjZBtUz7oLMakLCuANMGZ6lCMwKAhdbKDIHmHPOOSMBeyx70Ww7PpR64KqaP8/SxjGExGe1vpxdca+rWhG46hDSfSEgBISAEPAIiMDl0ehyLAJXF3B0SwgIASEgBISAEBACQkAICAEhIASEgBAYKgIicA0V7nGRWAmB6x3veEfA2IZ8/vOfj+S2cVHYIWTSjMwicLUH27ATgas9dnpiNAKzmpAwOke6IgSqEZgVBK7q3Iy9O23Hh1IC19gr+XByJH05HJzbpiICV1vEFF4ICAEhMHsjIAJXw/oXgashUAomBISAEBACQkAICAEhIASEgBAQAkJgjCCw3nrrhVVXXTU8+eST4eyzzw6TJ08Or3nNa8Lyyy8fHnvssXDDDTeEX/ziF+Hf//53wOCGh6ZlllkmTJo0Kfz5z3+O3oiuv/76bGnYrmannXaKcS6wwAIxDJ6dIHtceumlo56xvDzyyCPhggsuCCuvvHJYf/31w1JLLRU9Jvzxj38MeExIPSCtueaaYcUVVwxPP/10+P73vx9e/OIXh1122SUstthiYaGFForpPPjgg+Hhhx8OP/jBD2K+LXHub7fddmGJJZYI8847bywn4a699trwy1/+0oJ1frfZZpsY75/+9KdwxRVXhO233z6stdZaYZ555gkHHHBAJ1y3A9ZPXvWqV4UVVlghcEy5KNP06dPDV7/61YCXiCpSB0Y6yETkF3nggQeiZ7Gbb745sL1fTqZMmRLLSP2RTxa6qLsf//jH4Q9/+MOIR6hfws2YMSNcc801I+5xstFGG8X2Qn2CKW3jN7/5TaAOFl988XD//fd3PJ1Rb8T33//+N7at+eefP2y66aax/bzwhS+M9XHnnXeGH/7wh6PSsQu0xw033DC87GUvCzzzl7/8Jdx2220x3X/+858WbNQvHtle8YpXhCWXXDK2C/JIW95qq61im+KBj370ozFvox5+7gL5p55oWy95yUviVeqJdnbJJZfErRTt2VVWWSViQ5ujb9CfwPjiiy8OtJVU3va2t8U2TRsDg9133z32k7/+9a/h61//eho8e/685z0vbLHFFrGv0o7po7/73e9iH/n9738/6pkdd9wx1u29994bpk2bFvGhf1FvtMGf/exnsQ3yIP3/1a9+dVh66aWjRzvu0x7uvvvuEfGakZk+fd5554Vtt902luNFL3pRePTRRwP5OP/88ytxblsGS5x6RV8su+yygThIh/qde+65wwc+8IEY7Nxzzw3XXXedPRJ/S58rzWcvfX1Ext0J+pl6oz+RL/Q0bQzd949//COGRHdusMEG8Ri9cuONN7oY/v8QQiP9im1Bv/vd73ZuNEmjjZ5ti98g2iqFm2uuucLrXve62L6pG7w10udo1/TpKh3aASY5wJMPOg6dzPiB7qbvoxsZ49ABXnrVieBIv1xppZXi2EYfY2tg9MjHPvaxsOCCC8b0999/f59s9njRRReN/ZWbF110UcSG9oAuQa9zzaRt/fFcKdabbbZZ9Ki58MILxzieeOKJOK7Qn1OdT39Hd4I7HjZzHpXQ+fT7hx56KNaxlampDmasRa/xS1/517/+FUm8F154YdQ7Fp//nW+++cLmm28edSH9lDKg53/+85/H9ubDtjkuyUubsYl6fvvb3x6zxLzunnvuCVtvvXVgDkHbor0xZ8A7p+kan3/K+sY3vjHOkRgDqC9wp15oUya0M8ZUxlaEuEjrlltuCcxlmE/wx1yFeueYfNBnTzvttBHbFzP/2njjjWN8zDmfeuqpqBOvvPLKcPvtt1uS8ZfyMTdESIs6QUrnnvHh5/6VzM36OT6UErjA1s+frUymg6nvn/zkJ/G9gHkVeoM5EHM95hHMQb28/vWvj2NT2t8szOqrrx49mv7nP/8J5jGXvkgfRqi3++67z4LHX+ZVfJDBM7QV5hRNpBd9Obnh3JN8MYdC0Am846Tyghe8IOy2227xMvOvq666KrbpHO72LPXC/JOyo0vpS8zD6A+M+Tlpmmf/bJv3A/+cHdOG6fMIffjqq6+2W/HX48N8mPcrL2xPTr9ljo5+TAlcvJ/Stpdbbrk4PjPXQZcw7lVJm74Izvw10TWD0utV5dB1ISAEhIAQqEdABK56jGIIBmyJEBACQkAICAEhIASEgBAQAkJACAgBITB+EPCL5RAg1l133VGZxyDN1lh77bVXJJ2kATBGYLjwgqEaIgWGi5xAGDjuuOOiQcLuW14g/ECgwiiXEwzjP/rRjzq37DkufPzjH48G6E984hOd+/4AIz3EJQRiEIYHMxz5cBxjRDr22GNHGI0/97nPRUM9hkyMshBcTPbdd187rPwFl3322ScaZNJAGP0xjpGflMCFoZpyYqDJCSSEww8/fEReCYdhDONETiArQEK7/PLLO7cPO+ywSEDCCLrffvt1rmO4/vCHP9whxHVuzDygfWDUxnj7+OOPh4MOOijehgSy8847x2PSwJCOQS0VyDc50hJEi6o2QHo8g0ErlXe+850BI2EqlBdCBQYxpI7AhQF+6tSpaTTx/PTTT++QBSAQrLPOOtlwpIkh3JNjCPi1r30thscIBRkDYh2CcbIJERCSBe0IEk1OIJqx3aMXq1sWOjHAYpBPBQIYhjSMt6lQlhNOOCESMO2eEbgwWlL/EFpSoS0dffTRo+qqpAzEDYGFdpXrt7QlSF1ISuAqfa40n6V9PWY+8w9saRtGhk2DYNz+1re+FUkJYLD33nvHIOhavI2kAqkF0g9i/bZNGhitm+jZEvwG0VbRrfR5jMU5AQPac5VxPH2GsY3xAN1cJRAfPBGqF51IeuCdq3/6MwRO9AH6uAmBC4KrkXXQzxBkrE+hA+jrSEn9lWDNGPOhD32oo59TTJkXnHPOOZGoafcgAdl4A1nnxBNPtFud38985jMRM8YM316b6GAIxzvssEMHl06kzx1AOID47gVcISXkdCE6FKJrSiz1z1cdl+Sl7dhE3/jKV74Ss8AYAsGEekmFceoLX/hCbGt2r24+ddddd8U5H6TJI444IosP5JajjjoqtktwBC8I3rRNk7POOqtDBmO+Z2R2u+9/mR8wLzLx5SM/xxxzTLxlc8i2c08eLp2b9Xt8KCVwWdkpC3iiRxDTwRCxIA9Bok+FuvzmN785gsj45S9/OWJSNe5AZLL3DPQnH2wwd7a5LfqLcY66N7F5N+dV/dzC2m8v+rLN3HPPPfeMczjS5R0KUloq3hswRFPep6pwRwczh7Vt4dO4wOWUU04ZRWBqk2eLs+37gT3nf2n/X/rSl6KOpJ186lOf8rfjRyk27+cjAE++4/2J9woEkvkZZ5wxAhfqGgJXTiCsp2Sxkr7IGNhE1wxKr+fKpmtCQAgIASHQHAERuBpiJQJXQ6AUTAgIASEgBISAEBACQkAICAEhIASEwBhBwBsRLEsYVDDi4AHDDLp2D4MNZA2+RPaGawgCGGwQPC+w1Zwt2nOdr/QxqGBss+cwOkMSIk4klxeexaAO2YTnTfxWdv45DFAYsTGq8EteEIzzGBcgcOGRBoIPRhITCFB4ncJYvcgii3TybgZFC+cNSXaNX8rwkY98xF8adYzh/cADD+xgirGQdPHYhPcWLymBy4zQhOE58oqB2LyUcJ3yEQ5DNYKRzL765xkIYhjjeAbyhgnkGog7iBntUgJXWm7Ia5SZduDbiBFBiMuTFThHaFd40WINiWdNPLGOa97wjMGKfJMn6ga8EK5/4xvfGGE8fNe73jXC8EU+MTZT5tSgXkfgwnCE8R5ikhGsyDsCkQCyEIb6tddeO14jP9QL/YN2Tn0bNngewLOEiZEH7Nx+6Xtg3U3S/kW9U7dgivc0SzNtu1a3FrfhincHnkuFdkR9Ug7Djj77yU9+shPUCFx2wdoZeYDEYc+lpJLSMuBV6k1vepMl1/G2ApHD6wcCeAJX6XOl+eylr3cKlxy8+93vDquttlq8Sj3gAQVcIQKQHgL+6FTq7otf/GIHE68vY8CZ/3zbhRALMbZNGmBTp2dL8RtEWz3kkEM6hEf0AvjRPiG7Wf/GOwjElCbiySPoGPQB8UBs88RK6sE8cfWiE6lDW3un76Ir+PV9nnynfa2qLBikMV6nQpx4J4IAWlp/JVgffPDBUdeSH8YJ5gy0c8YqX24IIug7pB8ErhiR+2c6GOIzBAcT5iIQgkiTccgE0ph5esITG1vumjB3IT7mL36MhzDnidMWvuq3JC++f1OnTcYmT3CyvPAs7ZdxgnLkxhfqh3GLe4RnDKT+6Ad4zrG5IMQ19DLkUvAwEjH1DFbMzZgL0C5pnwjxWZocQ57G4xzeT/FAiaD38KKH3mPspY7sGQhgeF5FfPlyBK4Y6Ll/TeaeBC2Zmw1ifBgUgctjwnwVjGn/NrZz7omRJQQuxm50Bm0MwcMSRD0EL76bbLJJPGYOyLsG9V0npfqy7dzTk6WrCPjM+22OZeNB+t5ixDm/ZTftHf3BuIQetHGF8tPuSA9pm2eeKXk/4LmceN0N+c7yBenPvxcxf2ZsN+EDGryMIcyJ8fbmcbFwzEP5aIb3M8YkE9L1hOuSvthE1wxKr1s59CsEhIAQEALlCIjA1RA7e5lqGFzBhIAQEAJCQAgIASEgBISAEBACQkAICIFZjIBfLMdYgDES7xsIX4BDiDFhAZ0vrSGMIN5AyFfnfH2OvPe97w1s24OwRQ2etkww+ngvMvbVNffTvEAqMMMb9yEmmOEUL1oY+hD/nPcg4L9654t3vnw3wUhvhgC8h+FFzASjImmZgerb3/52Z2s5T2QCL57DEIsRq078l/oYN/E0gdccBBLQHnvs0TE6egIXnoPe8IY3xHAYc/BeYQYSDJLk1Qgc3jCM4cS8JniSFhF54703ZBpxwhO4MLDYFinUPUYTawPEj9cU8oF0I3BRJjwEmaEKL0oQGhC2RTHPHxi6MNJhgAUfcAIvk7e+9a1x20rOIWGw7SQCIYy8mRH55JNP7tQ5pEEIW9Z+CF9H4CIM4g1q/hmfHgY18kl+TPDKhWENgeyGEc/qzRO4eBbPA3g8oU3ViW/vePCinCaQpvCqZIZQ6tNIZ1a3hMVgjqEVQzvi+zLnGNJp9ybgCpENIX5rt57ARVzUhZWR8LRNI8d4z3mlZTj00EM7HmGmzdy+ia1WTdgaCy8wJp7AVfpcaT5L+7rlPfdL/6Cf0V7wcmH9iLDeCImHHzwYek8nRpzw8Vp7oM1RT/TptmkQXzc9W4qf5Y34+9FW0enmhSwlaUEuwYuTEUohKJJmN4F8YkRLSA2MJ77vQvxkK0DE6+SUwNVUJ3odDEGLMcD6Ln0eLybWz0oJXJDa8LrlvRqW1F8J1mxh9+Y3vznihSEGPG2M4aLXT0aw4nq/CFw5Hex1hp9vkK4nD0EcwgsR4p/x+od7lJGtAxmfUtIL97uJj7dJXkrHJk9wIj+M53i6tLbG1qzve9/7YlYZA8yDn9cBbMXrPU5COoFMQ7l922ScZ0xEUo+RXp9xH4I5hEJIVSZGFKLuPJmF+5444ucIvnx+3uPbOf246dyzdG42iPFhkAQu6pp5hhEVqVN0ps37jJQE9lYv1JXpXK6b+HEJHUp/RvyclDpAx5Eu+tjmdMyxIPnUSam+LJ17+vEKIhpjggnvGUYKRsfSVhHf5uy9BX3GfSsv83a/bTX9zT6+sPeW0jyXvB9YmdJfP/ey+Qdh2E6TbddN6KvMoU1oH4wXXpd4XGgH9HsIxSY+356cWdoXm+gar3/7qdetTPoVAkJACAiBcgRE4GqInQhcDYFSMCEgBISAEBACQkAICAEhIASEgBAQAmMEAb9YfvPNN4fvfOc7I3LmCRoYKYwMQiD/5bl9MY9BBwMOBggIBhgmUuFreww+CIZy8+jj85ISSAjrveh4g59/zgwhhPdGRU/ggiyFQRhJt9iJF2f+W3/99aOxlXO/xZ8ncKUEEns298s2RBhxwAUjBh4L+PXitzPxBC6wAjOMGRBpMKp6IW4MDIg3EH32s5/tkBK8gc2eZYtLDHEYiL73ve/Fy2aI8gQu82RA+hB0PJmKh3y9VBG4WFwzY62lD6mKtoL458jX8ssvH6/7rQrjhef+mUGFPEFmoR1571u5uvF1QDSejOXjTo+rCFw+PW9I8s9j7MbojbCVGluqIZ7AhYGcNtZEMMxTr0gOU6574yWGr1NmbreDWN1y7PsD556sCbmAPgm2Jt5TAoRMiJmI6Yeqtuk9F+BFjXorLYPvt3jggUyaim/zZmgrfa40n76dte3raXn8uWFN/UCC9QQuyHJGrKCNQYz1BAmvF4jTb3+Idx3Tx23TIK4qPVuKH3H2u6369u0JHaSFrLXWWmHLLbeMx5BMjbASL2T+TZkyJXof4xYGa7bS84KR3cgtntDiCVy5/lulE00HkwZ1xPa+Xnw/8yQZHyY99oQJDOiQ+HybKq2/Eqx9+XJjFXln/DTStXl56ReBK9XBnnybEv7IC6Q/yI6QvG0Ow3a/bGOG5OYvXPfEHfSyJyZwPycleSkdmzzBCZ0OmcQMY5Y3Xw+2bbQnVXvCoj0DLiussEIk9xrZzesnP5/jGU+qqNL16H/wv+222zpjnKXHr23TCJkGUg3iy1dF4MrVnZ/j+LyWzM0GNT4MksCVm4f5uQ3kLsjkSCmBi2d9/2DMgvhuxPEcCZlncuL1SRt9WTr39AQmr+/JG9twsx034ueJufcW7wEzV16/5aARR0vz7OdKOZ2bez+Ihcj8w+udvev5j3YYU/hoAl3Cuw9Cv+QdgnPeJ/iFlGdzYo+Ln79asn5+22tfJM46XTMovW7l0a8QEAJCQAj0hoDNUyGOMy9kXGGezh/H/E2ePDmep966cynPMXMR8P9XQHIhxuk1EbjGacUp20JACAgBISAEhIAQEAJCQAgIASEw2yLgF8uPOeaYgFHLixlj0i+nCcN2FhhSkTvuuCMcf/zxI0hdfnE9BnL/ML7hMYSFfbyHID4vtsjvHolepuzrdW8k8M81IXB5Y0tKZLH0/FfznlxkBC7yzZfg/DYRb9TOEeWIw3t1MQIXi0/mYcoIMLn0bMsqj6c3sGEIo46uvvrqUXXs4zPihCdwkT756EYMMGOpx8qTFUj3+9//vk8qHlv78nGb8Y08g3FO3vOe9wS2OETMQ5oZpMAA8kRKkCMsBmcW8ZBeCVy0ffoA6UHI8+SHmMDMf5vM3PqHLYAQX+9mrPJ4xUA1/7yRFi9LkEdSoV8ZuYntZSD9IVa3vo3EGzP/saYH7kjOYL7ddtt1CC5sH3n99dfHsEb4YXsuI+PFG+6feXWyNlVaBk8SsDp3ycRDSDjkFTECV+lzpfks7esx013+eeIEpJGbbrop/PSnP+1sz5d71PoE9zjGiyLiPRp5jxklaVQRuErxI3/9bqueNEz84IChHeKVLfxzvRdBRy622GLR+8+2227b2fbPG/RLdaL1s276IqdLu5XHE7hyY3Vp/ZVg3aR8H/zgBwPEOcT6fz8IXDlM/RwhR6QgDxDYISfwPF5LPWkK/Q4pIRW/bRnbAJ522mmRLL7iiiumQeN4iw4ryUvp2OQJTp7Y6TPnPeBwzDjtCU6EhaBxzTXXRIJablwkTFMClycH8Vw3IU62lGXctXG+LYGr6dyzdG42qPHB91ebP3bDyu5VzZ9NB+fm/jzr9b6NtVw3PdTWAxfP4jmUMci8CXIN8fOYZ690/99En1g++zH39GTXtNw2BjPvso8NyH0Od+u33E9JpVxD2E4VHcvHJ/Sz0vlyyfvBsznI/8/hae8FEMohoiLohfPPPz9+2GCkcz8H8bjwTsl7ixc8ToIjYu+BpX2RODyBK6drSvU6cUuEgBAQAkJg8AjYe5wIXDVYi8BVA5BuCwEhIASEgBAQAkJACAgBISAEhIAQGGMI+MVyjAe2/Zll04hW3shh93IErk033TTsuOOOMQiEHYg7OfFbgdjX3z4vLND7LZSIwxuGvYHKP9eEwOUNwbl0LL+HH354NCj57T2MwNV2C6RtttkmbL311jFq74nJ0rJfMzxZ+fxWQITBCJQTvi40sS3AqB+IRRhlvRAHntQgFGFM8Tib0c7INh7zHLHH4jUCmTeGe7ICW93hFSsV82Dh25cZfQjbpLzWzowoZB5R0rQ4Z0tDMyT1SuCyvgF+ZlBK0/R9xG8TaQQujPx2nD6bO0+3pKnDx+fN6jaHjycsmlHMp19H4Mp5LbHnvQETsmZpGagvjPNITldxna1b2cIVMaNy6XOl+Szt6zHTXf7Rbtl+yvd1gqOf8CoFGcm2sbVo/FZvZjjlnnmwg3yBLuYXKUnDG/I9IbYUP/IxiLbKFrWQHFJB17FFG/j47arScLlzvIHgXQUDvm25m4arInA11Yme6OK9QabpQG5mK12vS9Mw/twTuPz2phaml/prg7XXPdQD24blxOsg20awHwSunA72cwS28ILgVifmbYZwVXqZe9Z/razMWfBgk4oRZ0ryUjo2eQKXefhJ8+U9/tgYStvff//9IykrDc+cAC9ZeAb0nu18u04JhJ5UQbvGG1NOINGxFTPjLN7rctKWwJWbE/p5UK9zM7ZWLZkL5srmrw2KwJWbL5Cu90xkYy3XjciTEpm4h1Rtofjs3RA9tTGnN6EvMb/0bcfu5X59u2qrL0vnnuQDD7OkjdjWkJ7YBdmK+E1y7y3Wbwlj5EgLX/VbmueS94OqPHDd6yneQRZeeOHOdol8BIJ3NbzP8e7B+G4kcuqX8LxTIR6X3DzPf2zQa1+kbdfpmlK9Hgujf0JACAgBITBwBETgagixCFwNgVIwISAEhIAQEAJCQAgIASEgBISAEBACYwQBv1hu2+H4rJlBAWIXi+lePDnFPHB5bxHmJcM/Y8c+3RyBK5eXnBGN+HxcTQhcn/70p8OCCy4Yjazm/cvy5X+NRGCGVO4ZgavKqOWf98dmrODaWWedFdhyMidmjDHDhPfYkQufu+aNHhi43/SmN8UtCXMEA8p21FFHRQIIcVmZjcDlCWS33357OPHEE3NJdra3qiJwnXnmmaPIJUSUErgwbkNiayMXXnhhuOqqqzrb+eXaqsXnSQlmfLZ7Vb9VWyga2c6XOY3DG8RzBK5uBsY0Ls6957Hc/fSab6dWtznDqidRWF/2cXnyRM4DF9sn4b0gJ2bYxFBHfystA57EbDulnH4gbQhe1CtiRuXS50rzWdrXY6Zr/rG1KG0YLxRGBPGPULd45KDeEb9dl22jiJcoDKZIrv21TaOKwFWKH/kaRFsl3le/+tVhq622it4OOU/Fb6uW3vPnkEXQsxjoU4FQRz1YW60icDXViXgaBEsET08nnHBCmmQ8N0NzCYErRyrupf7IUFOs8aqF8R/xXgrjBffPE4KvvPLKcPHFF4cmBC4bs1PStRFnc32AumV+gzBfoD7rxHuvqwvLfSNI1RG4SvJSOjb58apqvM8RuCgP3pPYKhEiUepBifsIdUbdIZ5o043A5ecz8cHn/lW1T8YZPOxBZkTaErhyY0tu7lk6N0N/s60v0mYuGB/o8m9QBK7cfIFslBK4/HzKiE6+WLQL+pyNb+hT2oCNaT5s7rhUX/Yy9yQfEPN22GGHmCXzjurnm6l3p9x7i/XbVFflysm1XvPc9v2gKh9c99vO066ZZ+AJz96fzOOXnRuxP/Wu5nHJ9cUcgau0L/Ku4AlcOV1Tqte7YaV7QkAICAEh0D8EROBqiKUIXA2BUjAhIASEgBAQAkJACAgBISAEhIAQEAJjBIG6xfK2BK4tttgi4PUFMc9IuaLi5QASAgKpA6NbXV5yRjSe9881IXDts88+YZllluHR6DnJe6CKF5/7Z2QqIz5w2YzBnhjjn6k6ZkstiANIztsJ170xxghcSy+9dGeLSQhAGEa6CcYutj7KCcQMDIcrrLBCx7hJOG/wN+KEEbg85mZwzsVthidPZvIG96ZkBeK2LRsxYrGNTJ2wfR8ehMwDVzfjl98OpVcCl3ma8F6u0rx6QtGtt94a8FCEdCMPpHH4c7yNgCuCx6DU45IPyzHGVwxkiNVtziDbK4Grm3c28/Zkbaq0DN5r30EHHRS3LosFc/+8EdsIXKXPleaztK+7YtQeQpZYffXV43ZKeKGh/kxSEpIRe7iPN5upU6d2vNBByISokZOmaVQRuErxIy+DaKu+jOBFW2E7M3Ss91Lot3Pyz/hjTx5hizgIjP/X3p2FWlX+fxxfafRLiixoABu0jCwIpCDSopkQioaL5oQGo+EiL7roIiL+ENVFV0aZhdEMJiVCYXNYRAkSWIZ1UYoUDVZeBZEN/3/v9e97+LrcwzrP3tvfOfp+QPe09hpea61nrXOez3kehrTcsmVLHTTIIblBA1yEjQk0UDr1FhXrFUNp5fo8Puv0mHvg6hTgGmT/5eX1s8Yuhm+NXqny9+M54SBCI5TnnnuuDnu1CXDFPQz7ifuDKL3qYIIDMQRfBMzje90e83nGvLkW9yrbt2+vr71sQz5/4ztxLS9Zl9Jr0yABrlhvHhlakt786BGRcyHCOHxGoJfGtkECXDmwwb0j90ucf/QeGWG7uH8bVYCr9N5sVNeHfO2L+0e8+5Vu98+96mDmWRrgyj01cd43e9bi2s45kUuve888Hc8HqS9L7z1ZLtdL7kE51uNnhqiTm3UP03dyj/OWc5/7ljZlkHXO82/z80Gevvk8bz/3FAQoOffj3jCfs9zX33nnnbVVc4ja7NI2wFV6LrIN/QJcpfV608fXCiiggAKjETDA1dLVAFdLKCdTQAEFFFBAAQUUUEABBRRQYIII9PtleTR+durVqFMPXDQC0EMDhWFzli9fvsuW0sDB8IT0CJUbNvqtSw4T5Qaq/L02Aa5rrrmm/mtxVuzFF1+s1q9fv8s65gaBb7/9tl5fJioNcBEUIDxE6dZIfeKJJ9bDjDBNbB9GNERSaPBl+Z0Kw3jRaEpIhuAcjSn0hkahVxMaNnOhdxjCHFOmTKnfZqhFgk/RaBdhGz6MBqIczsrzIogXQwjmaUoDXDEcIw2zhKxieLe8TDz5RyF8QKPt/6QemroNu8T70TPHoAGu3LMUf7lPcKJZcgjirbfeqtasWVNP0is80JxHfp0bwQhvEYxrFs6vK6+8sj4GGFqP3iAosW9HEeDKx0teH4IJ7E/W6aeffqp7aivdhtyz1apVq+oAW14Wz/PwTBHgKv1e6XqWnuvNbcmvjzrqqOrss8+u3+IYilBeTEMwk+2kNAM8+Tykp7rTTz+97pkr1718r3QZ3QJcpX6sy7CPVXzmzJlT13EEtJol9wrTaQjR5vRxXaSOok6mx59c8lDCgwa4mG8EZLudZwRvaPznPGvu/7xe+Xm/AFfp/iuxju3rVDfFOueGdOoUgjlcx6j3KZ1CHvQExb6iNI/3XnXwtddeW58nfK9bED2uUxHmzqG+J554oqInw2YhXBJBbu47uBfoV0rWpfTaVBrgYthYQjeEzJvDZnMPw3U3eqtbuXJlxTkxSIArh4Cins+OuZeeUQW4Su/NRnF9YNsnWoArzou8X3ieezRqBrhySJNgPOcvdRql2x8B1B82/ov6ZLz1ZZzT4733jMXnsCUhpcWLF9cfNXuY481OP7fEecvnne5hOYeo7zCJMG/JOmNb8vMB69WrxPpTj0+bNq3+GY97DnrJzXUL9UQMG8t1iz/CiJJd2ga4Ss9FlnnjjTdWXAspnXrgGlW9Xi/Q/xRQQAEFBhYwwNWS0ABXSygnU0ABBRRQQAEFFFBAAQUUUGCCCPT7ZXk0VLcNcNHgQo8VNDAQvGGoLhpOcyGIwNAilNzo2m9dBglw5eFLTjnllOqGG26ol89f/0fvH/Ub//6XhwmKIfr4qDTAlYMsNA7RGN3s+Sv3ShYBLpYZ+4Dn/IU/jR+55F6eorGSYBbBL/bDtm3bKnpBapbcCBoNFxGcyA1f0YsA33/00UfrfZbnFY02vDeMAFder7fffrvusSwvj8YajjEahDjG6KmAx9zQ0qnBLIfymN+gAa4c+li3bl21YsWKvJp1OI7GNYaxo+TGyl7hgZ1m0niRe5zhvCJ4x/GUSx7uMK9X7NtOIQmOTxpWKSVDKPI9tp/l5ZLDBxFgK92GHCjJx1ksj+OCxkAeKdGwX/q90vUc5FyPbWk+zp07t+J4o8TwTM1pop5o9giHB4FZ6gIa1WN4s+awl6XLyAGuXM+W+rFdwz5Wqe+p9ymcezR+58I5GnVkt4Btnj569+nU0I/zPffcMzYE3zACXNFIzzrQix+9+eWSgw/DCnCV7r8S67iusk1Lly6t6EUulzysWjMYEmENGnC4FuVy66231r1A8d54Alx5GLbmEF/MixAkdRuFkCwhZ4YLu/zyy+v3cui7fuPf/3JvgI899lirAFfJupRem3LIYjxDKMZwaFyHua42r0k50BiBuEECXNxX0rsPpdP5TID5zDPPrD+PeyJe5O3LPRWW3ntGncu8296bjeL6wPInSoCrV0j+jDPOGAsOsc75nogAMccO9SeFOpYe3AgHUhh6j3nzs0i/Ulpflt57xvrkcB7XYX5moXBvRYA9l3zMxR+e5LD5J598Uj3//PP5K1Wuz954442KfyXrzExLfj7YaWU6vMh/sBAf54BWruf5vFNvudmlbYCLeZWci3zvxj4BrlHV6yzbooACCigwuIABrpaGBrhaQjmZAgoooIACCiiggAIKKKCAAhNEoN8vy+OX4m0DXGxWbsDllyo0bDGkCIVhv2hcjEYaGlpoYKP0W5fxBrhyzzM01tHwHaGpaHBkuTTYL1u2rG4gIvhEI2z0dtNsLI4GiOb7zKdfufnmm+vtZzoa2WlAIUDGMjGJHqX4PAe4cqMXyyVE9c033zBZPRziLbfcUu2333716+jdghc5ePXqq69W7777bj0N/7EslsmycyNKBCdygCv37kMjGj3YEJA66aSTKnr+ouEtSg7WZP9uvScQxGK/5tABPWTREM8xQkMwQRxCEBR6XGHoFRp/KTlsxO+l2D9xbL333nvVa6+9Voe7Zs2aVTd00TNZlEEDXLkhlnm+/vrr1ZtvvlnPnm2iYS1sNm/evNNwkKUBLmZ+22231fY8J5xHeCCOa4YwJcAVdnhE70Cxb0cV4GJf0TMWPYPRkE+wh+ODwpBAhM14n1K6DXH+MQ9COI8//ngdSmJfEI4gdBIlAly8Lv1e6XqWnuux7s1Hzm/qYvYrlvTwE733cEzTq0/07INLHF8xnzxsbLzXHBqudBn5PG/Ws6V+wz5Wc7iBQA6hH+oqCscOjbj0HknpNsRt/eG//+XrB6FFjjXqRobmvf766+thvGJ6ekBkyD9KthpPnZjrYM4hgnIbN26s50kD82WXXTZW7+W6tJ6gy3/9euDiayX7r8Q6rwuOHN8R4qKup+6I4GFzuMd8bjMcF59TD5x//vkV4c0o4wlw8Z0cTOZce+qpp+p69phjjqnr9rjmvvTSS9XHH39cLyb3LsSQfvTwSb1IiJLr7cknn1xP1+l+qv6gy3/jXZfSa1MOOI0nwJWHJuaazNCsUdfTMyjX7AMPPLDeOnoQ4hzMoUk8OOc5dvke52OvXnHyPeaPP/5YX1uZB8cK5wL3mVHyPUnevmEEuErvzYZ9fWBb83nHNT7On3BoPhIy5B4p33dHkIhpe9XBfN5tCEWOc8LAFP7Y4Mknn6zrRkKPDK8e92d8HgEu3uNnAY5bSg4v5WOf+fHzRL9SWl+W3nvm9SGwFHUD7+fjL0/XyZ37RoLE3JdTCGgRfOec4FgjmBj3doTZ+PmqdJ1Lfj7I69/pee6Nl8+py7nPjpLPW97rFJbOLuMJcJWei/3qGtZzVPU687YooIACCgwmYICrpZ8BrpZQTqaAAgoooIACCiiggAIKKKDABBHo98vykgAXDa00DuRGDH6RT8NDNEyw+c0h4Pqty3gDXDQGRa9CwR1Dr9FYz/JyYxLryPrFezS80gBLQ2yUaCwuCXCx/ny/6RLLZHmx7BzgYtm5xwles66U6G2I582QUO5Rh8/5Do3Y7J9YDu9Hjxg8j0a7HODi/Tw0DK+7ldxYVRpWYN65RxleY0NwhQbYKISWaNxjm6LQeEtvH1H4Ho1f4ZSNBw1wsYxLL720Dgp0Wx7vs36EZQjrRYmATXOfxee9HjmOOL+yBcsgyJP3azO0F/t2VAGuXutM6C+GcmS60m1gmFFCJXk7OS4imJef5wBX6fdK13OQc72bY+6dg2kIXnI8U5+EB69pQP7+++93mk3uGYQPCNTSANwsJcvoVc+W+o3iWCU4QgCUghN1OPVCPo8IkFCnRP3a9InXBBMvueSSeFnPjxexH5rnY/TiNEidSIhpxowZXZcZddswA1yl+6/EOvdOxUZSb7NNUXfzXqceM3NvS0zTLOHCPiGgEqVfHUzolyHQ8j0L65Rfb926tQ4Dxjyb9QzLZrn5ms97hLcJ0LQtJetScm3KAafxBLgIZ3FvEzZsI/UT+y7qZra1GdiI8zwc4vN+oQqGX+N4ifON73POxrHC8vP9AvuAZcUfFDD9MAJczKfk3mwU14cc4GK9+pU4l/J99zACXPSaRU9R3Qr7JvZbBLgIvZ522mn1V6iX6aGWc41CKO/ee+8d+87q1aurtWvX1p/1+q+0viy994x1aYaUcrA/puGxm/uFF15Yh/BjWrwoYcbz6H2L55SSdS75+eD/l9b7/xxgawbucm+CzKVTiDm7jCfAxfxKzsV+dQ3zHVW9zrwtCiiggAKDCRjgaulngKsllJMpoIACCiiggAIKKKCAAgooMEEEbr/99vqX06xOp1+WR4CLX47QsJ3LYYcdVjH0HoXeQOihIgq9K9DrQgxzE+/zSIMEjTDvv/9+frvqty40ehGEoeSAU/5eboBiutyDCK/pkemdd97hacXQgzQWTJs2rX6d/6PxkaGc6M0ml+h5pRlwytP0ek5D51133VX/1XyeDpNnnnmmHl6GQETevpguD0cX7/HIdxlS69lnnx1r9IrP6fGAsEFu/InPaCCLXqriPf76n33XafsWLFhQzysao1ku0xHOueqqq+pQTh6uiCGMaFynsG702tUsnXrgimnmz59ffz8aheN9HmkYopcPghHNQoCLxuvmNhNqIUQU6zSMABfL7rWeeNDbD72D5BLDfkWDdf6szXP2Ecdu9PCVv8Oxy9CT/Msl9m2n8E4O4TTPZeaRh2V84YUXxkKNsR2EMel9iDohF44xjuvmkG9MU7INfO+4446r7rjjjp2CN7xPwy+9kdH4S8kBLl6Xfq90PQc511nfZiGcQF0XPUU1P2f7CZx2smZaei6JsFK3RuXSZfSqZ0v8RnGsEgQgkDN9+vQmXf2ac5Wen5pDXXWc+J83m2G3mI7vEw667rrrKhqsKYRLqG8GqROpz+i5J/fUGMukXmP76BWxbYCLXoqYHyWCzTG//Fiy/0qtMcu9ZuX1aA75mT9rhib4jOsTPWNRJ3HONANcUXf1qoPpPYpjhvoxF+ZNPckQZ8w3F+4rqJ9waxbqXkILX375ZfOjvq9L1mW81ybCVgy3SunmnYeXztdQjkv2Qw5s5Y1im5cvX14Hq+J9wioMkRfX+C1btlRLlizZqRdXhpaOHibjezxyLjFkW3w3PuOehHsD7j3jWs9nbNcPP/wwtn05wJXvITvdB3e794xlltybDfv6kIcGj/Xq9RgBrrzt+f65Vx3MfLv1wMVn8+bNq66++upd7sG4nyYYyP0EJYLo7OMo1MH04pbLRRddNNbDJPcUBHWa512enueD1Je9zpte954sN/dCRT1BkKzTunZzZx7sy4ULF44FEnmPwnWE+zoCXM1Sss7j/fmgucxOr3MAi5+1+JkrCvuEezQesSGE2QxLZ5dO5yLtzxw3lGH8nJSvHd3qGpY1qnqdeVsUUEABBcoFDHC1tDPA1RLKyRRQQAEFFFBAAQUUUEABBRTYSwRozKBRmWGHaDCi0ezrr7/e5Zf2o+SgMZkeG/gFDz1mxF/2s0wa/2b909MGf2FNQy+9YtAoMJ7eMUrWfebMmfUwMzRkfPXVV3WDbrMho9N8+d0LoQDCKPQwwfBBNFIT2ulWaHw+9dRT6+/Q+xZD6n333XfVhg0bdgkWdZtHfp/GzMMPP7w2CksaZbBs9kiSv1fynLDYnDlz6v1D+IRjaP369dX27dt7zo7GcwIKNLowLdvKd0dVcD3hhBPqfwQCaYimEbLfeg66Ppxf9HhBkIuetWggZci2/1bh2KAhkeP0888/r3uF67cuJdtA0Ijtnj17dl2XEFri2OtXSr/HfEvWk++Vnut8t1M59NBD695KCChwzjGMJvUVAUkaRbuV6D2wV6NyfLdkGb3qWeZb6hfrNKxH6k4ayBlmjzqUUAfXJII84y1sE8EVvLi2EerJdTHXFYIa1D3UCcMoOMf6M5Quxz6hrVGXkv1XYn3QQQfV9wyc2wQfuD5yTe63jVwf8KYu5LrId5rB2VIjzBn+kHoNc9ap17y5rhNopY5iiDOuA5s2bWpVH/Zbx/Guy+68NlG/cj6w3Rz3XJM4vwjtsE86Fa4ZXDs5b7h+dQprdfoe78W9DfucexrqwBium8+pI7n3I9zKOsT9Cp8Nu5Tcm7EOw74+DHu7SufHOTDrn3tr9i3HAQG+Ud6DdVvP0vqy9N6z23qUvM/PTZxL/GwQ9Q7XrG6lZJ3jHKKuHsbPB93WbXe+X3ou9lvHUdbr/Zbt5woooIACnQUMcHV22eVdLo4WBRRQQAEFFFBAAQUUUEABBRRQQIE9R+Cmm26qG0FpTKcXi2bQjN5N6JWDsm7dumrFihV7zsa7JQrsAQIMvUdPIBQa0pctW7YHbJWboIACCiiggAIKKKCAAgoosDcKGOBqudcNcLWEcjIFFFBAAQUUUEABBRRQQAEFFFBgkgjk4dE++uijauXKlWNrTu8GDFkXwyY99NBDXXvaGPuSTxRQYOQCnJP0GEGPHYS3Yjhbhi2lVxyLAgoooIACCiiggAIKKKCAApNRwABXy71mgKsllJMpoIACCiiggAIKKKCAAgoooIACk0Tg+OOPr3vYIgxCYSir3377rWKoQIYqjLJ27dpq9erV8dJHBRT4LwowlNmiRYvqYRXj3GWoRXrRsyiggAIKKKCAAgoooIACCigwWQUMcLXccwa4WkI5mQIKKKCAAgoooIACCiiggAIKKDCJBObPn19dccUV1dSpUzuu9QcffFCtWrWq42e+qYACu18gAlyxZEKXDz/8cLV9+/Z4y0cFFFBAAQUUUEABBRRQQAEFJp2AAa6Wu8wAV0soJ1NAAQUUUEABBRRQQAEFFFBAAQUmmQDhrXPOOac6+uijq4MPPrjatm1b9cUXX1SbNm2qduzYMcm2xtVVYM8WOOCAA6oLLrigDl1u3bq12rhxY/XHH3/s2Rvt1imggAIKKKCAAgoooIACCuzxAga4Wu5iA1wtoZxMAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFWgsY4GpJZYCrJZSTKaCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiigQGsBA1wtqQxwtYRyMgUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFGgtYICrJZUBrpZQTqaAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAKtBQxwtaQywNUSyskUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFCgtYABrpZUBrhaQjmZAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKtBYwwNWSygBXSygnU0ABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAgdYCBrhaUhngagnlZAoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKNBawABXSyoDXC2hnEwBBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQVaCxjgakllgKsllJMpoIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKBAawEDXC2pDHC1hHIyBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUaC1ggKsllQGullBOpoACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAq0FDHC1pDLA1RLKyRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUKC1gAGu1lROqIACCiiggAIKKKCAAgoooIACCiigdw4mPAAAAw5JREFUgAIKKKCAAgoooIACCiiggAIKKKCAAgqMRuDXX3+tpk6dWu2zzz7VlClT6n8859+sWbPq1/vvv3/fhe8zb968/+07lRMooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgqMCRjgGqPwiQIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiwewUMcO1eb5emgAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCowJGOAao/CJAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKLB7BQxw7V5vl6aAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKjAkY4Bqj8MkoBQ455JDq2GOPrT799NPqr7/+GuWinLcCCiiwVwtY3+7Vu9+NV0ABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUECBSShggGsS7rTJtspHHHFE9eCDD9ar/eGHH1ZPP/30ZNuECbm+Rx55ZHXuuedWU6dOrX7//ffqlVdeqf78888Jua7DWqkFCxZUHE+UDRs2VJ999tmwZj3y+Zx11ll1iDEv6Oeff67WrFmT32r9fN99960uvvjiavr06fV32Pcvv/xytWPHjtbzcMI9T8D6ds/bp26RAgoooIACCiiggAIKKKCAAgoooIACCiiggAIKKKCAAgrs+QIGuAr38YwZM6o5c+ZUmzdvrrZu3Vo4l8nxtdmzZ1dbtmyp/v7776IVnjt3brV48eL6u3g98MADRfPxSzsL3H///RXHYZQlS5ZMqkBTrHfbx5kzZ1b33Xff2OS//PJLdffdd4+9nshP/vOf/1RLly7tuIqLFi3q+H6/N88777xq4cKFO032yCOP1L3c7fSmL/YqgdL6dtB6fq9CdmMVUEABBRRQQAEFFFBAAQUUUEABBRRQQAEFFFBAAQUUUECBIQsMK8D1f24uqiazdlIEAAAAAElFTkSuQmCC)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "O3ENsWYB27Mb" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Pk55Mjq_3DiR" - }, - "source": [ - "## Example Use Case 1 - Code Generator\n", - "### For this use case enter your system prompt and questions\n" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "id": "_1SZYJFB3HmQ" - }, - "outputs": [], - "source": [ - "# enter your system prompt if you have one\n", - "system_prompt = \"\"\"\n", - "You are a coding assistant helping users using litellm.\n", - "litellm is a light package to simplify calling OpenAI, Azure, Cohere, Anthropic, Huggingface API Endpoints\n", - "--\n", - "Sample Usage:\n", - "```\n", - "pip install litellm\n", - "from litellm import completion\n", - "## set ENV variables\n", - "os.environ[\"OPENAI_API_KEY\"] = \"openai key\"\n", - "os.environ[\"COHERE_API_KEY\"] = \"cohere key\"\n", - "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - "# openai call\n", - "response = completion(model=\"gpt-3.5-turbo\", messages=messages)\n", - "# cohere call\n", - "response = completion(\"command-nightly\", messages)\n", - "```\n", - "\n", - "\"\"\"\n", - "\n", - "\n", - "# qustions/logs you want to run the LLM on\n", - "questions = [\n", - " \"what is litellm?\",\n", - " \"why should I use LiteLLM\",\n", - " \"does litellm support Anthropic LLMs\",\n", - " \"write code to make a litellm completion call\",\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "AHH3cqeU3_ZT" - }, - "source": [ - "## Running questions\n", - "### Select from 100+ LLMs here: https://docs.litellm.ai/docs/providers" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "BpQD4A5339L3" - }, - "outputs": [], - "source": [ - "from litellm import completion, completion_cost\n", - "import os\n", - "import time\n", - "\n", - "# optional use litellm dashboard to view logs\n", - "# litellm.use_client = True\n", - "# litellm.token = \"ishaan_2@berri.ai\" # set your email\n", - "\n", - "\n", - "# set API keys\n", - "os.environ['TOGETHERAI_API_KEY'] = \"\"\n", - "os.environ['OPENAI_API_KEY'] = \"\"\n", - "os.environ['ANTHROPIC_API_KEY'] = \"\"\n", - "\n", - "\n", - "# select LLMs to benchmark\n", - "# using https://api.together.xyz/playground for llama2\n", - "# try any supported LLM here: https://docs.litellm.ai/docs/providers\n", - "\n", - "models = ['togethercomputer/llama-2-70b-chat', 'gpt-3.5-turbo', 'claude-instant-1.2']\n", - "data = []\n", - "\n", - "for question in questions: # group by question\n", - " for model in models:\n", - " print(f\"running question: {question} for model: {model}\")\n", - " start_time = time.time()\n", - " # show response, response time, cost for each question\n", - " response = completion(\n", - " model=model,\n", - " max_tokens=500,\n", - " messages = [\n", - " {\n", - " \"role\": \"system\", \"content\": system_prompt\n", - " },\n", - " {\n", - " \"role\": \"user\", \"content\": question\n", - " }\n", - " ],\n", - " )\n", - " end = time.time()\n", - " total_time = end-start_time # response time\n", - " # print(response)\n", - " cost = completion_cost(response) # cost for completion\n", - " raw_response = response['choices'][0]['message']['content'] # response string\n", - "\n", - "\n", - " # add log to pandas df\n", - " data.append(\n", - " {\n", - " 'Model': model,\n", - " 'Question': question,\n", - " 'Response': raw_response,\n", - " 'ResponseTime': total_time,\n", - " 'Cost': cost\n", - " })" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "apOSV3PBLa5Y" - }, - "source": [ - "## View Benchmarks for LLMs" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - }, - "id": "CJqBlqUh_8Ws", - "outputId": "e02c3427-d8c6-4614-ff07-6aab64247ff6" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Question: does litellm support Anthropic LLMs\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ModelQuestionResponseResponseTimeCost
6togethercomputer/llama-2-70b-chatdoes litellm support Anthropic LLMsYes, litellm supports Anthropic LLMs.\\n\\nIn the example usage you provided, the `completion` function is called with the `model` parameter set to `\"gpt-3.5-turbo\"` for OpenAI and `\"command-nightly\"` for Cohere.\\n\\nTo use an Anthropic LLM with litellm, you would set the `model` parameter to the name of the Anthropic model you want to use, followed by the version number, if applicable. For example:\\n```\\nresponse = completion(model=\"anthropic-gpt-2\", messages=messages)\\n```\\nThis would call the Anthropic GPT-2 model to generate a completion for the given input messages.\\n\\nNote that you will need to set the `ANTHROPIC_API_KEY` environment variable to your Anthropic API key before making the call. You can do this by running the following command in your terminal:\\n```\\nos.environ[\"ANTHROPIC_API_KEY\"] = \"your-anthropic-api-key\"\\n```\\nReplace `\"your-anthropic-api-key\"` with your actual Anthropic API key.\\n\\nOnce you've set the environment variable, you can use the `completion` function with the `model` parameter set to an Anthropic model name to call the Anthropic API and generate a completion.21.5130090.001347
7gpt-3.5-turbodoes litellm support Anthropic LLMsNo, currently litellm does not support Anthropic LLMs. It mainly focuses on simplifying the usage of OpenAI, Azure, Cohere, and Huggingface API endpoints.8.6565100.000342
8claude-instant-1.2does litellm support Anthropic LLMsYes, litellm supports calling Anthropic LLMs through the completion function.\\n\\nTo use an Anthropic model with litellm:\\n\\n1. Set the ANTHROPIC_API_KEY environment variable with your Anthropic API key\\n\\n2. Pass the model name as the 'model' argument to completion(). Anthropic model names follow the format 'anthropic/<model_name>'\\n\\nFor example:\\n\\n```python \\nimport os\\nfrom litellm import completion\\n\\nos.environ[\"ANTHROPIC_API_KEY\"] = \"your_anthropic_api_key\"\\n\\nmessages = [{\"content\": \"Hello\", \"role\": \"user\"}]\\n\\nresponse = completion(model=\"anthropic/constitutional\", messages=messages)\\n```\\n\\nThis would call the Constitutional AI model from Anthropic.\\n\\nSo in summary, litellm provides a simple interface to call any Anthropic models as long as you specify the model name correctly and set the ANTHROPIC_API_KEY env variable.9.6981950.001342
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Question: what is litellm?\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ModelQuestionResponseResponseTimeCost
0togethercomputer/llama-2-70b-chatwhat is litellm?Litellm is a lightweight Python package that simplifies calling various AI API endpoints, including OpenAI, Azure, Cohere, Anthropic, and Hugging Face. It provides a convenient interface for making requests to these APIs, allowing developers to easily integrate them into their applications. With Litellm, developers can quickly and easily interact with multiple AI models and services, without having to handle the details of authentication, API calls, and response parsing. This makes it easier to build and deploy AI-powered applications, and can help developers save time and effort.13.4796440.000870
1gpt-3.5-turbowhat is litellm?litellm is a light package that provides a simplified interface for making API calls to various language models and APIs. It abstracts away the complexities of handling network requests, authentication, and response parsing, making it easier for developers to integrate powerful language models into their applications.\\n\\nWith litellm, you can quickly make API calls to models like OpenAI's GPT-3.5 Turbo, Azure's Text Analytics, Cohere's Command API, Anthropic's API, and Huggingface's models. It also supports additional functionality like conversational AI, summarization, translation, and more.\\n\\nBy using litellm, you can focus on your application logic without getting tangled in the details of API integration, allowing you to quickly build intelligent and conversational applications.8.3243320.000566
2claude-instant-1.2what is litellm?litellm is a Python library that simplifies calling various AI API endpoints like OpenAI, Azure, Cohere, Anthropic, and Huggingface. \\n\\nSome key things to know about litellm:\\n\\n- It provides a consistent interface for completing prompts and generating responses from different AI models through a single method called completion().\\n\\n- You specify the API (e.g. OpenAI, Cohere etc.) and model either by name or by setting environment variables before making the completion call.\\n\\n- This avoids having to use different SDKs or APIs for each provider and standardizes the call structure. \\n\\n- It handles things like setting headers, encoding inputs, parsing responses so the user doesn't have to deal with those details.\\n\\n- The goal is to make it easy to try different AI APIs and models without having to change code or learn different interfaces.\\n\\n- It's lightweight with no other dependencies required besides what's needed for each API (e.g. openai, azure SDKs etc.).\\n\\nSo in summary, litellm is a small library that provides a common way to interact with multiple conversational AI APIs through a single Python method, avoiding the need to directly use each provider's specific SDK.10.3164880.001603
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Question: why should I use LiteLLM\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ModelQuestionResponseResponseTimeCost
3togethercomputer/llama-2-70b-chatwhy should I use LiteLLM\\nThere are several reasons why you might want to use LiteLLM:\\n\\n1. Simplified API calls: LiteLLM provides a simple and consistent API for calling various language models, making it easier to use multiple models and switch between them.\\n2. Environment variable configuration: LiteLLM allows you to set environment variables for API keys and model names, making it easier to manage and switch between different models and APIs.\\n3. Support for multiple models and APIs: LiteLLM supports a wide range of language models and APIs, including OpenAI, Azure, Cohere, Anthropic, and Hugging Face.\\n4. Easy integration with popular frameworks: LiteLLM can be easily integrated with popular frameworks such as PyTorch and TensorFlow, making it easy to use with your existing codebase.\\n5. Lightweight: LiteLLM is a lightweight package, making it easy to install and use, even on resource-constrained devices.\\n6. Flexible: LiteLLM allows you to define your own models and APIs, making it easy to use with custom models and APIs.\\n7. Extensive documentation: LiteLLM has extensive documentation, making it easy to get started and learn how to use the package.\\n8. Active community: LiteLLM has an active community of developers and users, making it easy to get help and feedback on your projects.\\n\\nOverall, LiteLLM can help you to simplify your workflow, improve your productivity, and make it easier to work with multiple language models and APIs.23.7778850.001443
4gpt-3.5-turbowhy should I use LiteLLMLiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenience.12.1098810.000881
5claude-instant-1.2why should I use LiteLLMHere are some key reasons why you may want to consider using LiteLLM:\\n\\n- Simplifies calling multiple large language models - LiteLLM provides a unified API to call models from different providers like OpenAI, Azure, HuggingFace, Anthropic etc. This avoids having to deal with different APIs from each provider.\\n\\n- Easy to get started - LiteLLM is very lightweight and simple to install with just one pip install. It has minimal configuration needed with just API keys. This lowers the barrier to start experimenting with LLMs.\\n\\n- Standard interface - The completion API provides a standard way to send messages and get responses back regardless of underlying model. This keeps the code interface consistent. \\n\\n- Model agnostic - Your code doesn't need to change to switch between different models. You can easily compare performance of models from different providers.\\n\\n- Open source - As it is open source, the library can be expanded to support more models and providers over time via community contributions.\\n\\n- Cost effective - LiteLLM helps avoid vendor lock-in. You can optimize costs by testing multiple models and choosing the best suitable and cost-effective option for your needs.\\n\\nSo in summary, L12.6102590.001568
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Question: write code to make a litellm completion call\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ModelQuestionResponseResponseTimeCost
9togethercomputer/llama-2-70b-chatwrite code to make a litellm completion call\\nTo make a completion call using litellm, you can use the `completion` function from the litellm package. Here's an example of how to use it:\\n```\\nfrom litellm import completion\\n\\n# Set ENV variables\\nos.environ[\"OPENAI_API_KEY\"] = \"your_openai_api_key\"\\nos.environ[\"COHERE_API_KEY\"] = \"your_cohere_api_key\"\\n\\n# Define the messages to be completed\\nmessages = [\\n {\\n \"content\": \"Hello, how are you?\",\\n \"role\": \"user\"\\n }\\n]\\n\\n# Make a completion call using OpenAI\\nresponse = completion(model=\"gpt-3.5-turbo\", messages=messages)\\n\\n# Make a completion call using Cohere\\nresponse = completion(\"command-nightly\", messages)\\n```\\nIn this example, we first set the ENV variables for the OpenAI and Cohere API keys. Then, we define a list of messages to be completed, which in this case contains a single message with the content \"Hello, how are you?\" and the role \"user\".\\n\\nNext, we make two completion calls using the `completion` function from litellm. The first call uses the OpenAI model `gpt-3.5-turbo` and passes in the list of messages. The second call uses the Cohere model `command-nightly` and passes in the same list of messages.\\n\\nThe `completion` function returns a response object that contains the completed messages. You can then use the `response.messages` attribute to access the completed messages.\\n\\nHere's an example of how to access the completed messages:\\n```\\n# Print the completed messages\\nprint(response.messages)\\n```\\nThis will print the completed messages, which should look something like this:\\n```\\n[{'content': 'Hello, how are you?', 'role': 'user', 'completion': 'I am doing well, thanks for asking.'}]\\n```\\nNote that the `completion` attribute contains the completed message. You can use this attribute to retrieve the completed message and use it in your application.32.1924530.001821
10gpt-3.5-turbowrite code to make a litellm completion callSure! Here's an example code to make a completion call using the litellm package:\\n\\n```python\\nfrom litellm import completion\\n\\n# Set the model and messages\\nmodel = \"gpt-3.5-turbo\"\\nmessages = [\\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\\n {\"role\": \"user\", \"content\": \"Who won the world series in 2020?\"},\\n {\"role\": \"assistant\", \"content\": \"The Los Angeles Dodgers won the World Series in 2020.\"},\\n {\"role\": \"user\", \"content\": \"Where was it played?\"}\\n]\\n\\n# Make the completion call\\nresponse = completion(model=model, messages=messages)\\n\\n# Print the assistant's reply\\nassistant_reply = response[\"choices\"][0][\"message\"][\"content\"]\\nprint(\"Assistant: \", assistant_reply)\\n```\\n\\nMake sure you have the litellm package installed (`pip install litellm`) and set the necessary environment variables for the API keys before running this code.9.3771550.000686
11claude-instant-1.2write code to make a litellm completion callHere is an example of making a completion call using litellm:\\n\\n```python\\nimport os\\nfrom litellm import completion\\n\\n# Set API keys as environment variables\\nos.environ[\"OPENAI_API_KEY\"] = \"your openai api key\" \\n\\n# Conversation context \\nmessages = [{\\n \"content\": \"Hello, how can I help you today?\",\\n \"role\": \"assistant\"\\n}]\\n\\n# Make completion call with GPT-3 model\\nresponse = completion(\\n model=\"gpt-3.5-turbo\", \\n messages=messages\\n)\\n\\nprint(response)\\n```\\n\\nTo break it down:\\n\\n- Import completion from litellm\\n- Set the OPENAI_API_KEY env var \\n- Define a messages list with the conversation context\\n- Call completion(), specifying the model (\"gpt-3.5-turbo\") and messages\\n- It will return the response from the API\\n- Print the response\\n\\nThis makes a simple completion call to OpenAI GPT-3 using litellm to handle the API details. You can also call other models like Cohere or Anthropic by specifying their name instead of the OpenAI9.8399880.001578
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from IPython.core.interactiveshell import InteractiveShell\n", - "InteractiveShell.ast_node_interactivity = \"all\"\n", - "from IPython.display import HTML\n", - "import pandas as pd\n", - "\n", - "df = pd.DataFrame(data)\n", - "grouped_by_question = df.groupby('Question')\n", - "\n", - "for question, group_data in grouped_by_question:\n", - " print(f\"Question: {question}\")\n", - " HTML(group_data.to_html())\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bmtAbC1rGVAm" - }, - "source": [ - "## Use Case 2 - Rewrite user input concisely" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "id": "boiHO1PhGXSL" - }, - "outputs": [], - "source": [ - "# enter your system prompt if you have one\n", - "system_prompt = \"\"\"\n", - "For a given user input, rewrite the input to make be more concise.\n", - "\"\"\"\n", - "\n", - "# user input for re-writing questions\n", - "questions = [\n", - " \"LiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenience\",\n", - " \"Hi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need!\",\n", - " \"Traceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack.\"\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fwNcC_obICUc" - }, - "source": [ - "## Run Questions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "KtBjZ1mUIBiJ" - }, - "outputs": [], - "source": [ - "from litellm import completion, completion_cost\n", - "import os\n", - "import time\n", - "\n", - "# optional use litellm dashboard to view logs\n", - "# litellm.use_client = True\n", - "# litellm.token = \"ishaan_2@berri.ai\" # set your email\n", - "\n", - "os.environ['TOGETHERAI_API_KEY'] = \"\"\n", - "os.environ['OPENAI_API_KEY'] = \"\"\n", - "os.environ['ANTHROPIC_API_KEY'] = \"\"\n", - "\n", - "models = ['togethercomputer/llama-2-70b-chat', 'gpt-3.5-turbo', 'claude-instant-1.2'] # enter llms to benchmark\n", - "data_2 = []\n", - "\n", - "for question in questions: # group by question\n", - " for model in models:\n", - " print(f\"running question: {question} for model: {model}\")\n", - " start_time = time.time()\n", - " # show response, response time, cost for each question\n", - " response = completion(\n", - " model=model,\n", - " max_tokens=500,\n", - " messages = [\n", - " {\n", - " \"role\": \"system\", \"content\": system_prompt\n", - " },\n", - " {\n", - " \"role\": \"user\", \"content\": \"User input:\" + question\n", - " }\n", - " ],\n", - " )\n", - " end = time.time()\n", - " total_time = end-start_time # response time\n", - " # print(response)\n", - " cost = completion_cost(response) # cost for completion\n", - " raw_response = response['choices'][0]['message']['content'] # response string\n", - " #print(raw_response, total_time, cost)\n", - "\n", - " # add to pandas df\n", - " data_2.append(\n", - " {\n", - " 'Model': model,\n", - " 'Question': question,\n", - " 'Response': raw_response,\n", - " 'ResponseTime': total_time,\n", - " 'Cost': cost\n", - " })\n", - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-PCYIzG5M0II" - }, - "source": [ - "## View Logs - Group by Question" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - }, - "id": "-3R5-2q8IiL2", - "outputId": "c4a0d9e5-bb21-4de0-fc4c-9f5e71d0f177" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Question: Hi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need!\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ModelQuestionResponseResponseTimeCost
3togethercomputer/llama-2-70b-chatHi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need!\\nHere's a more concise version of the user input:\\n\\n\"Hi everyone! I'm [your name] and I'm working on [your project/role involving LLMs]. I recently discovered LiteLLM and I'm excited to use it to [build an app/simplify my code/test different models etc]. Before LiteLLM, I struggled with [describe any issues you faced working with multiple LLMs]. I'm looking forward to using LiteLLM's unified API and automatic translation to achieve my goals. I'm eager to learn more about building impactful applications powered by LLMs and to be part of this community. Let me know if you have any questions or need further clarification.\"\\n\\nIn this revised version, we've kept the essential information and removed some of the extraneous language. We've also rephrased some of the sentences to make them more concise and easier to read.18.3006200.001200
4gpt-3.5-turboHi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need!User input: Hi, I'm [your name] and I'm excited about using LiteLLM to simplify working with different LLM providers. Before finding LiteLLM, I faced challenges working with multiple LLMs. With LiteLLM's unified API and automatic translation, I believe it will help me achieve my goals of [state your goals]. I look forward to being part of this community and learning how to build impactful applications with LLMs. Let me know if you need any further clarification or details.7.3854720.000525
5claude-instant-1.2Hi everyone! I'm [your name] and I'm currently working on [your project/role involving LLMs]. I came across LiteLLM and was really excited by how it simplifies working with different LLM providers. I'm hoping to use LiteLLM to [build an app/simplify my code/test different models etc]. Before finding LiteLLM, I was struggling with [describe any issues you faced working with multiple LLMs]. With LiteLLM's unified API and automatic translation between providers, I think it will really help me to [goals you have for using LiteLLM]. Looking forward to being part of this community and learning more about how I can build impactful applications powered by LLMs!Let me know if you would like me to modify or expand on any part of this suggested intro. I'm happy to provide any clarification or additional details you need!Here is a more concise rewrite of the user input:\\n\\nHi everyone, I'm [your name]. I'm currently [your project/role] and came across LiteLLM, which simplifies working with different LLMs through its unified API. I hope to [build an app/simplify code/test models] with LiteLLM since I previously struggled with [issues]. LiteLLM's automatic translation between providers will help me [goals] and build impactful LLM applications. Looking forward to learning more as part of this community. Let me know if you need any clarification on my plans to use LiteLLM.8.6282170.001022
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Question: LiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\n", - "\n", - "1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\n", - "\n", - "2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\n", - "\n", - "3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\n", - "\n", - "4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\n", - "\n", - "5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\n", - "\n", - "6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\n", - "\n", - "Overall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenience\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ModelQuestionResponseResponseTimeCost
0togethercomputer/llama-2-70b-chatLiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenienceHere's a more concise version of the user input:\\n\\nLiteLLM is a lightweight Python package that simplifies API calls to various language models. It abstracts away complexity, integrates seamlessly, supports multiple models, and handles authentication. It's ideal for rapid prototyping and has community support. It saves time and effort while providing flexibility and convenience.11.2942500.001251
1gpt-3.5-turboLiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenienceLiteLLM is a lightweight Python package that simplifies API calls to various language models. Here's why you should use it:\\n1. Simplified API Calls: Works with multiple models (OpenAI, Azure, Cohere, Anthropic, Huggingface).\\n2. Easy Integration: Import and start using it quickly in your codebase.\\n3. Flexibility: Supports GPT-3, GPT-Neo, chatGPT, etc. easily switch between models.\\n4. Convenience: Handles authentication and connection details, just set environment variables.\\n5. Quick Prototyping: Great for rapid prototyping and building interactive applications.\\n6. Community Support: Actively maintained and supported by a developer community.9.7783150.000795
2claude-instant-1.2LiteLLM is a lightweight Python package that simplifies the process of making API calls to various language models. Here are some reasons why you should use LiteLLM:\\n\\n1. **Simplified API Calls**: LiteLLM abstracts away the complexity of making API calls to different language models. It provides a unified interface for invoking models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more.\\n\\n2. **Easy Integration**: LiteLLM seamlessly integrates with your existing codebase. You can import the package and start making API calls with just a few lines of code.\\n\\n3. **Flexibility**: LiteLLM supports a variety of language models, including GPT-3, GPT-Neo, chatGPT, and more. You can choose the model that suits your requirements and easily switch between them.\\n\\n4. **Convenience**: LiteLLM handles the authentication and connection details for you. You just need to set the relevant environment variables, and the package takes care of the rest.\\n\\n5. **Quick Prototyping**: LiteLLM is ideal for rapid prototyping and experimentation. With its simple API, you can quickly generate text, chat with models, and build interactive applications.\\n\\n6. **Community Support**: LiteLLM is actively maintained and supported by a community of developers. You can find help, share ideas, and collaborate with others to enhance your projects.\\n\\nOverall, LiteLLM simplifies the process of making API calls to language models, saving you time and effort while providing flexibility and convenienceHere is a concise rewrite of the user input:\\n\\nLiteLLM is a lightweight Python package that simplifies accessing various language models. It provides a unified interface for models from OpenAI, Azure, Cohere, Anthropic, Huggingface, and more. Key benefits include simplified API calls, easy integration, flexibility to use different models, automated handling of authentication, and support for quick prototyping. The actively maintained package saves time by abstracting away complexity while offering convenience and a collaborative community.7.6975200.001098
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Question: Traceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack.\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
ModelQuestionResponseResponseTimeCost
6togethercomputer/llama-2-70b-chatTraceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack.\\nRewritten input: Traceloop is a platform for monitoring and debugging LLM outputs. It allows users to track performance, rollout changes confidently, and debug issues in production. It uses OpenTelemetry for full visibility into LLM requests, vector DB usage, and other infrastructure.9.0604440.000525
7gpt-3.5-turboTraceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack.Traceloop is a platform for monitoring and debugging the quality of your LLM outputs. It helps track performance, rollout changes, and debug issues in production. It is based on OpenTelemetry, providing visibility to LLM requests, vector DB usage, and other infrastructure in your stack.7.3046610.000283
8claude-instant-1.2Traceloop is a platform for monitoring and debugging the quality of your LLM outputs. It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. It is based on OpenTelemetry, so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack.Here is a more concise rewrite of the user input:\\n\\nTraceloop monitors and debugs LLM quality. It tracks LLM performance, enables confident changes, and debugs production issues. Based on OpenTelemetry, Traceloop provides full visibility into LLM requests, vector DB usage, and other stack infrastructure.7.9761580.000538
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 20, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from IPython.core.interactiveshell import InteractiveShell\n", - "InteractiveShell.ast_node_interactivity = \"all\"\n", - "from IPython.display import HTML\n", - "import pandas as pd\n", - "\n", - "df = pd.DataFrame(data_2)\n", - "grouped_by_question = df.groupby('Question')\n", - "\n", - "for question, group_data in grouped_by_question:\n", - " print(f\"Question: {question}\")\n", - " HTML(group_data.to_html())\n" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb b/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb deleted file mode 100644 index 338785ea5e3..00000000000 --- a/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb +++ /dev/null @@ -1,406 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ZwuaylskLxFu", - "outputId": "d684d6a3-32fe-4beb-c378-c39134bcf8cc" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting litellm==0.1.363\n", - " Downloading litellm-0.1.363-py3-none-any.whl (34 kB)\n", - "Requirement already satisfied: openai<0.28.0,>=0.27.8 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.27.8)\n", - "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (1.0.0)\n", - "Requirement already satisfied: tiktoken<0.5.0,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.4.0)\n", - "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (2.31.0)\n", - "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.65.0)\n", - "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.8.5)\n", - "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm==0.1.363) (2022.10.31)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.2.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.26.16)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (2023.7.22)\n", - "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (23.1.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (6.0.4)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.0.2)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.9.2)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.4.0)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.3.1)\n", - "Installing collected packages: litellm\n", - " Attempting uninstall: litellm\n", - " Found existing installation: litellm 0.1.362\n", - " Uninstalling litellm-0.1.362:\n", - " Successfully uninstalled litellm-0.1.362\n", - "Successfully installed litellm-0.1.363\n" - ] - } - ], - "source": [ - "!pip install litellm==\"0.1.363\"" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "W216G__XL19Q" - }, - "outputs": [], - "source": [ - "# @title Import litellm & Set env variables\n", - "import litellm\n", - "import os\n", - "\n", - "os.environ[\"ANTHROPIC_API_KEY\"] = \" \" #@param" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ff1lKwUMMLJj", - "outputId": "bfddf6f8-36d4-45e5-92dc-349083fa41b8" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "\n", - " Result from claude-instant-1 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" The Los Angeles Dodgers won the 2020 World Series, defeating the Tampa Bay Rays 4-2. It was the Dodgers' first World Series title since 1988.\"}}], 'created': 1691536677.2676156, 'model': 'claude-instant-1', 'usage': {'prompt_tokens': 30, 'completion_tokens': 32, 'total_tokens': 62}}\n", - "\n", - "\n", - " Result from claude-2 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ' The Los Angeles Dodgers won'}}], 'created': 1691536677.944753, 'model': 'claude-2', 'usage': {'prompt_tokens': 30, 'completion_tokens': 5, 'total_tokens': 35}}\n" - ] - } - ], - "source": [ - "# @title Request Claude Instant-1 and Claude-2\n", - "messages = [\n", - " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", - " {\"role\": \"user\", \"content\": \"Who won the world series in 2020?\"}\n", - " ]\n", - "\n", - "result = litellm.completion('claude-instant-1', messages)\n", - "print(\"\\n\\n Result from claude-instant-1\", result)\n", - "result = litellm.completion('claude-2', messages, max_tokens=5, temperature=0.2)\n", - "print(\"\\n\\n Result from claude-2\", result)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "06hWKnNQMrV-", - "outputId": "7fdec0eb-d4a9-4882-f9c4-987ff9a31114" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " Here\n", - "'s\n", - " a\n", - " quick\n", - " overview\n", - " of\n", - " how\n", - " a\n", - " court\n", - " case\n", - " can\n", - " reach\n", - " the\n", - " U\n", - ".\n", - "S\n", - ".\n", - " Supreme\n", - " Court\n", - ":\n", - "\n", - "\n", - "-\n", - " The\n", - " case\n", - " must\n", - " first\n", - " be\n", - " heard\n", - " in\n", - " a\n", - " lower\n", - " trial\n", - " court\n", - " (\n", - "either\n", - " a\n", - " state\n", - " court\n", - " or\n", - " federal\n", - " district\n", - " court\n", - ").\n", - " The\n", - " trial\n", - " court\n", - " makes\n", - " initial\n", - " r\n", - "ulings\n", - " and\n", - " produces\n", - " a\n", - " record\n", - " of\n", - " the\n", - " case\n", - ".\n", - "\n", - "\n", - "-\n", - " The\n", - " losing\n", - " party\n", - " can\n", - " appeal\n", - " the\n", - " decision\n", - " to\n", - " an\n", - " appeals\n", - " court\n", - " (\n", - "a\n", - " state\n", - " appeals\n", - " court\n", - " for\n", - " state\n", - " cases\n", - ",\n", - " or\n", - " a\n", - " federal\n", - " circuit\n", - " court\n", - " for\n", - " federal\n", - " cases\n", - ").\n", - " The\n", - " appeals\n", - " court\n", - " reviews\n", - " the\n", - " trial\n", - " court\n", - "'s\n", - " r\n", - "ulings\n", - " and\n", - " can\n", - " affirm\n", - ",\n", - " reverse\n", - ",\n", - " or\n", - " modify\n", - " the\n", - " decision\n", - ".\n", - "\n", - "\n", - "-\n", - " If\n", - " a\n", - " party\n", - " is\n", - " still\n", - " unsat\n", - "isf\n", - "ied\n", - " after\n", - " the\n", - " appeals\n", - " court\n", - " rules\n", - ",\n", - " they\n", - " can\n", - " petition\n", - " the\n", - " Supreme\n", - " Court\n", - " to\n", - " hear\n", - " the\n", - " case\n", - " through\n", - " a\n", - " writ\n", - " of\n", - " cert\n", - "ior\n", - "ari\n", - ".\n", - " \n", - "\n", - "\n", - "-\n", - " The\n", - " Supreme\n", - " Court\n", - " gets\n", - " thousands\n", - " of\n", - " cert\n", - " petitions\n", - " every\n", - " year\n", - " but\n", - " usually\n", - " only\n", - " agrees\n", - " to\n", - " hear\n", - " about\n", - " 100\n", - "-\n", - "150\n", - " of\n", - " cases\n", - " that\n", - " have\n", - " significant\n", - " national\n", - " importance\n", - " or\n", - " where\n", - " lower\n", - " courts\n", - " disagree\n", - " on\n", - " federal\n", - " law\n", - ".\n", - " \n", - "\n", - "\n", - "-\n", - " If\n", - " 4\n", - " out\n", - " of\n", - " the\n", - " 9\n", - " Just\n", - "ices\n", - " vote\n", - " to\n", - " grant\n", - " cert\n", - " (\n", - "agree\n", - " to\n", - " hear\n", - " the\n", - " case\n", - "),\n", - " it\n", - " goes\n", - " on\n", - " the\n", - " Supreme\n", - " Court\n", - "'s\n", - " do\n", - "cket\n", - " for\n", - " arguments\n", - ".\n", - "\n", - "\n", - "-\n", - " The\n", - " Supreme\n", - " Court\n", - " then\n", - " hears\n", - " oral\n", - " arguments\n", - ",\n", - " considers\n", - " written\n", - " brief\n", - "s\n", - ",\n", - " examines\n", - " the\n", - " lower\n", - " court\n", - " records\n", - ",\n", - " and\n", - " issues\n", - " a\n", - " final\n", - " ruling\n", - " on\n", - " the\n", - " case\n", - ",\n", - " which\n", - " serves\n", - " as\n", - " binding\n", - " precedent\n" - ] - } - ], - "source": [ - "# @title Streaming Example: Request Claude-2\n", - "messages = [\n", - " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", - " {\"role\": \"user\", \"content\": \"how does a court case get to the Supreme Court?\"}\n", - " ]\n", - "\n", - "result = litellm.completion('claude-2', messages, stream=True)\n", - "for part in result:\n", - " print(part.choices[0].delta.content or \"\")\n", - "\n" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/Evaluating_LLMs.ipynb b/cookbook/Evaluating_LLMs.ipynb deleted file mode 100644 index e27e8934f76..00000000000 --- a/cookbook/Evaluating_LLMs.ipynb +++ /dev/null @@ -1,579 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "Ys9n20Es2IzT" - }, - "source": [ - "# Evaluate Multiple LLM Providers with LiteLLM\n", - "\n", - "\n", - "\n", - "* Quality Testing\n", - "* Load Testing\n", - "* Duration Testing\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZXOXl23PIIP6" - }, - "outputs": [], - "source": [ - "!pip install litellm python-dotenv" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "LINuBzXDItq2" - }, - "outputs": [], - "source": [ - "from litellm import load_test_model, testing_batch_completion" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "EkxMhsWdJdu4" - }, - "outputs": [], - "source": [ - "import os \n", - "os.environ[\"OPENAI_API_KEY\"] = \"...\"\n", - "os.environ[\"ANTHROPIC_API_KEY\"] = \"...\"\n", - "os.environ[\"REPLICATE_API_KEY\"] = \"...\"" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "mv5XdnqeW5I_" - }, - "source": [ - "# Quality Test endpoint\n", - "\n", - "## Test the same prompt across multiple LLM providers\n", - "\n", - "In this example, let's ask some questions about Paul Graham" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "id": "XpzrR5m4W_Us" - }, - "outputs": [], - "source": [ - "models = [\"gpt-3.5-turbo\", \"gpt-3.5-turbo-16k\", \"gpt-4\", \"claude-instant-1\", {\"model\": \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"custom_llm_provider\": \"replicate\"}]\n", - "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n", - "prompts = [\"Who is Paul Graham?\", \"What is Paul Graham known for?\" , \"Is paul graham a writer?\" , \"Where does Paul Graham live?\", \"What has Paul Graham done?\"]\n", - "messages = [[{\"role\": \"user\", \"content\": context + \"\\n\" + prompt}] for prompt in prompts] # pass in a list of messages we want to test\n", - "result = testing_batch_completion(models=models, messages=messages)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "9nzeLySnvIIW" - }, - "source": [ - "## Visualize the data" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 403 - }, - "id": "X-2n7hdAuVAY", - "outputId": "69cc0de1-68e3-4c12-a8ea-314880010d94" - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Model Nameclaude-instant-1gpt-3.5-turbo-0613gpt-3.5-turbo-16k-0613gpt-4-0613replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781
Prompt
\\nIs paul graham a writer?Yes, Paul Graham is considered a writer in ad...Yes, Paul Graham is a writer. He has written s...Yes, Paul Graham is a writer. He has authored ...Yes, Paul Graham is a writer. He is an essayis...Yes, Paul Graham is an author. According to t...
\\nWhat has Paul Graham done?Paul Graham has made significant contribution...Paul Graham has achieved several notable accom...Paul Graham has made significant contributions...Paul Graham is known for his work on the progr...Paul Graham has had a diverse career in compu...
\\nWhat is Paul Graham known for?Paul Graham is known for several things:\\n\\n-...Paul Graham is known for his work on the progr...Paul Graham is known for his work on the progr...Paul Graham is known for his work on the progr...Paul Graham is known for many things, includi...
\\nWhere does Paul Graham live?Based on the information provided:\\n\\n- Paul ...According to the given information, Paul Graha...Paul Graham currently lives in England, where ...The text does not provide a current place of r...Based on the information provided, Paul Graha...
\\nWho is Paul Graham?Paul Graham is an influential computer scient...Paul Graham is an English computer scientist, ...Paul Graham is an English computer scientist, ...Paul Graham is an English computer scientist, ...Paul Graham is an English computer scientist,...
\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - "
\n", - " \n", - "
\n", - "\n", - "\n", - "\n", - " \n", - "\n", - "\n", - " \n", - " \n", - "\n", - " \n", - "
\n", - "
\n" - ], - "text/plain": [ - "Model Name claude-instant-1 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is considered a writer in ad... \n", - "\\nWhat has Paul Graham done? Paul Graham has made significant contribution... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for several things:\\n\\n-... \n", - "\\nWhere does Paul Graham live? Based on the information provided:\\n\\n- Paul ... \n", - "\\nWho is Paul Graham? Paul Graham is an influential computer scient... \n", - "\n", - "Model Name gpt-3.5-turbo-0613 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He has written s... \n", - "\\nWhat has Paul Graham done? Paul Graham has achieved several notable accom... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n", - "\\nWhere does Paul Graham live? According to the given information, Paul Graha... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n", - "\n", - "Model Name gpt-3.5-turbo-16k-0613 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He has authored ... \n", - "\\nWhat has Paul Graham done? Paul Graham has made significant contributions... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n", - "\\nWhere does Paul Graham live? Paul Graham currently lives in England, where ... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n", - "\n", - "Model Name gpt-4-0613 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He is an essayis... \n", - "\\nWhat has Paul Graham done? Paul Graham is known for his work on the progr... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n", - "\\nWhere does Paul Graham live? The text does not provide a current place of r... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n", - "\n", - "Model Name replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781 \n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is an author. According to t... \n", - "\\nWhat has Paul Graham done? Paul Graham has had a diverse career in compu... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for many things, includi... \n", - "\\nWhere does Paul Graham live? Based on the information provided, Paul Graha... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist,... " - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "# Create an empty list to store the row data\n", - "table_data = []\n", - "\n", - "# Iterate through the list and extract the required data\n", - "for item in result:\n", - " prompt = item['prompt'][0]['content'].replace(context, \"\") # clean the prompt for easy comparison\n", - " model = item['response']['model']\n", - " response = item['response']['choices'][0]['message']['content']\n", - " table_data.append([prompt, model, response])\n", - "\n", - "# Create a DataFrame from the table data\n", - "df = pd.DataFrame(table_data, columns=['Prompt', 'Model Name', 'Response'])\n", - "\n", - "# Pivot the DataFrame to get the desired table format\n", - "table = df.pivot(index='Prompt', columns='Model Name', values='Response')\n", - "table" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "zOxUM40PINDC" - }, - "source": [ - "# Load Test endpoint\n", - "\n", - "Run 100+ simultaneous queries across multiple providers to see when they fail + impact on latency" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZkQf_wbcIRQ9" - }, - "outputs": [], - "source": [ - "models=[\"gpt-3.5-turbo\", \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"claude-instant-1\"]\n", - "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n", - "prompt = \"Where does Paul Graham live?\"\n", - "final_prompt = context + prompt\n", - "result = load_test_model(models=models, prompt=final_prompt, num_calls=5)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "8vSNBFC06aXY" - }, - "source": [ - "## Visualize the data" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 552 - }, - "id": "SZfiKjLV3-n8", - "outputId": "00f7f589-b3da-43ed-e982-f9420f074b8d" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAIXCAYAAACy1HXAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABn5UlEQVR4nO3dd1QT2d8G8Cf0ojQBEUFRsSv2FXvvvSx2saNi7733ihXELotd7KuIir33sjZUsIuKVGmS+/7hy/yM6K7RYEZ4PufkaO5Mkm/IJHly594ZhRBCgIiIiEiGdLRdABEREdG3MKgQERGRbDGoEBERkWwxqBAREZFsMagQERGRbDGoEBERkWwxqBAREZFsMagQERGRbDGoEBERkWwxqBCR7Dk5OaFLly7aLkNtc+fORd68eaGrq4uSJUtquxyNO3bsGBQKBbZv367tUtSmUCgwadIktW8XGhoKhUKBdevWabwm+joGFVKxfPlyKBQKlC9fXtulyI6TkxMUCoV0MTU1xR9//IENGzZou7TfTuoX3PdcfleHDh3CiBEjUKlSJaxduxYzZszQdkmys27dOul1PnXqVJrlQgg4OjpCoVCgcePGWqiQ5EBP2wWQvPj7+8PJyQkXLlxASEgInJ2dtV2SrJQsWRJDhw4FALx8+RKrVq2Cu7s7EhMT0bNnTy1X9/soXLgw/Pz8VNpGjx6NLFmyYOzYsWnWv3fvHnR0fq/fVUePHoWOjg5Wr14NAwMDbZcja0ZGRti4cSMqV66s0n78+HE8e/YMhoaGWqqM5IBBhSSPHz/GmTNnEBAQAA8PD/j7+2PixIm/tAalUomkpCQYGRn90sf9Xjlz5kTHjh2l6126dEHevHmxcOFCBhU1ZM+eXeXvCACzZs2CtbV1mnYAv+UXVXh4OIyNjTUWUoQQSEhIgLGxsUbuT04aNmyIbdu2YfHixdDT+9/X0saNG1GmTBm8fftWi9WRtv1eP1EoXfn7+8PS0hKNGjVC69at4e/vLy1LTk6GlZUVunbtmuZ20dHRMDIywrBhw6S2xMRETJw4Ec7OzjA0NISjoyNGjBiBxMREldsqFAr069cP/v7+KFq0KAwNDXHw4EEAwLx581CxYkVky5YNxsbGKFOmzFf3hcfHx2PAgAGwtrZG1qxZ0bRpUzx//vyr+6CfP3+Obt26IXv27DA0NETRokWxZs2aH/6b2djYoFChQnj48KFKu1KphJeXF4oWLQojIyNkz54dHh4eeP/+vcp6ly5dQr169WBtbQ1jY2PkyZMH3bp1k5an7g+fN28eFi5ciNy5c8PY2BjVqlXDrVu30tRz9OhRVKlSBaamprCwsECzZs1w584dlXUmTZoEhUKBkJAQdOnSBRYWFjA3N0fXrl3x4cMHlXWDgoJQuXJlWFhYIEuWLChYsCDGjBmjss73vtY/48sxKqm7DE6dOoUBAwbAxsYGFhYW8PDwQFJSEiIjI9G5c2dYWlrC0tISI0aMwJcnitfUa/Q1CoUCa9euRVxcnLRrI3VMw8ePHzF16lTky5cPhoaGcHJywpgxY9L8vZycnNC4cWMEBgaibNmyMDY2xooVK/71cc+fP4/69evD3NwcJiYmqFatGk6fPq2yTlhYGPr27YuCBQvC2NgY2bJlw59//onQ0NA09xcZGYnBgwfDyckJhoaGcHBwQOfOndMEB6VSienTp8PBwQFGRkaoVasWQkJC/rXWz7Vr1w7v3r1DUFCQ1JaUlITt27ejffv2X71NXFwchg4dCkdHRxgaGqJgwYKYN29emtc5MTERgwcPho2NjfT58OzZs6/ep6Y/H0hDBNH/K1SokOjevbsQQogTJ04IAOLChQvS8m7dugkLCwuRmJiocrv169cLAOLixYtCCCFSUlJE3bp1hYmJiRg0aJBYsWKF6Nevn9DT0xPNmjVTuS0AUbhwYWFjYyMmT54sli1bJq5evSqEEMLBwUH07dtXLF26VCxYsED88ccfAoDYt2+fyn24ubkJAKJTp05i2bJlws3NTZQoUUIAEBMnTpTWe/XqlXBwcBCOjo5iypQpwtvbWzRt2lQAEAsXLvzPv0/u3LlFo0aNVNqSk5OFnZ2dyJ49u0p7jx49hJ6enujZs6fw8fERI0eOFKampqJcuXIiKSlJCCHE69evhaWlpShQoICYO3euWLlypRg7dqwoXLiwdD+PHz8WAETx4sWFk5OTmD17tpg8ebKwsrISNjY24tWrV9K6QUFBQk9PTxQoUEDMmTNHTJ48WVhbWwtLS0vx+PFjab2JEycKAKJUqVKiZcuWYvny5aJHjx4CgBgxYoS03q1bt4SBgYEoW7asWLRokfDx8RHDhg0TVatWldZR57X+L0WLFhXVqlX75t/e3d1dur527VoBQJQsWVLUr19fLFu2THTq1El6DpUrVxbt27cXy5cvF40bNxYAxPr169PlNfoaPz8/UaVKFWFoaCj8/PyEn5+fePjwoRBCCHd3dwFAtG7dWixbtkx07txZABDNmzdP85ydnZ2FpaWlGDVqlPDx8RHBwcHffMwjR44IAwMDUaFCBTF//nyxcOFC4eLiIgwMDMT58+el9bZt2yZKlCghJkyYIHx9fcWYMWOEpaWlyJ07t4iLi5PWi4mJEcWKFRO6urqiZ8+ewtvbW0ydOlWUK1dOeo8GBwdL21KZMmXEwoULxaRJk4SJiYn4448//vVv9PnrePHiRVGxYkXRqVMnadmuXbuEjo6OeP78eZr3nlKpFDVr1hQKhUL06NFDLF26VDRp0kQAEIMGDVJ5jI4dOwoAon379mLp0qWiZcuWwsXF5Yc/H1Lfk2vXrv3P50eawaBCQgghLl26JACIoKAgIcSnDwIHBwcxcOBAaZ3AwEABQOzdu1fltg0bNhR58+aVrvv5+QkdHR1x8uRJlfV8fHwEAHH69GmpDYDQ0dERt2/fTlPThw8fVK4nJSWJYsWKiZo1a0ptly9f/uqHU5cuXdJ8EHXv3l3kyJFDvH37VmXdtm3bCnNz8zSP96XcuXOLunXrijdv3og3b96ImzdvSl+Onp6e0nonT54UAIS/v7/K7Q8ePKjSvnPnTpWA9zWpH4rGxsbi2bNnUvv58+cFADF48GCprWTJksLW1la8e/dOart+/brQ0dERnTt3ltpSg0q3bt1UHqtFixYiW7Zs0vWFCxcKAOLNmzffrE+d1/q//EhQqVevnlAqlVJ7hQoVhEKhEL1795baPn78KBwcHFTuW5Ov0be4u7sLU1NTlbZr164JAKJHjx4q7cOGDRMAxNGjR1WeMwBx8ODB/3wspVIp8ufPn+bv8eHDB5EnTx5Rp04dlbYvnT17VgAQGzZskNomTJggAIiAgICvPp4Q/wsqhQsXVvkBs2jRIgFA3Lx581/r/jyoLF26VGTNmlWq788//xQ1atSQ/hafB5Vdu3YJAGLatGkq99e6dWuhUChESEiIEOJ/f+++ffuqrNe+ffsf/nxgUPn1uOuHAHza7ZM9e3bUqFEDwKeu6zZt2mDz5s1ISUkBANSsWRPW1tbYsmWLdLv3798jKCgIbdq0kdq2bduGwoULo1ChQnj79q10qVmzJgAgODhY5bGrVauGIkWKpKnp833x79+/R1RUFKpUqYIrV65I7am7ifr27aty2/79+6tcF0Jgx44daNKkCYQQKnXVq1cPUVFRKvf7LYcOHYKNjQ1sbGxQvHhx+Pn5oWvXrpg7d67K8zc3N0edOnVUHqdMmTLIkiWL9PwtLCwAAPv27UNycvK/Pm7z5s2RM2dO6foff/yB8uXL4++//wbwaWDvtWvX0KVLF1hZWUnrubi4oE6dOtJ6n+vdu7fK9SpVquDdu3eIjo5WqW/37t1QKpVfrUvd11rTunfvrjIzqHz58hBCoHv37lKbrq4uypYti0ePHqnUrenX6Hukvg5DhgxRaU8doL1//36V9jx58qBevXr/eb/Xrl3DgwcP0L59e7x79056PnFxcahVqxZOnDghvYafv6+Sk5Px7t07ODs7w8LCQuU9sGPHDpQoUQItWrRI83hfzsbq2rWrylicKlWqAIDK3/y/uLm5IT4+Hvv27UNMTAz27dv3zd0+f//9N3R1dTFgwACV9qFDh0IIgQMHDkjrAUiz3qBBg1Sua+rzgdJHhgkqJ06cQJMmTWBvbw+FQoFdu3al+2M+f/4cHTt2lMZQFC9eHJcuXUr3x9W0lJQUbN68GTVq1MDjx48REhKCkJAQlC9fHq9fv8aRI0cAAHp6emjVqhV2794t7U8PCAhAcnKySlB58OABbt++LX2hp14KFCgA4NMgw8/lyZPnq3Xt27cPrq6uMDIygpWVFWxsbODt7Y2oqChpnbCwMOjo6KS5jy9nK7158waRkZHw9fVNU1fquJsv6/qa8uXLIygoCAcPHsS8efNgYWGB9+/fq3xIP3jwAFFRUbC1tU3zWLGxsdLjVKtWDa1atcLkyZNhbW2NZs2aYe3atV8d25E/f/40bQUKFJDGFYSFhQEAChYsmGa9woULS19an8uVK5fKdUtLSwCQxmi0adMGlSpVQo8ePZA9e3a0bdsWW7duVQkt6r7WmvblczA3NwcAODo6pmn/fOxJerxG3yN1e/1y+7Szs4OFhYX0Oqb61nvjSw8ePAAAuLu7p3k+q1atQmJiovS+iY+Px4QJE6SxHdbW1rCxsUFkZKTKe+vhw4coVqzYdz3+f21L38PGxga1a9fGxo0bERAQgJSUFLRu3fqr64aFhcHe3h5Zs2ZVaS9cuLC0PPVfHR0d5MuXT2W9L98nmvp8oPSRYWb9xMXFoUSJEujWrRtatmyZ7o/3/v17VKpUCTVq1MCBAwdgY2ODBw8eSG/Q38nRo0fx8uVLbN68GZs3b06z3N/fH3Xr1gUAtG3bFitWrMCBAwfQvHlzbN26FYUKFUKJEiWk9ZVKJYoXL44FCxZ89fG+/BL52iyGkydPomnTpqhatSqWL1+OHDlyQF9fH2vXrsXGjRvVfo6pX64dO3aEu7v7V9dxcXH5z/uxtrZG7dq1AQD16tVDoUKF0LhxYyxatEj6laxUKmFra6syGPlzNjY2ACAdKOvcuXPYu3cvAgMD0a1bN8yfPx/nzp1DlixZ1H6e6tDV1f1qu/j/wYjGxsY4ceIEgoODsX//fhw8eBBbtmxBzZo1cejQIejq6qr9Wmvat57D19rFZ4Mstf0afe/xYb53hk/q9j137txvHlgutdb+/ftj7dq1GDRoECpUqABzc3MoFAq0bdv2mz1n/+W/tqXv1b59e/Ts2ROvXr1CgwYNpB6t9KapzwdKHxkmqDRo0AANGjT45vLExESMHTsWmzZtQmRkJIoVK4bZs2ejevXqP/R4s2fPhqOjI9auXSu1fe+vH7nx9/eHra0tli1blmZZQEAAdu7cCR8fHxgbG6Nq1arIkSMHtmzZgsqVK+Po0aNpjnuRL18+XL9+HbVq1frhA3bt2LEDRkZGCAwMVJma+vnfGwBy584NpVKJx48fq/Q6fDnjIHXEf0pKihQ0NKFRo0aoVq0aZsyYAQ8PD5iamiJfvnw4fPgwKlWq9F1fNK6urnB1dcX06dOxceNGdOjQAZs3b0aPHj2kdVJ/MX/u/v37cHJyAvDp7wB8Ot7Il+7evQtra2uYmpqq/fx0dHRQq1Yt1KpVCwsWLMCMGTMwduxYBAcHo3bt2hp5rbUhPV6j75G6vT548ED69Q8Ar1+/RmRkpPQ6qiu1x8DMzOw/t+/t27fD3d0d8+fPl9oSEhIQGRmZ5j6/NrMsPbVo0QIeHh44d+6cyi7mL+XOnRuHDx9GTEyMSq/K3bt3peWp/yqVSjx8+FClF+XL90l6fT6QZmSYXT//pV+/fjh79iw2b96MGzdu4M8//0T9+vW/+gXwPfbs2YOyZcvizz//hK2tLUqVKoWVK1dquOr0Fx8fj4CAADRu3BitW7dOc+nXrx9iYmKwZ88eAJ++uFq3bo29e/fCz88PHz9+VNntA3za1/z8+fOv/j3i4+PT7IL4Gl1dXSgUCml8DPBpqu6Xu/RS998vX75cpX3JkiVp7q9Vq1bYsWPHVz9837x58581fcvIkSPx7t076fm6ubkhJSUFU6dOTbPux48fpS+E9+/fp/nFmfpr+MtdC7t27cLz58+l6xcuXMD58+elcJ4jRw6ULFkS69evV/nCuXXrFg4dOoSGDRuq/bwiIiLStH1ZnyZea21Ij9foe6S+Dl5eXirtqT1SjRo1Uvs+AaBMmTLIly8f5s2bh9jY2DTLP9++dXV10zynJUuWqLzXAKBVq1a4fv06du7cmeb+1O0p+V5ZsmSBt7c3Jk2ahCZNmnxzvYYNGyIlJQVLly5VaV+4cCEUCoX0vkj9d/HixSrrffn3T8/PB/p5GaZH5d88efIEa9euxZMnT2Bvbw8AGDZsGA4ePPjDh7Z+9OgRvL29MWTIEIwZMwYXL17EgAEDYGBg8M2uQznas2cPYmJi0LRp068ud3V1hY2NDfz9/aVA0qZNGyxZsgQTJ05E8eLFVX4ZAkCnTp2wdetW9O7dG8HBwahUqRJSUlJw9+5dbN26VTouxL9p1KgRFixYgPr166N9+/YIDw/HsmXL4OzsjBs3bkjrlSlTBq1atYKXlxfevXsHV1dXHD9+HPfv3weg2sU+a9YsBAcHo3z58ujZsyeKFCmCiIgIXLlyBYcPH/7qF/P3aNCgAYoVK4YFCxbA09MT1apVg4eHB2bOnIlr166hbt260NfXx4MHD7Bt2zYsWrQIrVu3xvr167F8+XK0aNEC+fLlQ0xMDFauXAkzM7M0wcLZ2RmVK1dGnz59kJiYCC8vL2TLlg0jRoyQ1pk7dy4aNGiAChUqoHv37oiPj8eSJUtgbm7+Q+c0mTJlCk6cOIFGjRohd+7cCA8Px/Lly+Hg4CAdQVQTr7U2pMdr9D1KlCgBd3d3+Pr6IjIyEtWqVcOFCxewfv16NG/eXBrMri4dHR2sWrUKDRo0QNGiRdG1a1fkzJkTz58/R3BwMMzMzLB3714AQOPGjeHn5wdzc3MUKVIEZ8+exeHDh5EtWzaV+xw+fDi2b9+OP//8E926dUOZMmUQERGBPXv2wMfHR2V3ryZ9z+dnkyZNUKNGDYwdOxahoaEoUaIEDh06hN27d2PQoEFSD1PJkiXRrl07LF++HFFRUahYsSKOHDny1WO8pNfnA2mAVuYapTMAYufOndL1ffv2CQDC1NRU5aKnpyfc3NyEEELcuXNHAPjXy8iRI6X71NfXFxUqVFB53P79+wtXV9df8hw1pUmTJsLIyEjl+Alf6tKli9DX15em7SmVSuHo6PjV6YGpkpKSxOzZs0XRokWFoaGhsLS0FGXKlBGTJ08WUVFR0nr4Ymrv51avXi3y588vDA0NRaFChcTatWulqbWfi4uLE56ensLKykpkyZJFNG/eXNy7d08AELNmzVJZ9/Xr18LT01M4OjoKfX19YWdnJ2rVqiV8fX3/82/1teOopFq3bl2aKYu+vr6iTJkywtjYWGTNmlUUL15cjBgxQrx48UIIIcSVK1dEu3btRK5cuYShoaGwtbUVjRs3FpcuXZLuI3Uq5Ny5c8X8+fOFo6OjMDQ0FFWqVBHXr19PU8fhw4dFpUqVhLGxsTAzMxNNmjQR//zzj8o6qX/DL6cdp04VTT3mypEjR0SzZs2Evb29MDAwEPb29qJdu3bi/v37Krf73tf6v/zI9OQvpw1/67l9baqwEJp5jb7lW4+ZnJwsJk+eLPLkySP09fWFo6OjGD16tEhISEjznL+1vX3L1atXRcuWLUW2bNmEoaGhyJ07t3BzcxNHjhyR1nn//r3o2rWrsLa2FlmyZBH16tUTd+/eTfM3FkKId+/eiX79+omcOXMKAwMD4eDgINzd3aXPgtTpydu2bVO53fdO4f3W6/ilr/0tYmJixODBg4W9vb3Q19cX+fPnF3PnzlWZni2EEPHx8WLAgAEiW7ZswtTUVDRp0kQ8ffo0zfRkIb7v84HTk389hRDp1IenRQqFAjt37kTz5s0BAFu2bEGHDh1w+/btNIO+smTJAjs7OyQlJf3nVLps2bJJg+xy586NOnXqYNWqVdJyb29vTJs2TaWLnrTj2rVrKFWqFP766y906NBB2+X8sNDQUOTJkwdz585VOfIvEVFmkSl2/ZQqVQopKSkIDw+X5vd/ycDAAIUKFfru+6xUqVKaAVn379//4cFw9OPi4+PTDIj08vKCjo4OqlatqqWqiIhIEzJMUImNjVXZ7/j48WNcu3YNVlZWKFCgADp06IDOnTtj/vz5KFWqFN68eYMjR47AxcXlhwawDR48GBUrVsSMGTPg5uaGCxcuwNfXF76+vpp8WvQd5syZg8uXL6NGjRrQ09PDgQMHcODAAfTq1Svdp8cSEVE60/a+J01J3Vf65SV1n2tSUpKYMGGCcHJyEvr6+iJHjhyiRYsW4saNGz/8mHv37hXFihWTxlB8zzgH0rxDhw6JSpUqCUtLS6Gvry/y5csnJk2aJJKTk7Vd2k/7fIwKEVFmlCHHqBAREVHGkGmOo0JERES/HwYVIiIikq3fejCtUqnEixcvkDVr1t/q8N1ERESZmRACMTExsLe3h47Ov/eZ/NZB5cWLF5zVQURE9Jt6+vQpHBwc/nWd3zqopJ6M6unTpzAzM9NyNURERPQ9oqOj4ejoqHJSyW/5rYNK6u4eMzMzBhUiIqLfzPcM2+BgWiIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki09bRdARETy5TRqv7ZLIC0LndVIq4/PHhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki3ZBJVZs2ZBoVBg0KBB2i6FiIiIZEIWQeXixYtYsWIFXFxctF0KERERyYjWg0psbCw6dOiAlStXwtLSUtvlEBERkYxoPah4enqiUaNGqF279n+um5iYiOjoaJULERERZVx62nzwzZs348qVK7h48eJ3rT9z5kxMnjw5nasiIiIiudBaj8rTp08xcOBA+Pv7w8jI6LtuM3r0aERFRUmXp0+fpnOVREREpE1a61G5fPkywsPDUbp0aaktJSUFJ06cwNKlS5GYmAhdXV2V2xgaGsLQ0PBXl0pERERaorWgUqtWLdy8eVOlrWvXrihUqBBGjhyZJqQQERFR5qO1oJI1a1YUK1ZMpc3U1BTZsmVL005ERESZk9Zn/RARERF9i1Zn/Xzp2LFj2i6BiIiIZIQ9KkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWz8UVB4+fIhx48ahXbt2CA8PBwAcOHAAt2/f1mhxRERElLmpHVSOHz+O4sWL4/z58wgICEBsbCwA4Pr165g4caLGCyQiIqLMS+2gMmrUKEybNg1BQUEwMDCQ2mvWrIlz585ptDgiIiLK3NQOKjdv3kSLFi3StNva2uLt27caKYqIiIgI+IGgYmFhgZcvX6Zpv3r1KnLmzKmRooiIiIiAHwgqbdu2xciRI/Hq1SsoFAoolUqcPn0aw4YNQ+fOndOjRiIiIsqk1A4qM2bMQKFCheDo6IjY2FgUKVIEVatWRcWKFTFu3Lj0qJGIiIgyKT11b2BgYICVK1di/PjxuHXrFmJjY1GqVCnkz58/PeojIiKiTEztoJIqV65cyJUrlyZrISIiIlKhdlARQmD79u0IDg5GeHg4lEqlyvKAgACNFUdERESZm9pBZdCgQVixYgVq1KiB7NmzQ6FQpEddREREROoHFT8/PwQEBKBhw4bpUQ8RERGRRO1ZP+bm5sibN2961EJERESkQu2gMmnSJEyePBnx8fHpUQ8RERGRRO1dP25ubti0aRNsbW3h5OQEfX19leVXrlzRWHFERESUuakdVNzd3XH58mV07NiRg2mJiIgoXakdVPbv34/AwEBUrlw5PeohIiIikqg9RsXR0RFmZmbpUQsRERGRCrWDyvz58zFixAiEhoamQzlERERE/6P2rp+OHTviw4cPyJcvH0xMTNIMpo2IiNBYcUSZndOo/dougbQsdFYjbZdApFVqBxUvL690KIOIiIgorR+a9UNERET0K3xXUImOjpYG0EZHR//ruhxoS0RERJryXUHF0tISL1++hK2tLSwsLL567BQhBBQKBVJSUjReJBEREWVO3xVUjh49CisrKwBAcHBwuhZERERElOq7gkq1atWQN29eXLx4EdWqVUvvmoiIiIgAqHEcldDQUO7WISIiol9K7QO+aZK3tzdcXFxgZmYGMzMzVKhQAQcOHNBmSURERCQjak1PDgwMhLm5+b+u07Rp0+++PwcHB8yaNQv58+eHEALr169Hs2bNcPXqVRQtWlSd0oiIiCgDUiuo/NcxVNSd9dOkSROV69OnT4e3tzfOnTvHoEJERETqBZVXr17B1tY2XQpJSUnBtm3bEBcXhwoVKnx1ncTERCQmJkrX/+uYLkRERPR7++4xKl87doom3Lx5E1myZIGhoSF69+6NnTt3okiRIl9dd+bMmTA3N5cujo6O6VITERERycN3BxUhRLoUULBgQVy7dg3nz59Hnz594O7ujn/++eer644ePRpRUVHS5enTp+lSExEREcnDd+/6cXd3h7GxscYLMDAwgLOzMwCgTJkyuHjxIhYtWoQVK1akWdfQ0BCGhoYar4GIiIjk6buDytq1a9OzDolSqVQZh0JERESZl9pnT9ak0aNHo0GDBsiVKxdiYmKwceNGHDt2DIGBgdosi4iIiGRCq0ElPDwcnTt3xsuXL2Fubg4XFxcEBgaiTp062iyLiIiIZEKrQWX16tXafHgiIiKSuR8+hH5ISAgCAwMRHx8PIP1mBREREVHmpXZQeffuHWrXro0CBQqgYcOGePnyJQCge/fuGDp0qMYLJCIiosxL7aAyePBg6Onp4cmTJzAxMZHa27Rpg4MHD2q0OCIiIsrc1B6jcujQIQQGBsLBwUGlPX/+/AgLC9NYYURERERq96jExcWp9KSkioiI4MHYiIiISKPUDipVqlTBhg0bpOsKhQJKpRJz5sxBjRo1NFocERERZW5q7/qZM2cOatWqhUuXLiEpKQkjRozA7du3ERERgdOnT6dHjURERJRJqd2jUqxYMdy/fx+VK1dGs2bNEBcXh5YtW+Lq1avIly9fetRIREREmdQPHfDN3NwcY8eO1XQtRERERCrU7lE5ePAgTp06JV1ftmwZSpYsifbt2+P9+/caLY6IiIgyN7WDyvDhwxEdHQ0AuHnzJoYMGYKGDRvi8ePHGDJkiMYLJCIiosxL7V0/jx8/RpEiRQAAO3bsQJMmTTBjxgxcuXIFDRs21HiBRERElHmp3aNiYGCADx8+AAAOHz6MunXrAgCsrKyknhYiIiIiTVC7R6Vy5coYMmQIKlWqhAsXLmDLli0AgPv376c5Wi0RERHRz1C7R2Xp0qXQ09PD9u3b4e3tjZw5cwIADhw4gPr162u8QCIiIsq81O5RyZUrF/bt25emfeHChRopiIiIiCjVDx1HRalUIiQkBOHh4VAqlSrLqlatqpHCiIiIiNQOKufOnUP79u0RFhYGIYTKMoVCgZSUFI0VR0RERJmb2kGld+/eKFu2LPbv348cOXJAoVCkR11ERERE6geVBw8eYPv27XB2dk6PeoiIiIgkas/6KV++PEJCQtKjFiIiIiIVaveo9O/fH0OHDsWrV69QvHhx6Ovrqyx3cXHRWHFERESUuakdVFq1agUA6Natm9SmUCgghOBgWiIiItKoHzrXDxEREdGvoHZQyZ07d3rUQURERJTGDx3w7eHDh/Dy8sKdO3cAAEWKFMHAgQORL18+jRZHREREmZvaQSUwMBBNmzZFyZIlUalSJQDA6dOnUbRoUezduxd16tTReJHa4jRqv7ZLIC0LndVI2yUQEWVqageVUaNGYfDgwZg1a1aa9pEjR2aooEJERETapfZxVO7cuYPu3bunae/WrRv++ecfjRRFREREBPxAULGxscG1a9fStF+7dg22traaqImIiIgIwA/s+unZsyd69eqFR48eoWLFigA+jVGZPXs2hgwZovECiYiIKPNSO6iMHz8eWbNmxfz58zF69GgAgL29PSZNmoQBAwZovEAiIiLKvNQOKgqFAoMHD8bgwYMRExMDAMiaNavGCyMiIiL6oeOoAEB4eDju3bsHAChUqBBsbGw0VhQRERER8AODaWNiYtCpUyfY29ujWrVqqFatGuzt7dGxY0dERUWlR41ERESUSakdVHr06IHz589j//79iIyMRGRkJPbt24dLly7Bw8MjPWokIiKiTErtXT/79u1DYGAgKleuLLXVq1cPK1euRP369TVaHBEREWVuaveoZMuWDebm5mnazc3NYWlpqZGiiIiIiIAfCCrjxo3DkCFD8OrVK6nt1atXGD58OMaPH6/R4oiIiChzU3vXj7e3N0JCQpArVy7kypULAPDkyRMYGhrizZs3WLFihbTulStXNFcpERERZTpqB5XmzZunQxlEREREaakdVCZOnJgedRARERGlofYYladPn+LZs2fS9QsXLmDQoEHw9fXVaGFEREREageV9u3bIzg4GMCnQbS1a9fGhQsXMHbsWEyZMkXjBRIREVHmpXZQuXXrFv744w8AwNatW1G8eHGcOXMG/v7+WLdunabrIyIiokxM7aCSnJwMQ0NDAMDhw4fRtGlTAJ/O9/Py5UvNVkdERESZmtpBpWjRovDx8cHJkycRFBQkHY32xYsXyJYtm8YLJCIiosxL7aAye/ZsrFixAtWrV0e7du1QokQJAMCePXukXUJEREREmqD29OTq1avj7du3iI6OVjlkfq9evWBiYqLR4oiIiChzU7tHBQCEELh8+TJWrFiBmJgYAICBgQGDChEREWmU2j0qYWFhqF+/Pp48eYLExETUqVMHWbNmxezZs5GYmAgfH5/0qJOIiIgyIbV7VAYOHIiyZcvi/fv3MDY2ltpbtGiBI0eOaLQ4IiIiytzU7lE5efIkzpw5AwMDA5V2JycnPH/+XGOFEREREando6JUKpGSkpKm/dmzZ8iaNatGiiIiIiICfiCo1K1bF15eXtJ1hUKB2NhYTJw4EQ0bNtRkbURERJTJqb3rZ/78+ahXrx6KFCmChIQEtG/fHg8ePIC1tTU2bdqUHjUSERFRJqV2UHFwcMD169exZcsWXL9+HbGxsejevTs6dOigMriWiIiI6GepHVQAQE9PDx06dECHDh2ktpcvX2L48OFYunSpxoojIiKizE2toHL79m0EBwfDwMAAbm5usLCwwNu3bzF9+nT4+Pggb9686VUnERERZULfPZh2z549KFWqFAYMGIDevXujbNmyCA4ORuHChXHnzh3s3LkTt2/fTs9aiYiIKJP57qAybdo0eHp6Ijo6GgsWLMCjR48wYMAA/P333zh48KB0FmUiIiIiTfnuoHLv3j14enoiS5Ys6N+/P3R0dLBw4UKUK1cuPesjIiKiTOy7g0pMTAzMzMwAALq6ujA2NuaYFCIiIkpXag2mDQwMhLm5OYBPR6g9cuQIbt26pbJO06ZNNVcdERERZWpqBRV3d3eV6x4eHirXFQrFVw+vT0RERPQjvjuoKJXK9KyDiIiIKA21z/VDRERE9KtoNajMnDkT5cqVQ9asWWFra4vmzZvj3r172iyJiIiIZESrQeX48ePw9PTEuXPnEBQUhOTkZNStWxdxcXHaLIuIiIhk4ofO9aMpBw8eVLm+bt062Nra4vLly6hataqWqiIiIiK50GpQ+VJUVBQAwMrK6qvLExMTkZiYKF2Pjo7+JXURERGRdvzQrp/IyEisWrUKo0ePRkREBADgypUreP78+Q8XolQqMWjQIFSqVAnFihX76jozZ86Eubm5dHF0dPzhxyMiIiL5Uzuo3LhxAwUKFMDs2bMxb948REZGAgACAgIwevToHy7E09MTt27dwubNm7+5zujRoxEVFSVdnj59+sOPR0RERPKndlAZMmQIunTpggcPHsDIyEhqb9iwIU6cOPFDRfTr1w/79u1DcHAwHBwcvrmeoaEhzMzMVC5ERESUcak9RuXixYtYsWJFmvacOXPi1atXat2XEAL9+/fHzp07cezYMeTJk0fdcoiIiCgDUzuoGBoafnUQ6/3792FjY6PWfXl6emLjxo3YvXs3smbNKgUdc3NzGBsbq1saERERZTBq7/pp2rQppkyZguTkZACfzu/z5MkTjBw5Eq1atVLrvry9vREVFYXq1asjR44c0mXLli3qlkVEREQZkNpBZf78+YiNjYWtrS3i4+NRrVo1ODs7I2vWrJg+fbpa9yWE+OqlS5cu6pZFREREGZDau37Mzc0RFBSEU6dO4caNG4iNjUXp0qVRu3bt9KiPiIiIMrEfPuBb5cqVUblyZU3WQkRERKRC7aCyePHir7YrFAoYGRnB2dkZVatWha6u7k8XR0RERJmb2kFl4cKFePPmDT58+ABLS0sAwPv372FiYoIsWbIgPDwcefPmRXBwMI8cS0RERD9F7cG0M2bMQLly5fDgwQO8e/cO7969w/3791G+fHksWrQIT548gZ2dHQYPHpwe9RIREVEmonaPyrhx47Bjxw7ky5dPanN2dsa8efPQqlUrPHr0CHPmzFF7qjIRERHRl9TuUXn58iU+fvyYpv3jx4/SAdvs7e0RExPz89URERFRpqZ2UKlRowY8PDxw9epVqe3q1avo06cPatasCQC4efMmD4dPREREP03toLJ69WpYWVmhTJkyMDQ0hKGhIcqWLQsrKyusXr0aAJAlSxbMnz9f48USERFR5qL2GBU7OzsEBQXh7t27uH//PgCgYMGCKFiwoLROjRo1NFchERERZVo/fMC3QoUKoVChQpqshYiIiEjFDwWVZ8+eYc+ePXjy5AmSkpJUli1YsEAjhRERERGpHVSOHDmCpk2bIm/evLh79y6KFSuG0NBQCCFQunTp9KiRiIiIMim1B9OOHj0aw4YNw82bN2FkZIQdO3bg6dOnqFatGv7888/0qJGIiIgyKbWDyp07d9C5c2cAgJ6eHuLj45ElSxZMmTIFs2fP1niBRERElHmpHVRMTU2lcSk5cuTAw4cPpWVv377VXGVERESU6ak9RsXV1RWnTp1C4cKF0bBhQwwdOhQ3b95EQEAAXF1d06NGIiIiyqTUDioLFixAbGwsAGDy5MmIjY3Fli1bkD9/fs74ISIiIo1SK6ikpKTg2bNncHFxAfBpN5CPj0+6FEZERESk1hgVXV1d1K1bF+/fv0+veoiIiIgkag+mLVasGB49epQetRARERGpUDuoTJs2DcOGDcO+ffvw8uVLREdHq1yIiIiINEXtwbQNGzYEADRt2hQKhUJqF0JAoVAgJSVFc9URERFRpqZ2UAkODk6POoiIiIjSUDuoVKtWLT3qICIiIkpD7TEqAHDy5El07NgRFStWxPPnzwEAfn5+OHXqlEaLIyIiosxN7aCyY8cO1KtXD8bGxrhy5QoSExMBAFFRUZgxY4bGCyQiIqLM64dm/fj4+GDlypXQ19eX2itVqoQrV65otDgiIiLK3NQOKvfu3UPVqlXTtJubmyMyMlITNREREREB+IGgYmdnh5CQkDTtp06dQt68eTVSFBERERHwA0GlZ8+eGDhwIM6fPw+FQoEXL17A398fw4YNQ58+fdKjRiIiIsqk1J6ePGrUKCiVStSqVQsfPnxA1apVYWhoiGHDhqF///7pUSMRERFlUmoHFYVCgbFjx2L48OEICQlBbGwsihQpgixZsqRHfURERJSJqb3r56+//sKHDx9gYGCAIkWK4I8//mBIISIionShdlAZPHgwbG1t0b59e/z99988tw8RERGlG7WDysuXL7F582YoFAq4ubkhR44c8PT0xJkzZ9KjPiIiIsrE1A4qenp6aNy4Mfz9/REeHo6FCxciNDQUNWrUQL58+dKjRiIiIsqk1B5M+zkTExPUq1cP79+/R1hYGO7cuaOpuoiIiIh+7KSEHz58gL+/Pxo2bIicOXPCy8sLLVq0wO3btzVdHxEREWViaveotG3bFvv27YOJiQnc3Nwwfvx4VKhQIT1qIyIiokxO7aCiq6uLrVu3ol69etDV1VVZduvWLRQrVkxjxREREVHmpnZQ8ff3V7keExODTZs2YdWqVbh8+TKnKxMREZHG/NAYFQA4ceIE3N3dkSNHDsybNw81a9bEuXPnNFkbERERZXJq9ai8evUK69atw+rVqxEdHQ03NzckJiZi165dKFKkSHrVSERERJnUd/eoNGnSBAULFsSNGzfg5eWFFy9eYMmSJelZGxEREWVy392jcuDAAQwYMAB9+vRB/vz507MmIiIiIgBq9KicOnUKMTExKFOmDMqXL4+lS5fi7du36VkbERERZXLfHVRcXV2xcuVKvHz5Eh4eHti8eTPs7e2hVCoRFBSEmJiY9KyTiIiIMiG1Z/2YmpqiW7duOHXqFG7evImhQ4di1qxZsLW1RdOmTdOjRiIiIsqkfnh6MgAULFgQc+bMwbNnz7Bp0yZN1UREREQE4CeDSipdXV00b94ce/bs0cTdEREREQHQUFAhIiIiSg8MKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbWg0qJ06cQJMmTWBvbw+FQoFdu3ZpsxwiIiKSGa0Glbi4OJQoUQLLli3TZhlEREQkU3rafPAGDRqgQYMG2iyBiIiIZEyrQUVdiYmJSExMlK5HR0drsRoiIiJKb7/VYNqZM2fC3Nxcujg6Omq7JCIiIkpHv1VQGT16NKKioqTL06dPtV0SERERpaPfatePoaEhDA0NtV0GERER/SK/VY8KERERZS5a7VGJjY1FSEiIdP3x48e4du0arKyskCtXLi1WRkRERHKg1aBy6dIl1KhRQ7o+ZMgQAIC7uzvWrVunpaqIiIhILrQaVKpXrw4hhDZLICIiIhnjGBUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLVkElWXLlsHJyQlGRkYoX748Lly4oO2SiIiISAa0HlS2bNmCIUOGYOLEibhy5QpKlCiBevXqITw8XNulERERkZZpPagsWLAAPXv2RNeuXVGkSBH4+PjAxMQEa9as0XZpREREpGVaDSpJSUm4fPkyateuLbXp6Oigdu3aOHv2rBYrIyIiIjnQ0+aDv337FikpKciePbtKe/bs2XH37t006ycmJiIxMVG6HhUVBQCIjo5Ol/qUiR/S5X7p95Fe29b34jZI3AZJ29JjG0y9TyHEf66r1aCirpkzZ2Ly5Mlp2h0dHbVQDWUG5l7aroAyO26DpG3puQ3GxMTA3Nz8X9fRalCxtraGrq4uXr9+rdL++vVr2NnZpVl/9OjRGDJkiHRdqVQiIiIC2bJlg0KhSPd6M5Po6Gg4Ojri6dOnMDMz03Y5lAlxGyRt4zaYfoQQiImJgb29/X+uq9WgYmBggDJlyuDIkSNo3rw5gE/h48iRI+jXr1+a9Q0NDWFoaKjSZmFh8QsqzbzMzMz4BiWt4jZI2sZtMH38V09KKq3v+hkyZAjc3d1RtmxZ/PHHH/Dy8kJcXBy6du2q7dKIiIhIy7QeVNq0aYM3b95gwoQJePXqFUqWLImDBw+mGWBLREREmY/WgwoA9OvX76u7ekh7DA0NMXHixDS72oh+FW6DpG3cBuVBIb5nbhARERGRFmj9yLRERERE38KgQkRERLLFoEJERESyxaBCREREssWgQkRERLLFoEJERESyxaBCREREssWgQkRERLLFoEJERESyxaBCvyWlUqntEoiI6BdgUKHfko7Op0337du3AACeCYJ+tS/DMrdB0oYvt8OM+COOQYV+W4sWLULz5s3x8OFDKBQKbZdDmYyOjg6ioqIQGBgIANwGSSt0dHQQGRmJuXPn4v3799KPuIwk4z0jyrC+/MWqr68PY2NjGBgYaKkiysyUSiXmz58PDw8P7Nu3T9vlUCZ26NAhLFiwAEuXLtV2KemCZ0+m3050dDTMzMwAAFFRUTA3N9dyRZRZKJVKlV+sd+7cwerVqzF79mzo6upqsTLKTFJSUlS2t+TkZGzZsgXt2rXLkNshgwr9VgYPHoyUlBSMHj0aOXLk0HY5lAlFRkYiMjISjo6OKl8KX355EP2ML0Pxl969e4fTp0+jYsWKsLa2ltoz4nbIXT8ka1/maAcHB2zYsCHDvRHp9yCEwKhRo1C+fHmEhoaqLOM2ST/j5cuXePHiBd68eQPg09iTf+tH2Lp1K5o3b47jx4+rtGfE7ZA9KiQbqb8EhBBQKBTf/EXx/v17WFpaaqFCymj+61fr19YJCwvDuHHjsG7dugz5pUC/3tq1a7Fs2TI8ffoU+fLlQ+XKlTFnzhyVdb7WU+Ll5YV+/fpBT0/vV5b7yzGokFakhhHg0xtQCAE9PT08f/4cO3fuRNeuXWFqagrg0+4eS0tLTJgwIc1tiX7U5wHk6NGjePLkCZydnZE3b17Y29urrBMVFQWlUpkmIGfEbnb6tfbt2wc3NzcsX74cJiYmePToEebMmYOKFSti/fr1yJYtm/SZ9/btW4SEhMDV1VXlPj5+/Jihwwp3/dAvkZqHo6OjER8fD4VCgUOHDiEkJAS6urrQ09NDWFgYSpUqhRcvXkghJS4uDvr6+li4cCEiIiIYUkgjhBBSSBk1ahS6dOmCefPmoVevXhg2bBguXrwI4FP3e2JiIiZMmIDSpUvj3bt3KvfDkEI/6+LFi2jUqBG6dOkCNzc3jBgxAoGBgbhx4wY6dOgA4NPU9+TkZPj5+aFixYo4deqUyn1k5JACMKjQL/Tq1SsUL14cx48fx8aNG1G/fn38888/AD7tzilatChatGiB6dOnS7cxNTXFiBEj8ODBA1hZWTGkkEakbkfz5s3DX3/9hU2bNuHWrVto2bIl9u7di3HjxuHs2bMAAAMDA5QqVQq1atWChYWFFqumjOjx48d4+fKlSlu5cuWwZ88eXL58GT179gTw6XAMjRs3xvTp09P0qGR4gugX6tq1qzAzMxM6Ojpi5cqVUntSUpLYsmWLSElJkdqUSqU2SqRM4vXr16Jly5ZizZo1Qggh9uzZI8zMzETv3r1FqVKlRK1atcS5c+eEEKrb4sePH7VSL2VMgYGBInv27GLz5s1SW+r25u/vL5ydncXFixfT3C45OfmX1aht7FGhXyL1sM6enp6IiYmBgYEB7OzskJCQAODTrwU3NzeVQYvsPaH0ZGtrixEjRqB+/fq4evUqPD09MW3aNHh7e6NVq1Y4d+4cPD09cfnyZZVtkbt7SJMKFy6M6tWrw8/PD0eOHAHwv8++kiVLIjw8XDpVyOcy+u6ezzGo0C+RGkAcHR1x6tQpuLu7o23btti9ezfi4+PTrJ8Rz1dB2vOt7alUqVLIkSMHDhw4ABcXF/Tq1QsAYGVlBVdXVzRp0gSlSpX6laVSJuPo6IjevXsjMjISCxcuxJ49e6RlOXLkQJ48ebRYnTxknkhGWiH+f/Dry5cvkZycjFy5csHW1hYVK1ZEQkICunfvjnXr1qFx48YwMjKCj48PateuDWdnZ22XThmE+Gzg7KpVqxAeHg4DAwMMGzZMOv1CYmIinj9/jtDQUBQsWBCHDh1C06ZN0b9//3+dKk/0M1JnjVWvXh3Lly/HmDFjMHLkSAQGBsLFxQVbt26FQqFAnTp1tF2qVnF6MqW7gIAATJo0Ca9fv0ajRo3QokULNGnSBADQtWtX7Ny5E0OHDsXr16/h7e2NmzdvokiRIlqumjKaiRMnwsvLC+XKlcOFCxdQvnx5+Pn5wc7ODnv37sW0adPw/v176OvrQwiBGzduQE9PjzPNKF2kblcBAQFYvnw5Dh06hLt37yI4OBhLly6Fo6MjLCws4O/vD319/Uw9FZ5BhdLV7du3Ua9ePQwePBgmJibYtGkTDA0N4e7ujo4dOwIABg4ciCtXriAxMRG+vr4oWbKkdoumDOHzXpCPHz/C3d0d/fv3R6lSpRAaGopGjRrBzs4OO3fuhI2NDfbv34+QkBDExsZi5MiR0NPTy9RfDqQZqYFEfHHsKF1dXQQEBKBz585YsGCBtNsR+LS96ujoqGy/mWlMypcYVCjd3L17F9u2bUN8fDxmzJgBALh58yYmTJiA6OhodO3aVQorr169gqmpKbJmzarNkimD+Dyk3LlzB9HR0VixYgUmTJgAJycnAJ+mhdapUwfZs2fHrl27YGNjo3IfDCn0sz7fDt++fQuFQoFs2bIB+PSZV7p0aUyYMAG9e/eWbvNlDx579BhUKB0IIfD+/Xs0btwY//zzD5o0aQI/Pz9p+Y0bNzBhwgTEx8ejbdu26Nq1qxarpYxs+PDhUtf569evERAQgAYNGkgf/I8fP0aDBg0ghMDp06dVTu5G9DM+DxhTp07Frl27EB0dDWtra0yfPh01a9bE8+fPkTNnTi1XKn8cHUYap1AoYGVlhZkzZ6Jo0aK4cuUKgoKCpOUuLi6YOnUqkpOTpTcvkSZ8Prtn3759OHjwIBYvXozly5cjT548GDt2LK5fvy4dKTlPnjzYt28fSpYsyfNHkUalhpQpU6Zg0aJF0vR3a2trdOjQAevXr0/Ti0dfxx4V0ohvdU8eP34cY8aMgZ2dHTw9PVGzZk1p2e3bt2Fubg4HB4dfWSplAgEBAThz5gyyZcuG0aNHAwBiY2NRunRpmJmZYdWqVShRokSabZa7e0iT3r17h7p168LT0xPdunWT2nv16oW9e/ciODgYhQoV4u6d/8AeFfppqW+yM2fOYMGCBRg/fjxOnz6N5ORkVKtWDVOmTMGrV6+wdOlSHDt2TLpd0aJFGVJI4+Lj4zF+/HgsWLAAt2/fltqzZMmCK1euICYmBh4eHtL5fD7HkEKa9PHjR7x9+1bqrUs9wKWvry/s7e2xcOFCADy45X9hUKGf8vkUuwYNGuD06dPYs2cPxowZg+nTpyMpKQm1atXClClT8O7dO0ydOhUnT57UdtmUgRkbG+PkyZOoXbs2Ll++jD179iAlJQXA/8LK3bt3sWLFCi1XShnJ13ZOZM+eHXZ2dlizZg0AwMjICElJSQAAZ2dnBpTvxKBCPyW1J2XAgAFYsGABduzYgW3btuHy5cvYsmULxo0bJ4WVUaNGQV9fn0daJI35fEyKEEL6srCyssLGjRthaWmJuXPnIjAwUFpmamqKV69ewdfXVys1U8ajVCql0PHixQuEh4fjw4cPAIBJkybh7t270sye1IMMPnv2jCe5/E4co0I/JPWNqVAosHz5cly7dg2+vr54/PgxateujcqVK8PMzAzbtm2Dh4cHxowZA0NDQ3z48AEmJibaLp8ygM+nfi5ZsgTXr1/Ho0ePMGjQIJQuXRoODg548+YNmjVrBl1dXYwZMwb16tVTOcIsx6TQz/D394erqyvy5csHABg9ejQCAwMRFhaG2rVro2nTpujQoQNWrlyJqVOnIlu2bChWrBgePnyIyMhI6aCC9O8YVOi7pH4pfB40rl27hpIlSyI6OhpPnz6Fs7Mz6tevjzx58mDNmjWIioqSjjDbpUsXTJ8+nYPG6Kd9uQ2NHj0aq1evRq9evfDs2TOcPXsWzZo1Q69eveDs7Iw3b96gZcuWePPmDdatWwdXV1ctVk8ZxYEDB9C4cWOMHDkSgwYNwoEDBzBixAh4eXnh3bt3uHLlCgIDAzF+/Hj07t0bN2/ehJeXF3R0dGBpaYkZM2bwoILfK13PzUwZyqNHj0S7du3EP//8I7Zu3SoUCoW4cOGCdErymzdvikKFConz588LIYR4+PChaNy4sRgzZox48uSJNkunDCYlJUUIIYSfn5/IkyePuHz5shBCiJMnTwqFQiHy588vBg4cKB49eiSEEOLly5eiV69e4uPHj1qrmTKepUuXCgcHBzF16lTRr18/sXLlSmnZ06dPxZQpU4STk5M4ePDgV2+fnJz8q0r9rbHPib5bQkICTp48iS5duuDatWtYu3YtypUrJ+0GEkLg48ePOHv2LIoWLYoNGzYAAIYNG8ZjVNBP69SpE2xsbLBgwQLo6OggOTkZBgYG6N27N0qXLo1du3aha9euWLVqFV69eoVp06ZBR0cHPXv2ROHChaXBs/wFSz8rKSkJBgYG8PT0hImJCUaPHo2YmBhMmzZNWsfBwQGdO3fGoUOHcOnSJdSrVy/NyS252+c7aTsp0e8h9Resj4+P0NHRESVKlBBXr15VWScqKkp06dJF5MuXTzg5OQkbGxvply7Rz4iKihKTJ08WVlZWYtKkSVL78+fPxevXr8XLly9F2bJlxfz586X17e3tRY4cOcSiRYuEEELq+SPSlJkzZ4rw8HDh7+8vTExMRMOGDcX9+/dV1mnTpo1o2bKllirMGDjrh/6TEAI6OjoQQsDe3h7z58/Hx48fMW7cOJw6dUpaz8zMDPPmzcPy5csxceJEnD9/HqVLl9Zi5ZQRxMTEwMzMDH369MG4cePg5eWFiRMnAgDs7e1ha2uLly9f4v3799L4k+fPn6Nu3bqYMGECPD09AfBYFfTzxGdDOtevX4+pU6fiwYMHaN++PRYuXIgrV67Ax8cH9+7dAwBER0fj8ePHyJUrl7ZKzhDY70T/Svz/wMWjR4/i+PHjGDRoEJo0aYLatWvDzc0Ns2bNwpgxY1CxYkUAn046WLduXS1XTRnFiBEjsGLFCjx8+BA2Njbo2LEjhBCYOnUqAGDy5MkAPoUZXV1dnD59GkIIzJo1CyYmJtKUUO7uIU1IDbtHjhzB1atX4evrK3329erVC8nJyZg8eTIOHjyI0qVLIy4uDklJSZgzZ442y/79abM7h+Qttat8+/btwtzcXIwePVpcvHhRWn7jxg1RpEgR0bhxY/HXX3+JSZMmCYVCIZ4+fcpudtKI69evi6pVq4qCBQuKN2/eCCGECA8PF/PnzxcWFhZiwoQJ0rr9+vUT+fLlEw4ODsLV1VUkJSUJIbjLhzTr2LFjonjx4iJbtmxi165dQgghEhMTpeWrV68WWbJkEaVLlxYbNmyQBnBz4OyP4/Rk+lcXLlxA/fr1MXv2bPTs2VNqj46OhpmZGe7cuYOePXsiPj4eUVFR2Lp1K3f3kEacPXsWb968QZEiRdCmTRvExsZKZzh+8+YN/Pz8MHXqVOlkb8CnKfMKhQLFixeHjo4OPn78yAGL9FPEF9PhY2NjMXfuXPj6+qJ8+fLYtGkTjI2NkZycDH19fQDAggULcObMGWzbtg0KhYI9ej+JQYX+1dKlS7Fz504cOXIEUVFROHr0KP766y/cuXMHw4YNQ7du3RAeHo6oqCiYm5vD1tZW2yVTBtG5c2e8ePEChw8fRmhoKFq3bo2YmJg0YWXatGno168fpkyZonJ7fjmQJi1btgwODg5o1qwZ4uPjMW/ePOzcuRPVq1fHjBkzYGRkpBJWUgPOl0GH1MfBtPSv7OzscPnyZcycOROtW7fG2rVrYWRkhEaNGqFHjx64f/8+bG1tkT9/foYU0qhly5bh2bNnWLp0KZycnLBp0yaYm5ujUqVKePv2LWxsbNCpUydMmDAB06ZNw+rVq1Vuz5BCmvLmzRscPXoUffv2xcGDB2FsbIwhQ4agcePGOHPmDMaOHYuEhATo6+vj48ePAMCQokHsUSFJ6psqNjYWWbJkAQC8fv0aS5YswdatW1GzZk106dIFf/zxB16/fo2mTZti3bp1KFq0qJYrp4wmtTdk8eLFuHr1KhYsWABLS0vcvXsXnTt3RlRUlNSz8urVKxw/fhytWrXibh7SiC+PdwIA169fx+LFi3H48GH4+PigQYMGiIuLw5w5c3D48GEULlwYy5cvl87lQ5rDHhWSKBQK7N+/H+3atUP16tWxbt066OnpYdq0aTh//jx8fHzg6uoKHR0dLFmyBHFxcexFoXSR2htSvXp1nDhxAvv37wcAFCxYEH5+frC0tETVqlXx+vVr2NnZoU2bNtDT05N+zRL9jNSQ8urVK6mtRIkSGDhwIGrUqIHevXvj4MGDMDU1xYgRI/DHH39AR0dH2u1DGqalQbwkQ6dPnxZGRkZi+PDhon79+sLFxUV4eHiIkJAQaZ3g4GDRq1cvYWVlleaAb0Q/KvWAgl/j4+MjChQoIO7duye13bt3Tzg5OYm2bdv+ivIok/h8O9y8ebPImzevykxHIYS4du2aaNasmciVK5c4duyYEEKI+Ph4aXbZv23L9GPYo0IAgLCwMAQFBWH69OmYM2cODhw4gF69euHGjRuYOXMmHj16hLi4OJw9exbh4eE4fvw4SpYsqe2yKQP4vJv9woULOHPmDI4fPy4tb9q0KcqXL4/g4GCprUCBAjhx4gT++uuvX14vZUyJiYnSdpiUlIR8+fKhUKFC8PT0xOXLl6X1SpQogebNm+Pp06eoW7cuzpw5AyMjI2lMype7jOjn8S+aCS1duhR///23dP3evXto06YN1qxZAyMjI6nd09MTHTp0wO3btzFnzhxERkZi+PDhWL9+PYoVK6aN0imD+fyDfcyYMejSpQu6desGd3d3tGnTBtHR0ciRI4e0/z85OVm6raOjI3R1dZGSkqKt8imDOHDgAPz8/AAAPXv2RM2aNVG2bFkMHToUdnZ28PDwwKVLl6T1c+XKhbZt22L+/PkoX7681M6Bs+lE21069Gs9fvxYtG/fXjx48EClfdSoUcLW1la0bNlSOrBWKm9vb1GwYEExYMAAHrSI0sW8efNEtmzZxPnz50VKSoqYMWOGUCgU4tSpU9I6lSpVEh4eHlqskjKqdu3aCScnJ1GvXj1hbW0trl+/Li07evSoaN68uShWrJg4cOCAePz4sWjevLkYOnSotA7Pyp2+GFQyobi4OCGEEOfOnRPbt2+X2idMmCCKFy8uxo0bJ16/fq1ym5UrV4rHjx//yjIpk1AqlcLd3V34+voKIYTYsWOHsLCwED4+PkIIIWJiYoQQQhw4cEA0bdpU3LhxQ2u1UsZVsmRJoVAoVE56merkyZOiU6dOQqFQiAIFCggXFxfpRxuPfJz+OJcvEzI2NkZkZCRmzpyJ58+fQ1dXF82bN8fkyZORnJyM/fv3QwiBgQMHwsbGBgDQo0cPLVdNGVVCQgLOnz+P6tWr49ixY3B3d8fcuXPh4eGBjx8/Ys6cOahQoQJcXV0xZcoUXLhwAcWLF9d22ZRBJCUlISEhAc7OzsiVKxe2bNmCnDlzom3bttJhGipXrozy5cujZ8+eSE5ORrVq1aCrq8sjH/8iHKOSCSkUClhYWGDo0KHIkycPvLy8EBAQAACYMWMG6tevj6CgIMyYMQNv377VcrWUkdy4cQPPnj0DAAwePBjHjx+HsbEx2rdvj7/++gsNGzbEwoULpZMJvn//HpcuXcK9e/dgaWkJPz8/5M6dW5tPgTIYAwMDmJmZYdu2bdi9ezfKlSuHOXPmYPPmzYiJiZHWS0hIQJUqVVCzZk1pbBRDyq/BoJIJiU+7/FClShUMHjwYlpaWWLx4sUpYcXV1xdWrV1VOa070o4QQuH//PmrUqIE1a9agd+/eWLRoESwtLQEArq6uCAsLQ/ny5VGhQgUAwIsXL9ClSxdERkaiX79+AIB8+fKhdu3aWnselPEIIaBUKqXr69evR8WKFbFw4UJs2LABT548Qc2aNfHnn39K6wM88vGvxCPTZkKpR/2MioqCiYkJbty4genTp+P9+/cYOHAgmjdvDuDTYaNTd/0QacLKlSsxYsQIJCQkYPfu3ahbt650ROQtW7ZgypQpEEJAT08PxsbGUCqVOHPmDPT19XnuHvppERERsLKyUmlL3f62bduGoKAg+Pr6AgB69eqFY8eOISUlBVZWVjh9+jSPOqsl7FHJZD5+/AhdXV2EhoaievXqOHToEMqUKYNhw4bBxsYGkydPxr59+wCAIYU0JvUXq6OjIwwNDWFmZoZz584hNDRUmtLZpk0bbNiwAVOmTIGbmxtGjhyJc+fOSedPYUihn7Fo0SKUK1dOZXcOACmkdOnSBSVKlJDafX19sWLFCixZsgTnzp2DgYEBj3ysLdoZw0u/wrdGo4eEhIjs2bOLHj16qEyrO3bsmOjUqZMIDQ39VSVSBvflNpiUlCTi4+OFt7e3yJkzpxgzZsx/bm+c+kk/a8WKFcLQ0FBs3LgxzbInT56I4sWLi6VLl0ptX9vmuB1qD3f9ZFDi/7szz549izt37iAkJASdO3dGjhw5sH79ely6dAnr169Pc4bPhIQElYO+Ef2oz484GxERgZiYGJWBsF5eXpg3bx66d++Orl27wsnJCU2aNMHYsWPh6uqqrbIpg1m5ciX69+8PPz8//Pnnn4iMjERcXBwSEhJga2uLrFmz4sGDB8ifP7+2S6VvYFDJwHbs2IFevXpJJ2978+YN2rRpg5EjRyJr1qzaLo8ysM9DypQpU3Do0CHcunULbm5uaNGiBRo0aADgU1jx8vJCsWLF8O7dOzx58gShoaE8uRtpxKNHj+Ds7Aw3Nzds3rwZt27dQt++ffHmzRuEhYWhRo0a6NOnDxo3bqztUulfcG5VBnXr1i0MHjwY8+fPR5cuXRAdHQ0LCwsYGxszpFC6Sw0pEyZMgK+vL+bOnQsnJyf07t0bDx48QGRkJNq1a4dBgwbB2toa169fR0JCAk6ePCmdBZlTP+ln2djYYPbs2ZgwYQKGDRuGQ4cOoUqVKmjWrBmio6Oxfft2jBs3DtbW1uzFkzNt7ncizTh69Kh4+PBhmrYKFSoIIYS4c+eOyJ07t+jRo4e0/OHDh9znSunq6NGjomjRouLEiRNCCCHOnDkjDAwMRJEiRUT58uXFtm3bpHU/PzUDT9NAmpSQkCDmzZsndHR0RLdu3URSUpK07NKlS6JgwYJi2bJlWqyQ/gtn/fzGhBC4evUqGjRoAG9vb4SFhUnLnj9/DiEEYmNjUb9+fdStWxcrVqwAAAQFBcHb2xvv37/XVumUAYkv9iLnzJkTffr0QZUqVXDo0CE0btwYvr6+CAoKwsOHD7F48WKsXr0aAFR6T9iTQppkaGiI3r17Y8eOHejRowf09fWlbbVMmTIwMjLC06dPtVwl/RsGld+YQqFAqVKlMH/+fGzduhXe3t549OgRAKBRo0Z4/fo1zMzM0KhRI/j6+krd8YGBgbhx4wane5LGKJVKaUD2o0ePEBcXh/z586Ndu3ZISEjAokWLMGDAAHTq1An29vYoWrQoQkJCcOfOHS1XTpmBqakpGjRoIB1MMHVbDQ8Ph7GxMYoWLarN8ug/8KfLbyx1P76npycAYO7cudDV1UWPHj2QJ08ejB8/HjNmzMDHjx/x4cMHhISEYNOmTVi1ahVOnTolHRWU6Gd8PnB2woQJOHv2LIYPH44aNWrAysoKcXFxePnyJUxMTKCjo4PExEQ4OTlhxIgRqF+/vparp4xIfDaTMZWhoaH0/5SUFLx9+xY9e/aEQqFAu3btfnWJpAYGld9Yao/IoUOHoKOjg+TkZHh5eSEhIQEjR46Em5sb4uPjMWPGDGzfvh3Zs2eHgYEBgoODUaxYMS1XTxnF5yFlxYoV8PX1RalSpaSZO4mJibCyssKpU6ekAbPv3r3DmjVroKOjoxJ0iH5EWFgYIiIikC1bNtjZ2f3rEWSTk5Ph5+eHTZs2ISIiAufOnZPO3cNeZnni9OTfXGBgoHQiN1NTUzx48ACLFy9G3759MXLkSNjY2CAmJgbHjx+Hk5MTbG1tYWtrq+2y6Tf3Zbi4f/8+mjdvjtmzZ6NJkyZp1rt48SLGjRuH2NhYWFlZISAgAPr6+gwp9NM2bNiA+fPnIzw8HNbW1ujfv7/UU5Lqy+0sKCgIt2/fRr9+/TjL7DfAoPIbUyqV6NChAxQKBTZu3Ci1L1myBCNGjICnpyf69u2LvHnzarFKymhatmyJMWPGoGzZslLbtWvXUL9+fRw/fhwFCxb86kEEExISIISAkZERFAoFvxzop23YsAGenp7S4fFnzJiBR48e4fTp09K2lRpSIiMjcejQIbi5uancB3tS5I8/ZX5jqb8QUrvYk5KSAAD9+/eHh4cH1q5di8WLF6vMBiL6Webm5nBxcVFpMzIywvv373Hr1i2pLfX8PmfPnsWOHTugo6MDY2NjKBQKKJVKhhT6KZcuXcLUqVOxdOlSdOvWDcWLF8fgwYPh7OyMM2fO4Pbt24iOjpZ2i69fvx59+/bFX3/9pXI/DCnyx6DyG3rx4oX0/4IFC2Lv3r0IDw+HgYEBkpOTAQAODg4wMTFBcHAwjI2NtVUqZSDPnz8HAKxduxYGBgZYvHgxDh06hKSkJDg7O6NNmzaYO3cuDh8+DIVCAR0dHaSkpGD69OkIDg5WGTfA3T30sxITEzFo0CA0atRIaps0aRKOHDmCdu3aoXPnzmjbti0iIiKgr6+Phg0bYtiwYRw4+xvirp/fzPXr19GvXz+0b98effr0QVJSEmrWrIm3b9/i2LFjsLOzAwCMHDkSRYsWRePGjdOc1pxIXT179gQAjB49WtqV6OLigrdv32Lz5s2oWrUqTp48iYULF+LmzZvo0KEDDAwMcOTIEbx58wZXrlxhDwpplFKpxJs3b5A9e3YAQOfOnXH48GHs2bMHjo6OOH78OKZNm4aRI0eiffv2KmNWuLvn98KfNb8ZExMTWFhYYPv27Vi3bh0MDAywYsUK2NjYoHDhwmjevDnq1q2LRYsWoWzZsgwppBEuLi44ePAgvL29ERISAgC4ceMGChYsiA4dOuDEiROoUqUKpkyZgs6dO8PPzw9Hjx5Frly5cPnyZWnAIpGm6OjoSCEFAIYNG4bz58+jbNmyyJ49Oxo0aICIiAi8fv06zVRlhpTfC3tUfkMhISEYM2YMXr16hZ49e6JTp05ISUnBvHnzEBYWBiEE+vfvjyJFimi7VMpA1qxZgwkTJqBt27bo2bMnChYsCACoWrUqHj9+DH9/f1StWhUA8OHDB5iYmEi35cBZ+tWePXuGjh07YtiwYTzp4G+OQeU3cOXKFbx8+VJlX2xISAjGjRuH0NBQ9O/fHx06dNBihZSRfT61c/Xq1ZgwYQLatWuXJqyEhYVhw4YNqFChgsp4lK8dfItIHZ9vQ6n/T/33zZs3sLGxUVk/Li4O7dq1Q1RUFI4ePcoelN8cg4rMxcTEoFGjRtDV1cWIESPQoEEDaVloaCjq168PExMT9OjRA3379tVipZTRfOsYJytXrsTkyZPRpk0b9OrVSworNWvWxOnTp3Hu3DmUKlXqV5dLGdTXtsPUtoCAAGzatAmLFi2Cvb094uPjsXv3bvj5+eH58+e4ePEi9PX1OSblN8cxKjKVmh+zZs2KOXPmQE9PD0uXLsX+/fuldZycnFCjRg28evUKR44cQWRkpJaqpYzm8y+HM2fOIDg4GNevXwfwaWDt+PHjsXnzZvj6+uLevXsAgKNHj6JHjx5ppi4T/ahTp05JJwwcMmQIZs2aBeDT+JQtW7agc+fOqF27Nuzt7QF8OqHl48ePkTdvXly6dAn6+vr4+PEjQ8pvjj0qMpPanZn6CyD1C+P8+fMYNWoUTE1N0adPH2k30NChQ5E3b160bNkSOXLk0HL1lBF83s0+ZMgQbNmyBbGxsXBwcECuXLlw4MABAMCKFSswbdo0tG3bFu7u7iqnZeAvWPoZQghERUXB1tYWDRo0gLW1NQICAnDy5EkUK1YMkZGRcHV1haenJ/r37y/d5vPPToDbYUbBoCIjqW+04OBg7NmzBxEREahcuTL+/PNPWFhY4Ny5cxg/fjwSExORN29emJiYYMuWLbh+/TocHBy0XT5lAJ+HlEOHDmHQoEHw9fWFhYUF/vnnH0ycOBGmpqa4dOkSgE9jVjw8PODl5YV+/fpps3TKgMLDw5E3b16kpKRgx44daNiwobTsa2NTvjaWhX5/3PUjIwqFAjt37kSTJk3w4cMHfPjwAX5+fujTpw8iIiLg6uqKefPmoVq1aggJCcGjR49w9OhRhhTSmNQP9j179mDz5s2oXbs2KleujGLFiqF169bYsGEDYmNj0adPHwBA9+7dsXv3buk6kaYkJibi1atXMDExga6uLtasWSNNjQcAa2tr6f+pR0H+PJgwpGQc7FGRkUuXLqFt27YYNWoUevTogbCwMJQuXRrGxsYoWbIkNmzYACsrK+ncKV9OASXShIiICDRu3BjXr19HjRo1sG/fPpXlY8aMwenTp/H333/D1NRUamc3O/2sbw3gDg0NhYuLC2rUqIEFCxYgX758WqiOtIU9Kloyc+ZMjB07VvolAHw6RLmrqyt69OiB0NBQ1KpVC82bN8e4ceNw8eJF9O3bFxERETAyMgIAhhTSiM+3QQCwsrLC+vXrUadOHVy9ehVr165VWZ4/f368e/cO8fHxKu0MKfQzPg8px44dw8aNG3H9+nU8f/4cTk5OOH36NIKDgzFixAhpAHeLFi2wZMkSbZZNvwB7VLRkyZIlGDhwIGbMmIERI0ZIb9A7d+6gYMGCaNasmfSFoVQqUbJkSYSEhKBRo0bYsmULz5VCGvH5l8PDhw+hUChgYmICOzs7PH78GJ6enoiLi8Off/4JDw8PvH79Gu7u7jAyMsK+ffvYvU4aN2zYMKxfvx56enrIkiUL7OzssHDhQpQtWxY3b95EjRo14OTkhKSkJHz8+BHXr1+XTsxKGZSgX06pVAohhFi5cqXQ0dERU6dOFcnJydLyp0+fisKFC4t9+/YJIYSIiIgQ7dq1E0uWLBHPnj3TSs2U8aRuh0IIMXHiRFG8eHFRqFAhkSNHDuHr6yuEECIkJEQ0bNhQGBkZiYIFC4oWLVqIevXqifj4eCGEECkpKVqpnTKOz7fDoKAgUaJECXHy5EkREREhdu/eLVq0aCGcnZ3FlStXhBBCPHjwQEyZMkVMnz5d+tz8/POTMh4GlV9MqVRKb0ylUin++usvoaOjI6ZNmyZ96IeHh4uSJUsKDw8PERoaKsaMGSPKlSsnXr9+rc3SKYOaMmWKsLGxEYGBgSI2Nla0aNFCWFhYiNu3bwshhHj06JFo1KiRKFmypFi4cKF0u4SEBC1VTBnR+vXrRb9+/USvXr1U2i9evCjq168v3N3dRWxsrBBCNdwwpGR83H+gBQqFAocPH8bQoUNRpkwZ6Rwqs2bNghAClpaW6NChA44fPw5XV1ds2LABPj4+sLW11XbplAF8PiZFqVTiwoULWLhwIerWrYugoCAcO3YMM2bMQJEiRZCcnIw8efJg/vz5yJ49O/bv34+AgAAAgKGhobaeAmUA4otRB7t27cKyZctw7do1JCYmSu1ly5ZFlSpVcOrUKaSkpABQndHDc0hlAtpOSpnRjh07hLGxsZg6daq4ePGiEEIIX19faTeQEEIkJiaK27dvi6CgIPH06VNtlksZ1IQJE8SsWbNEzpw5xb1790RwcLDIkiWL8Pb2FkII8eHDBzF27FgRGhoqhBDi/v37onHjxqJs2bIiICBAm6XTb+7zHhF/f3+xYcMGIYQQ/fr1ExYWFmLZsmUiKipKWicwMFAUKlRI2hYpc2FQ+cXu3bsn8uTJI5YvX55m2YoVK6TdQESa9vl4ks2bNwtHR0dx69Yt0bFjR1GvXj1hYmIiVq9eLa3z/PlzUaVKFbFhwwbptnfu3BGtW7cWYWFhv7x+yhg+3w5v3bolSpUqJUqUKCF2794thBDC3d1d5M+fX0yfPl2EhISIkJAQUatWLVGtWjWVgEOZB/vMfrEnT55AX19f5QiLqTMvevXqBVNTU3Tq1AmGhoYYNmyYFiuljCZ1ds/x48dx7NgxDB06FEWLFpUOJFirVi1069YNwKeTYfbo0QO6urpo3749dHR0oFQqUahQIWzcuJGzLOiHpW6Hw4cPx+PHj2FsbIy7d+9i8ODB+PjxI9atW4du3bph3LhxWLJkCSpVqoQsWbJgy5YtUCgU3zzWCmVcDCq/WGxsrMrxJ5RKpbS/9dixYyhTpgy2bNmict4UIk159eoVunfvjvDwcIwZMwYA0Lt3bzx8+BBHjx5FqVKlkD9/fjx58gQJCQm4ePEidHV1VQ7mxjEB9LPWrVuHVatW4ciRI8iTJw8SExPh7u6OmTNnQkdHB2vWrIGJiQm2bt2K+vXro23btjA0NERSUhIMDAy0XT79Yoylv1iJEiXw9u1b+Pr6Avj06yI1qOzevRsbN25Ey5YtUbhwYW2WSRmUnZ0dAgICkD17duzduxeXL1+Grq4u5s6diylTpqBmzZqws7NDmzZtvnn2WR47hX5WSEgIihUrhpIlS8Lc3Bx2dnZYs2YNdHV1MXjwYOzcuRNLly5F7dq1sWDBAuzZswcxMTEMKZkUfxr9Ynny5MHSpUvRu3dvJCcno3PnztDV1cW6deuwbt06nD17lkf4pHTl4uKCHTt2wN3dHT4+Pujfvz9cXFzQtGlTNG3aVGXdlJQU9qCQxoj/P1GgoaEhEhISkJSUBCMjIyQnJyNnzpyYOXMmGjduDC8vLxgbG2Pjxo1o3749hg0bBj09Pbi5uWn7KZAW8Mi0WqBUKrFjxw54eHjA1NQURkZG0NXVxaZNm1CqVCltl0eZxNWrV9GjRw+UKVMGAwcORNGiRbVdEmUSN2/eRKlSpTB+/HhMnDhRag8MDMTKlSvx/v17pKSk4NixYwCArl27Yvz48cibN6+WKiZtYlDRohcvXiAsLAwKhQJ58uRB9uzZtV0SZTJXr16Fh4cHcufOjTlz5iBPnjzaLokyiXXr1qFXr14YNGgQ2rRpA0tLSwwYMAAVK1ZEixYtULRoUezfvx8NGjTQdqmkZQwqRJnchQsX4OPjg1WrVnE2Bf1SO3bsQN++fWFgYAAhBGxtbXHmzBm8fv0aderUwfbt2+Hi4qLtMknLGFSISBo7wKmf9Ks9f/4cT58+RXJyMipVqgQdHR2MHj0au3btQnBwMOzs7LRdImkZgwoRAfhfWCHSltu3b2P27Nn4+++/cfjwYZQsWVLbJZEMcDg/EQHgtGPSro8fPyIpKQm2trY4fvw4B3eThD0qREQkG8nJyTzyMalgUCEiIiLZ4qg5IiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSL6rRw7dgwKhQKRkZHffRsnJyd4eXmlW01ElH4YVIhIo7p06QKFQoHevXunWebp6QmFQoEuXbr8+sKI6LfEoEJEGufo6IjNmzcjPj5eaktISMDGjRuRK1cuLVZGRL8bBhUi0rjSpUvD0dERAQEBUltAQABy5cqFUqVKSW2JiYkYMGAAbG1tYWRkhMqVK+PixYsq9/X333+jQIECMDY2Ro0aNRAaGprm8U6dOoUqVarA2NgYjo6OGDBgAOLi4tLt+RHRr8OgQkTpolu3bli7dq10fc2aNejatavKOiNGjMCOHTuwfv16XLlyBc7OzqhXrx4iIiIAAE+fPkXLli3RpEkTXLt2DT169MCoUaNU7uPhw4eoX78+WrVqhRs3bmDLli04deoU+vXrl/5PkojSHYMKEaWLjh074tSpUwgLC0NYWBhOnz6Njh07Ssvj4uLg7e2NuXPnokGDBihSpAhWrlwJY2NjrF69GgDg7e2NfPnyYf78+ShYsCA6dOiQZnzLzJkz0aFDBwwaNAj58+dHxYoVsXjxYmzYsAEJCQm/8ikTUTrgSQmJKF3Y2NigUaNGWLduHYQQaNSoEaytraXlDx8+RHJyMipVqiS16evr448//sCdO3cAAHfu3EH58uVV7rdChQoq169fv44bN27A399fahNCQKlU4vHjxyhcuHB6PD0i+kUYVIgo3XTr1k3aBbNs2bJ0eYzY2Fh4eHhgwIABaZZx4C7R749BhYjSTf369ZGUlASFQoF69eqpLMuXLx8MDAxw+vRp5M6dG8CnM+devHgRgwYNAgAULlwYe/bsUbnduXPnVK6XLl0a//zzD5ydndPviRCR1nCMChGlG11dXdy5cwf//PMPdHV1VZaZmpqiT58+GD58OA4ePIh//vkHPXv2xIcPH9C9e3cAQO/evfHgwQMMHz4c9+7dw8aNG7Fu3TqV+xk5ciTOnDmDfv364dq1a3jw4AF2797NwbREGQSDChGlKzMzM5iZmX112axZs9CqVSt06tQJpUuXRkhICAIDA2FpaQng066bHTt2YNeuXShRogR8fHwwY8YMlftwcXHB8ePHcf/+fVSpUgWlSpXChAkTYG9vn+7PjYjSn0IIIbRdBBEREdHXsEeFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhk6/8AHoK08GWUizwAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "## calculate avg response time\n", - "unique_models = set(result[\"response\"]['model'] for result in result[\"results\"])\n", - "model_dict = {model: {\"response_time\": []} for model in unique_models}\n", - "for completion_result in result[\"results\"]:\n", - " model_dict[completion_result[\"response\"][\"model\"]][\"response_time\"].append(completion_result[\"response_time\"])\n", - "\n", - "avg_response_time = {}\n", - "for model, data in model_dict.items():\n", - " avg_response_time[model] = sum(data[\"response_time\"]) / len(data[\"response_time\"])\n", - "\n", - "models = list(avg_response_time.keys())\n", - "response_times = list(avg_response_time.values())\n", - "\n", - "plt.bar(models, response_times)\n", - "plt.xlabel('Model', fontsize=10)\n", - "plt.ylabel('Average Response Time')\n", - "plt.title('Average Response Times for each Model')\n", - "\n", - "plt.xticks(models, [model[:15]+'...' if len(model) > 15 else model for model in models], rotation=45)\n", - "plt.show()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "inSDIE3_IRds" - }, - "source": [ - "# Duration Test endpoint\n", - "\n", - "Run load testing for 2 mins. Hitting endpoints with 100+ queries every 15 seconds." - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "id": "ePIqDx2EIURH" - }, - "outputs": [], - "source": [ - "models=[\"gpt-3.5-turbo\", \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"claude-instant-1\"]\n", - "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n", - "prompt = \"Where does Paul Graham live?\"\n", - "final_prompt = context + prompt\n", - "result = load_test_model(models=models, prompt=final_prompt, num_calls=100, interval=15, duration=120)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 552 - }, - "id": "k6rJoELM6t1K", - "outputId": "f4968b59-3bca-4f78-a88b-149ad55e3cf7" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAIXCAYAAABghH+YAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABwdUlEQVR4nO3dd1QU198G8GfpoNKUooKCYuwIaiL2GrGLJnYFOxrsNZbYFTsYG2JDjV2xRKOIir33EhsWLBGwUaXJ3vcPX+bnCiYsLi6Oz+ecPbp37ux+lx3YZ+/cmVEIIQSIiIiIZEJH2wUQERERaRLDDREREckKww0RERHJCsMNERERyQrDDREREckKww0RERHJCsMNERERyQrDDREREckKww0RERHJCsMNEcmSg4MDunfvru0y1DZnzhyUKFECurq6cHFx0XY5GnfkyBEoFAps27ZN26WoTaFQYNKkSWqv9+jRIygUCgQFBWm8Jsoaww19tiVLlkChUKBatWraLiXPcXBwgEKhkG758uXDDz/8gLVr12q7tK9Oxodidm5fqwMHDmDUqFGoWbMmVq9ejRkzZmi7pDwnKChIep9PnDiRabkQAvb29lAoFGjRooUWKqS8QE/bBdDXb/369XBwcMC5c+cQHh4OJycnbZeUp7i4uGD48OEAgOfPn2PFihXw8vJCSkoK+vTpo+Xqvh5ly5bFunXrVNrGjBmD/PnzY9y4cZn637lzBzo6X9f3t8OHD0NHRwcrV66EgYGBtsvJ04yMjLBhwwbUqlVLpf3o0aN4+vQpDA0NtVQZ5QUMN/RZHj58iFOnTiE4OBje3t5Yv349Jk6c+EVrUCqVSE1NhZGR0Rd93uwqWrQounbtKt3v3r07SpQoAT8/P4YbNdjY2Kj8HAFg5syZKFSoUKZ2AF/lh1t0dDSMjY01FmyEEEhOToaxsbFGHi8vadasGbZu3Yrff/8denr/+yjbsGEDqlSpgpcvX2qxOtK2r+trDeU569evh4WFBZo3b46ff/4Z69evl5alpaXB0tISPXr0yLReXFwcjIyMMGLECKktJSUFEydOhJOTEwwNDWFvb49Ro0YhJSVFZV2FQoEBAwZg/fr1KF++PAwNDbF//34AwNy5c1GjRg0ULFgQxsbGqFKlSpb79pOSkjBo0CAUKlQIBQoUQKtWrfDs2bMs96k/e/YMPXv2hI2NDQwNDVG+fHmsWrUqxz8zKysrlClTBvfv31dpVyqV8Pf3R/ny5WFkZAQbGxt4e3vjzZs3Kv0uXLgAd3d3FCpUCMbGxnB0dETPnj2l5Rn79+fOnQs/Pz8UL14cxsbGqFu3Lm7cuJGpnsOHD6N27drIly8fzM3N0bp1a9y6dUulz6RJk6BQKBAeHo7u3bvD3NwcZmZm6NGjB96+favSNzQ0FLVq1YK5uTny58+P0qVLY+zYsSp9svtef46P59xk7M44ceIEBg0aBCsrK5ibm8Pb2xupqamIiYmBp6cnLCwsYGFhgVGjRkEIofKYmnqPsqJQKLB69WokJiZKu10y5mi8e/cOU6dORcmSJWFoaAgHBweMHTs208/LwcEBLVq0QEhICKpWrQpjY2MsW7bsX5/37NmzaNKkCczMzGBiYoK6devi5MmTKn0iIiLwyy+/oHTp0jA2NkbBggXRrl07PHr0KNPjxcTEYOjQoXBwcIChoSHs7Ozg6emZKWwolUpMnz4ddnZ2MDIyQsOGDREeHv6vtX6oU6dOePXqFUJDQ6W21NRUbNu2DZ07d85yncTERAwfPhz29vYwNDRE6dKlMXfu3Ezvc0pKCoYOHQorKyvp78PTp0+zfExN/30gDRFEn6FMmTKiV69eQgghjh07JgCIc+fOSct79uwpzM3NRUpKisp6a9asEQDE+fPnhRBCpKeni8aNGwsTExMxZMgQsWzZMjFgwAChp6cnWrdurbIuAFG2bFlhZWUlJk+eLBYvXiwuX74shBDCzs5O/PLLL2LRokVi/vz54ocffhAAxJ49e1Qeo3379gKA6Natm1i8eLFo3769qFSpkgAgJk6cKPWLjIwUdnZ2wt7eXkyZMkUsXbpUtGrVSgAQfn5+//nzKV68uGjevLlKW1pamrC1tRU2NjYq7b179xZ6enqiT58+IiAgQIwePVrky5dPfP/99yI1NVUIIURUVJSwsLAQ3333nZgzZ45Yvny5GDdunChbtqz0OA8fPhQARMWKFYWDg4OYNWuWmDx5srC0tBRWVlYiMjJS6hsaGir09PTEd999J2bPni0mT54sChUqJCwsLMTDhw+lfhMnThQAhKurq2jbtq1YsmSJ6N27twAgRo0aJfW7ceOGMDAwEFWrVhULFiwQAQEBYsSIEaJOnTpSH3Xe6/9Svnx5Ubdu3U/+7L28vKT7q1evFgCEi4uLaNKkiVi8eLHo1q2b9Bpq1aolOnfuLJYsWSJatGghAIg1a9bkynuUlXXr1onatWsLQ0NDsW7dOrFu3Tpx//59IYQQXl5eAoD4+eefxeLFi4Wnp6cAIDw8PDK9ZicnJ2FhYSF+/fVXERAQIMLCwj75nIcOHRIGBgaievXqYt68ecLPz084OzsLAwMDcfbsWanf1q1bRaVKlcSECRNEYGCgGDt2rLCwsBDFixcXiYmJUr/4+HhRoUIFoaurK/r06SOWLl0qpk6dKr7//nvpdzQsLEzalqpUqSL8/PzEpEmThImJifjhhx/+9Wf04ft4/vx5UaNGDdGtWzdp2c6dO4WOjo549uxZpt89pVIpGjRoIBQKhejdu7dYtGiRaNmypQAghgwZovIcXbt2FQBE586dxaJFi0Tbtm2Fs7Nzjv8+ZPxOrl69+j9fH2kGww3l2IULFwQAERoaKoR4/8fDzs5ODB48WOoTEhIiAIg///xTZd1mzZqJEiVKSPfXrVsndHR0xPHjx1X6BQQECADi5MmTUhsAoaOjI27evJmpprdv36rcT01NFRUqVBANGjSQ2i5evJjlH7Tu3btn+uPVq1cvUbhwYfHy5UuVvh07dhRmZmaZnu9jxYsXF40bNxYvXrwQL168ENevX5c+UH18fKR+x48fFwDE+vXrVdbfv3+/SvuOHTtUQmFWMv6QGhsbi6dPn0rtZ8+eFQDE0KFDpTYXFxdhbW0tXr16JbVdvXpV6OjoCE9PT6ktI9z07NlT5bnatGkjChYsKN338/MTAMSLFy8+WZ867/V/yUm4cXd3F0qlUmqvXr26UCgUol+/flLbu3fvhJ2dncpja/I9+hQvLy+RL18+lbYrV64IAKJ3794q7SNGjBAAxOHDh1VeMwCxf//+/3wupVIpSpUqlenn8fbtW+Ho6Ch+/PFHlbaPnT59WgAQa9euldomTJggAIjg4OAsn0+I/4WbsmXLqnzpWbBggQAgrl+//q91fxhuFi1aJAoUKCDV165dO1G/fn3pZ/FhuNm5c6cAIKZNm6byeD///LNQKBQiPDxcCPG/n/cvv/yi0q9z5845/vvAcPPlcbcU5dj69ethY2OD+vXrA3g/rN6hQwds2rQJ6enpAIAGDRqgUKFC2Lx5s7TemzdvEBoaig4dOkhtW7duRdmyZVGmTBm8fPlSujVo0AAAEBYWpvLcdevWRbly5TLV9OHcgjdv3iA2Nha1a9fGpUuXpPaMXVi//PKLyroDBw5UuS+EwPbt29GyZUsIIVTqcnd3R2xsrMrjfsqBAwdgZWUFKysrVKxYEevWrUOPHj0wZ84clddvZmaGH3/8UeV5qlSpgvz580uv39zcHACwZ88epKWl/evzenh4oGjRotL9H374AdWqVcNff/0F4P3k5itXrqB79+6wtLSU+jk7O+PHH3+U+n2oX79+Kvdr166NV69eIS4uTqW+Xbt2QalUZlmXuu+1pvXq1UvliKpq1apBCIFevXpJbbq6uqhatSoePHigUrem36PsyHgfhg0bptKeMUl97969Ku2Ojo5wd3f/z8e9cuUK7t27h86dO+PVq1fS60lMTETDhg1x7Ngx6T388PcqLS0Nr169gpOTE8zNzVV+B7Zv345KlSqhTZs2mZ7v46PYevTooTK3qHbt2gCg8jP/L+3bt0dSUhL27NmD+Ph47Nmz55O7pP766y/o6upi0KBBKu3Dhw+HEAL79u2T+gHI1G/IkCEq9zX194Fyxzcdbo4dO4aWLVuiSJEiUCgU2LlzZ64/57Nnz9C1a1dpTkjFihVx4cKFXH9eTUtPT8emTZtQv359PHz4EOHh4QgPD0e1atUQFRWFQ4cOAQD09PTw008/YdeuXdL8gODgYKSlpamEm3v37uHmzZtSCMi4fffddwDeT7T8kKOjY5Z17dmzB25ubjAyMoKlpSWsrKywdOlSxMbGSn0iIiKgo6OT6TE+PsrrxYsXiImJQWBgYKa6MuYRfVxXVqpVq4bQ0FDs378fc+fOhbm5Od68eaPyh/3evXuIjY2FtbV1pudKSEiQnqdu3br46aefMHnyZBQqVAitW7fG6tWrs5yrUqpUqUxt3333nTRPIiIiAgBQunTpTP3Kli0rfdB9qFixYir3LSwsAECac9KhQwfUrFkTvXv3ho2NDTp27IgtW7aoBB1132tN+/g1mJmZAQDs7e0ztX84lyY33qPsyNheP94+bW1tYW5uLr2PGT71u/Gxe/fuAQC8vLwyvZ4VK1YgJSVF+r1JSkrChAkTpLkqhQoVgpWVFWJiYlR+t+7fv48KFSpk6/n/a1vKDisrKzRq1AgbNmxAcHAw0tPT8fPPP2fZNyIiAkWKFEGBAgVU2suWLSstz/hXR0cHJUuWVOn38e+Jpv4+UO74po+WSkxMRKVKldCzZ0+0bds215/vzZs3qFmzJurXr499+/bBysoK9+7dk36pvyaHDx/G8+fPsWnTJmzatCnT8vXr16Nx48YAgI4dO2LZsmXYt28fPDw8sGXLFpQpUwaVKlWS+iuVSlSsWBHz58/P8vk+/uDJ6uiP48ePo1WrVqhTpw6WLFmCwoULQ19fH6tXr8aGDRvUfo0ZH8hdu3aFl5dXln2cnZ3/83EKFSqERo0aAQDc3d1RpkwZtGjRAgsWLJC+jSuVSlhbW6tMyP6QlZUVAEgnPztz5gz+/PNPhISEoGfPnpg3bx7OnDmD/Pnzq/061aGrq5tlu/j/CZnGxsY4duwYwsLCsHfvXuzfvx+bN29GgwYNcODAAejq6qr9Xmvap15DVu3ig4mm2n6Psnv+nuweGZWxfc+ZM+eTJwvMqHXgwIFYvXo1hgwZgurVq8PMzAwKhQIdO3b85Ajdf/mvbSm7OnfujD59+iAyMhJNmzaVRs5ym6b+PlDu+KbDTdOmTdG0adNPLk9JScG4ceOwceNGxMTEoEKFCpg1axbq1auXo+ebNWsW7O3tsXr1aqktu9+y8pr169fD2toaixcvzrQsODgYO3bsQEBAAIyNjVGnTh0ULlwYmzdvRq1atXD48OFM5yUpWbIkrl69ioYNG+b4JGzbt2+HkZERQkJCVA4D/vDnDQDFixeHUqnEw4cPVUY3Pj5SI+NIifT0dCmcaELz5s1Rt25dzJgxA97e3siXLx9KliyJgwcPombNmtn6cHJzc4ObmxumT5+ODRs2oEuXLti0aRN69+4t9cn4Zv6hu3fvwsHBAcD7nwPw/nwwH7t9+zYKFSqEfPnyqf36dHR00LBhQzRs2BDz58/HjBkzMG7cOISFhaFRo0Yaea+1ITfeo+zI2F7v3bsnjTIAQFRUFGJiYqT3UV0ZIxOmpqb/uX1v27YNXl5emDdvntSWnJyMmJiYTI+Z1RF5ualNmzbw9vbGmTNnVHZ/f6x48eI4ePAg4uPjVUZvbt++LS3P+FepVOL+/fsqozUf/57k1t8H0oxverfUfxkwYABOnz6NTZs24dq1a2jXrh2aNGmS5YdGduzevRtVq1ZFu3btYG1tDVdXVyxfvlzDVee+pKQkBAcHo0WLFvj5558z3QYMGID4+Hjs3r0bwPsPu59//hl//vkn1q1bh3fv3qnskgLe7zt/9uxZlj+PpKSkTLtHsqKrqwuFQiHN9wHeHxb98e7GjPkIS5YsUWlfuHBhpsf76aefsH379iz/YL948eI/a/qU0aNH49WrV9Lrbd++PdLT0zF16tRMfd+9eyd9iLx58ybTN9uMb90f7/bYuXMnnj17Jt0/d+4czp49KwX6woULw8XFBWvWrFH5kLpx4wYOHDiAZs2aqf26Xr9+nant4/o08V5rQ268R9mR8T74+/urtGeMfDVv3lztxwSAKlWqoGTJkpg7dy4SEhIyLf9w+9bV1c30mhYuXKjyuwYAP/30E65evYodO3Zkejx1R2SyK3/+/Fi6dCkmTZqEli1bfrJfs2bNkJ6ejkWLFqm0+/n5QaFQSL8XGf/+/vvvKv0+/vnn5t8H+nzf9MjNv3n8+DFWr16Nx48fo0iRIgCAESNGYP/+/Tk+LfqDBw+wdOlSDBs2DGPHjsX58+cxaNAgGBgYfHJYMy/avXs34uPj0apVqyyXu7m5wcrKCuvXr5dCTIcOHbBw4UJMnDgRFStWVPkGCgDdunXDli1b0K9fP4SFhaFmzZpIT0/H7du3sWXLFum8Hf+mefPmmD9/Ppo0aYLOnTsjOjoaixcvhpOTE65duyb1q1KlCn766Sf4+/vj1atXcHNzw9GjR3H37l0AqsP/M2fORFhYGKpVq4Y+ffqgXLlyeP36NS5duoSDBw9m+WGeHU2bNkWFChUwf/58+Pj4oG7duvD29oavry+uXLmCxo0bQ19fH/fu3cPWrVuxYMEC/Pzzz1izZg2WLFmCNm3aoGTJkoiPj8fy5cthamqaKYw4OTmhVq1a6N+/P1JSUuDv74+CBQti1KhRUp85c+agadOmqF69Onr16oWkpCQsXLgQZmZmObqGzpQpU3Ds2DE0b94cxYsXR3R0NJYsWQI7OzvpTLKaeK+1ITfeo+yoVKkSvLy8EBgYiJiYGNStWxfnzp3DmjVr4OHhIU3oV5eOjg5WrFiBpk2bonz58ujRoweKFi2KZ8+eISwsDKampvjzzz8BAC1atMC6detgZmaGcuXK4fTp0zh48CAKFiyo8pgjR47Etm3b0K5dO/Ts2RNVqlTB69evsXv3bgQEBKjsitak7Pz9bNmyJerXr49x48bh0aNHqFSpEg4cOIBdu3ZhyJAh0kiWi4sLOnXqhCVLliA2NhY1atTAoUOHsjwHT279fSAN0MoxWnkQALFjxw7p/p49ewQAkS9fPpWbnp6eaN++vRBCiFu3bgkA/3obPXq09Jj6+vqievXqKs87cOBA4ebm9kVeo6a0bNlSGBkZqZzf4mPdu3cX+vr60iGSSqVS2NvbZ3koZobU1FQxa9YsUb58eWFoaCgsLCxElSpVxOTJk0VsbKzUDx8dRv2hlStXilKlSglDQ0NRpkwZsXr1aukw5g8lJiYKHx8fYWlpKfLnzy88PDzEnTt3BAAxc+ZMlb5RUVHCx8dH2NvbC319fWFraysaNmwoAgMD//NnldV5bjIEBQVlOjw0MDBQVKlSRRgbG4sCBQqIihUrilGjRol//vlHCCHEpUuXRKdOnUSxYsWEoaGhsLa2Fi1atBAXLlyQHiPjsNM5c+aIefPmCXt7e2FoaChq164trl69mqmOgwcPipo1awpjY2NhamoqWrZsKf7++2+VPhk/w48P8c44LDfjnDiHDh0SrVu3FkWKFBEGBgaiSJEiolOnTuLu3bsq62X3vf4vOTkU/ONDtD/12rI6LFsIzbxHn/Kp50xLSxOTJ08Wjo6OQl9fX9jb24sxY8aI5OTkTK/5U9vbp1y+fFm0bdtWFCxYUBgaGorixYuL9u3bi0OHDkl93rx5I3r06CEKFSok8ufPL9zd3cXt27cz/YyFEOLVq1diwIABomjRosLAwEDY2dkJLy8v6W9BxqHgW7duVVkvu4dLf+p9/FhWP4v4+HgxdOhQUaRIEaGvry9KlSol5syZo3IovBBCJCUliUGDBomCBQuKfPnyiZYtW4onT55kOhRciOz9feCh4F+eQohcGiv8yigUCuzYsQMeHh4AgM2bN6NLly64efNmpolv+fPnh62tLVJTU//zsMWCBQtKEw2LFy+OH3/8EStWrJCWL126FNOmTVPZfUDaceXKFbi6uuKPP/5Aly5dtF1Ojj169AiOjo6YM2eOyhmgiYi+Fdwt9Qmurq5IT09HdHS0dP6FjxkYGKBMmTLZfsyaNWtmmpR29+7dHE8IpJxLSkrKNCnU398fOjo6qFOnjpaqIiIiTfimw01CQoLKftSHDx/iypUrsLS0xHfffYcuXbrA09MT8+bNg6urK168eIFDhw7B2dk5R5P4hg4diho1amDGjBlo3749zp07h8DAQAQGBmryZVE2zJ49GxcvXkT9+vWhp6eHffv2Yd++fejbt2+uH4pMRES5TNv7xbQpY9/vx7eMfcipqaliwoQJwsHBQejr64vChQuLNm3aiGvXruX4Of/8809RoUIFaU5IduZtkOYdOHBA1KxZU1hYWAh9fX1RsmRJMWnSJJGWlqbt0j7bh3NuiIi+RZxzQ0RERLLC89wQERGRrDDcEBERkax8cxOKlUol/vnnHxQoUOCrOvU7ERHRt0wIgfj4eBQpUgQ6Ov8+NvPNhZt//vmHR8MQERF9pZ48eQI7O7t/7fPNhZuMC6Y9efIEpqamWq6GiIiIsiMuLg729vYqFz79lG8u3GTsijI1NWW4ISIi+spkZ0oJJxQTERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs6Gm7ACLSLIdf92q7BNKyRzOba/X5uQ2StrdBjtwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrGg13CxduhTOzs4wNTWFqakpqlevjn379v3rOlu3bkWZMmVgZGSEihUr4q+//vpC1RIREdHXQKvhxs7ODjNnzsTFixdx4cIFNGjQAK1bt8bNmzez7H/q1Cl06tQJvXr1wuXLl+Hh4QEPDw/cuHHjC1dOREREeZVCCCG0XcSHLC0tMWfOHPTq1SvTsg4dOiAxMRF79uyR2tzc3ODi4oKAgIBsPX5cXBzMzMwQGxsLU1NTjdVNlFfwooWk7YsWchuk3NgG1fn8zjNzbtLT07Fp0yYkJiaievXqWfY5ffo0GjVqpNLm7u6O06dPf/JxU1JSEBcXp3IjIiIi+dJ6uLl+/Try588PQ0ND9OvXDzt27EC5cuWy7BsZGQkbGxuVNhsbG0RGRn7y8X19fWFmZibd7O3tNVo/ERER5S1aDzelS5fGlStXcPbsWfTv3x9eXl74+++/Nfb4Y8aMQWxsrHR78uSJxh6biIiI8h49bRdgYGAAJycnAECVKlVw/vx5LFiwAMuWLcvU19bWFlFRUSptUVFRsLW1/eTjGxoawtDQULNFExERUZ6l9ZGbjymVSqSkpGS5rHr16jh06JBKW2ho6Cfn6BAREdG3R6sjN2PGjEHTpk1RrFgxxMfHY8OGDThy5AhCQkIAAJ6enihatCh8fX0BAIMHD0bdunUxb948NG/eHJs2bcKFCxcQGBiozZdBREREeYhWw010dDQ8PT3x/PlzmJmZwdnZGSEhIfjxxx8BAI8fP4aOzv8Gl2rUqIENGzZg/PjxGDt2LEqVKoWdO3eiQoUK2noJRERElMdoNdysXLnyX5cfOXIkU1u7du3Qrl27XKqIiIiIvnZ5bs4NERER0edguCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWdFquPH19cX333+PAgUKwNraGh4eHrhz586/rhMUFASFQqFyMzIy+kIVExERUV6n1XBz9OhR+Pj44MyZMwgNDUVaWhoaN26MxMTEf13P1NQUz58/l24RERFfqGIiIiLK6/S0+eT79+9XuR8UFARra2tcvHgRderU+eR6CoUCtra2uV0eERERfYXy1Jyb2NhYAIClpeW/9ktISEDx4sVhb2+P1q1b4+bNm5/sm5KSgri4OJUbERERyVeeCTdKpRJDhgxBzZo1UaFChU/2K126NFatWoVdu3bhjz/+gFKpRI0aNfD06dMs+/v6+sLMzEy62dvb59ZLICIiojwgz4QbHx8f3LhxA5s2bfrXftWrV4enpydcXFxQt25dBAcHw8rKCsuWLcuy/5gxYxAbGyvdnjx5khvlExERUR6h1Tk3GQYMGIA9e/bg2LFjsLOzU2tdfX19uLq6Ijw8PMvlhoaGMDQ01ESZRERE9BXQ6siNEAIDBgzAjh07cPjwYTg6Oqr9GOnp6bh+/ToKFy6cCxUSERHR10arIzc+Pj7YsGEDdu3ahQIFCiAyMhIAYGZmBmNjYwCAp6cnihYtCl9fXwDAlClT4ObmBicnJ8TExGDOnDmIiIhA7969tfY6iIiIKO/QarhZunQpAKBevXoq7atXr0b37t0BAI8fP4aOzv8GmN68eYM+ffogMjISFhYWqFKlCk6dOoVy5cp9qbKJiIgoD9NquBFC/GefI0eOqNz38/ODn59fLlVEREREX7s8MaFYThx+3avtEkjLHs1sru0SiIi+aXnmUHAiIiIiTWC4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZyVG4uX//PsaPH49OnTohOjoaALBv3z7cvHlTo8URERERqUvtcHP06FFUrFgRZ8+eRXBwMBISEgAAV69excSJEzVeIBEREZE61A43v/76K6ZNm4bQ0FAYGBhI7Q0aNMCZM2c0WhwRERGRutQON9evX0ebNm0ytVtbW+Ply5caKYqIiIgop9QON+bm5nj+/Hmm9suXL6No0aIaKYqIiIgop9QONx07dsTo0aMRGRkJhUIBpVKJkydPYsSIEfD09MyNGomIiIiyTe1wM2PGDJQpUwb29vZISEhAuXLlUKdOHdSoUQPjx4/PjRqJiIiIsk3tq4IbGBhg+fLl+O2333Djxg0kJCTA1dUVpUqVyo36iIiIiNSidrjJUKxYMRQrVkyTtRARERF9NrXDjRAC27ZtQ1hYGKKjo6FUKlWWBwcHa6w4IiIiInWpHW6GDBmCZcuWoX79+rCxsYFCociNuoiIiIhyRO1ws27dOgQHB6NZs2a5UQ8RERHRZ1H7aCkzMzOUKFEiN2ohIiIi+mxqh5tJkyZh8uTJSEpKyo16iIiIiD6L2rul2rdvj40bN8La2hoODg7Q19dXWX7p0iWNFUdERESkLrXDjZeXFy5evIiuXbtyQjERERHlOWqHm7179yIkJAS1atXKjXqIiIiIPovac27s7e1hamqaG7UQERERfTa1w828efMwatQoPHr0KBfKISIiIvo8au+W6tq1K96+fYuSJUvCxMQk04Ti169fa6w4IiIiInWpHW78/f1zoQwiIiIizcjR0VJEREREeVW2wk1cXJw0iTguLu5f+3KyMREREWlTtsKNhYUFnj9/Dmtra5ibm2d5bhshBBQKBdLT0zVeJBEREVF2ZSvcHD58GJaWlgCAsLCwXC2IiIiI6HNkK9zUrVsXJUqUwPnz51G3bt3cromIiIgox7J9nptHjx5xlxMRERHleWqfxI+IiIgoL1PrUPCQkBCYmZn9a59WrVp9VkFEREREn0OtcPNf57jh0VJERESkbWrtloqMjIRSqfzkjcGGiIiItC3b4Sarc9sQERER5TXZDjdCiNysg4iIiEgjsh1uvLy8YGxsnJu1EBEREX22bE8oXr16dW7WQURERKQRPM8NERERyQrDDREREckKww0RERHJSo7DTXh4OEJCQpCUlAQgZ0dT+fr64vvvv0eBAgVgbW0NDw8P3Llz5z/X27p1K8qUKQMjIyNUrFgRf/31l9rPTURERPKkdrh59eoVGjVqhO+++w7NmjXD8+fPAQC9evXC8OHD1Xqso0ePwsfHB2fOnEFoaCjS0tLQuHFjJCYmfnKdU6dOoVOnTujVqxcuX74MDw8PeHh44MaNG+q+FCIiIpIhtcPN0KFDoaenh8ePH8PExERq79ChA/bv36/WY+3fvx/du3dH+fLlUalSJQQFBeHx48e4ePHiJ9dZsGABmjRpgpEjR6Js2bKYOnUqKleujEWLFqn7UoiIiEiG1Lq2FAAcOHAAISEhsLOzU2kvVaoUIiIiPquY2NhYAIClpeUn+5w+fRrDhg1TaXN3d8fOnTuz7J+SkoKUlBTpflxc3GfVSERERHmb2iM3iYmJKiM2GV6/fg1DQ8McF6JUKjFkyBDUrFkTFSpU+GS/yMhI2NjYqLTZ2NggMjIyy/6+vr4wMzOTbvb29jmukYiIiPI+tcNN7dq1sXbtWum+QqGAUqnE7NmzUb9+/RwX4uPjgxs3bmDTpk05foysjBkzBrGxsdLtyZMnGn18IiIiylvU3i01e/ZsNGzYEBcuXEBqaipGjRqFmzdv4vXr1zh58mSOihgwYAD27NmDY8eOZdrd9TFbW1tERUWptEVFRcHW1jbL/oaGhp81okRERERfF7VHbipUqIC7d++iVq1aaN26NRITE9G2bVtcvnwZJUuWVOuxhBAYMGAAduzYgcOHD8PR0fE/16levToOHTqk0hYaGorq1aur9dxEREQkT2qP3ACAmZkZxo0b99lP7uPjgw0bNmDXrl0oUKCANG/GzMxMukinp6cnihYtCl9fXwDA4MGDUbduXcybNw/NmzfHpk2bcOHCBQQGBn52PURERPT1U3vkZv/+/Thx4oR0f/HixXBxcUHnzp3x5s0btR5r6dKliI2NRb169VC4cGHptnnzZqnP48ePpXPpAECNGjWwYcMGBAYGolKlSti2bRt27tz5r5OQiYiI6Nuh9sjNyJEjMWvWLADA9evXMWzYMAwfPhxhYWEYNmyYWlcPz85ZjY8cOZKprV27dmjXrl22n4eIiIi+HWqHm4cPH6JcuXIAgO3bt6Nly5aYMWMGLl26hGbNmmm8QCIiIiJ1qL1bysDAAG/fvgUAHDx4EI0bNwbw/sR7PEEeERERaZvaIze1atXCsGHDULNmTZw7d06aH3P37t3/PIybiIiIKLepPXKzaNEi6OnpYdu2bVi6dCmKFi0KANi3bx+aNGmi8QKJiIiI1KH2yE2xYsWwZ8+eTO1+fn4aKYiIiIjoc+ToPDdKpRLh4eGIjo6GUqlUWVanTh2NFEZERESUE2qHmzNnzqBz586IiIjIdCi3QqFAenq6xoojIiIiUpfa4aZfv36oWrUq9u7di8KFC0OhUORGXUREREQ5ona4uXfvHrZt2wYnJ6fcqIeIiIjos6h9tFS1atUQHh6eG7UQERERfTa1R24GDhyI4cOHIzIyEhUrVoS+vr7KcmdnZ40VR0RERKQutcPNTz/9BADo2bOn1KZQKCCE4IRiIiIi0rocXVuKiIiIKK9SO9wUL148N+ogIiIi0ogcncTv/v378Pf3x61btwAA5cqVw+DBg1GyZEmNFkdERESkLrWPlgoJCUG5cuVw7tw5ODs7w9nZGWfPnkX58uURGhqaGzUSERERZZvaIze//vorhg4dipkzZ2ZqHz16NH788UeNFUdERESkLrVHbm7duoVevXplau/Zsyf+/vtvjRRFRERElFNqhxsrKytcuXIlU/uVK1dgbW2tiZqIiIiIckzt3VJ9+vRB37598eDBA9SoUQMAcPLkScyaNQvDhg3TeIFERERE6lA73Pz2228oUKAA5s2bhzFjxgAAihQpgkmTJmHQoEEaL5CIiIhIHWqHG4VCgaFDh2Lo0KGIj48HABQoUEDjhRERERHlRI7OcwMA0dHRuHPnDgCgTJkysLKy0lhRRERERDml9oTi+Ph4dOvWDUWKFEHdunVRt25dFClSBF27dkVsbGxu1EhERESUbWqHm969e+Ps2bPYu3cvYmJiEBMTgz179uDChQvw9vbOjRqJiIiIsk3t3VJ79uxBSEgIatWqJbW5u7tj+fLlaNKkiUaLIyIiIlKX2iM3BQsWhJmZWaZ2MzMzWFhYaKQoIiIiopxSO9yMHz8ew4YNQ2RkpNQWGRmJkSNH4rffftNocURERETqUnu31NKlSxEeHo5ixYqhWLFiAIDHjx/D0NAQL168wLJly6S+ly5d0lylRERERNmgdrjx8PDIhTKIiIiINEPtcDNx4sTcqIOIiIhII9Sec/PkyRM8ffpUun/u3DkMGTIEgYGBGi2MiIiIKCfUDjedO3dGWFgYgPcTiRs1aoRz585h3LhxmDJlisYLJCIiIlKH2uHmxo0b+OGHHwAAW7ZsQcWKFXHq1CmsX78eQUFBmq6PiIiISC1qh5u0tDQYGhoCAA4ePIhWrVoBeH99qefPn2u2OiIiIiI1qR1uypcvj4CAABw/fhyhoaHSWYn/+ecfFCxYUOMFEhEREalD7XAza9YsLFu2DPXq1UOnTp1QqVIlAMDu3bul3VVERERE2qL2oeD16tXDy5cvERcXp3K5hb59+8LExESjxRERERGpS+2RGwAQQuDixYtYtmwZ4uPjAQAGBgYMN0RERKR1ao/cREREoEmTJnj8+DFSUlLw448/okCBApg1axZSUlIQEBCQG3USERERZYvaIzeDBw9G1apV8ebNGxgbG0vtbdq0waFDhzRaHBEREZG61B65OX78OE6dOgUDAwOVdgcHBzx79kxjhRERERHlhNojN0qlEunp6Znanz59igIFCmikKCIiIqKcUjvcNG7cGP7+/tJ9hUKBhIQETJw4Ec2aNdNkbURERERqU3u31Lx58+Du7o5y5cohOTkZnTt3xr1791CoUCFs3LgxN2okIiIiyja1R27s7Oxw9epVjBs3DkOHDoWrqytmzpyJy5cvw9raWq3HOnbsGFq2bIkiRYpAoVBg586d/9r/yJEjUCgUmW6RkZHqvgwiIiKSKbVHbgBAT08PXbp0QZcuXaS258+fY+TIkVi0aFG2HycxMRGVKlVCz5490bZt22yvd+fOHZiamkr31Q1VREREJF9qhZubN28iLCwMBgYGaN++PczNzfHy5UtMnz4dAQEBKFGihFpP3rRpUzRt2lStdYD3Ycbc3Fzt9YiIiEj+sr1bavfu3XB1dcWgQYPQr18/VK1aFWFhYShbtixu3bqFHTt24ObNm7lZq8TFxQWFCxfGjz/+iJMnT/5r35SUFMTFxanciIiISL6yHW6mTZsGHx8fxMXFYf78+Xjw4AEGDRqEv/76C/v375euDp6bChcujICAAGzfvh3bt2+Hvb096tWrh0uXLn1yHV9fX5iZmUk3e3v7XK+TiIiItCfb4ebOnTvw8fFB/vz5MXDgQOjo6MDPzw/ff/99btanonTp0vD29kaVKlVQo0YNrFq1CjVq1ICfn98n1xkzZgxiY2Ol25MnT75YvURERPTlZXvOTXx8vDSJV1dXF8bGxmrPsckNP/zwA06cOPHJ5YaGhjA0NPyCFREREZE2qTWhOCQkBGZmZgDen6n40KFDuHHjhkqfVq1aaa66bLhy5QoKFy78RZ+TiIiI8i61wo2Xl5fKfW9vb5X7CoUiy0szfEpCQgLCw8Ol+w8fPsSVK1dgaWmJYsWKYcyYMXj27BnWrl0LAPD394ejoyPKly+P5ORkrFixAocPH8aBAwfUeRlEREQkY9kON0qlUuNPfuHCBdSvX1+6P2zYMADvQ1RQUBCeP3+Ox48fS8tTU1MxfPhwPHv2DCYmJnB2dsbBgwdVHoOIiIi+bTk6iZ+m1KtXD0KITy4PCgpSuT9q1CiMGjUql6siIiKir5nal18gIiIiyssYboiIiEhWGG6IiIhIVhhuiIiISFZyFG5iYmKwYsUKjBkzBq9fvwYAXLp0Cc+ePdNocURERETqUvtoqWvXrqFRo0YwMzPDo0eP0KdPH1haWiI4OBiPHz+WzklDREREpA1qj9wMGzYM3bt3x71792BkZCS1N2vWDMeOHdNocURERETqUjvcnD9/PtOZiQGgaNGiiIyM1EhRRERERDmldrgxNDREXFxcpva7d+/CyspKI0URERER5ZTa4aZVq1aYMmUK0tLSALy/ntTjx48xevRo/PTTTxovkIiIiEgdaoebefPmISEhAdbW1khKSkLdunXh5OSEAgUKYPr06blRIxEREVG2qX20lJmZGUJDQ3HixAlcu3YNCQkJqFy5Mho1apQb9RERERGpJccXzqxVqxZq1aqlyVqIiIiIPpva4eb333/Psl2hUMDIyAhOTk6oU6cOdHV1P7s4IiIiInWpHW78/Pzw4sULvH37FhYWFgCAN2/ewMTEBPnz50d0dDRKlCiBsLAw2Nvba7xgIiIion+j9oTiGTNm4Pvvv8e9e/fw6tUrvHr1Cnfv3kW1atWwYMECPH78GLa2thg6dGhu1EtERET0r9QeuRk/fjy2b9+OkiVLSm1OTk6YO3cufvrpJzx48ACzZ8/mYeFERESkFWqP3Dx//hzv3r3L1P7u3TvpDMVFihRBfHz851dHREREpCa1w039+vXh7e2Ny5cvS22XL19G//790aBBAwDA9evX4ejoqLkqiYiIiLJJ7XCzcuVKWFpaokqVKjA0NIShoSGqVq0KS0tLrFy5EgCQP39+zJs3T+PFEhEREf0Xtefc2NraIjQ0FLdv38bdu3cBAKVLl0bp0qWlPvXr19dchURERERqyPFJ/MqUKYMyZcposhYiIiKiz5ajcPP06VPs3r0bjx8/Rmpqqsqy+fPna6QwIiIiopxQO9wcOnQIrVq1QokSJXD79m1UqFABjx49ghAClStXzo0aiYiIiLJN7QnFY8aMwYgRI3D9+nUYGRlh+/btePLkCerWrYt27drlRo1ERERE2aZ2uLl16xY8PT0BAHp6ekhKSkL+/PkxZcoUzJo1S+MFEhEREalD7XCTL18+aZ5N4cKFcf/+fWnZy5cvNVcZERERUQ6oPefGzc0NJ06cQNmyZdGsWTMMHz4c169fR3BwMNzc3HKjRiIiIqJsUzvczJ8/HwkJCQCAyZMnIyEhAZs3b0apUqV4pBQRERFpnVrhJj09HU+fPoWzszOA97uoAgICcqUwIiIiopxQa86Nrq4uGjdujDdv3uRWPURERESfRe0JxRUqVMCDBw9yoxYiIiKiz6Z2uJk2bRpGjBiBPXv24Pnz54iLi1O5EREREWmT2hOKmzVrBgBo1aoVFAqF1C6EgEKhQHp6uuaqIyIiIlKT2uEmLCwsN+ogIiIi0gi1w03dunVzow4iIiIijVB7zg0AHD9+HF27dkWNGjXw7NkzAMC6detw4sQJjRZHREREpC61w8327dvh7u4OY2NjXLp0CSkpKQCA2NhYzJgxQ+MFEhEREakjR0dLBQQEYPny5dDX15faa9asiUuXLmm0OCIiIiJ1qR1u7ty5gzp16mRqNzMzQ0xMjCZqIiIiIsoxtcONra0twsPDM7WfOHECJUqU0EhRRERERDmldrjp06cPBg8ejLNnz0KhUOCff/7B+vXrMWLECPTv3z83aiQiIiLKNrUPBf/111+hVCrRsGFDvH37FnXq1IGhoSFGjBiBgQMH5kaNRERERNmmdrhRKBQYN24cRo4cifDwcCQkJKBcuXLInz9/btRHREREpBa1d0v98ccfePv2LQwMDFCuXDn88MMPDDZERESUZ6gdboYOHQpra2t07twZf/3112ddS+rYsWNo2bIlihQpAoVCgZ07d/7nOkeOHEHlypVhaGgIJycnBAUF5fj5iYiISH7UDjfPnz/Hpk2boFAo0L59exQuXBg+Pj44deqU2k+emJiISpUqYfHixdnq//DhQzRv3hz169fHlStXMGTIEPTu3RshISFqPzcRERHJk9pzbvT09NCiRQu0aNECb9++xY4dO7BhwwbUr18fdnZ2uH//frYfq2nTpmjatGm2+wcEBMDR0RHz5s0DAJQtWxYnTpyAn58f3N3d1X0pREREJENqh5sPmZiYwN3dHW/evEFERARu3bqlqbqydPr0aTRq1Eilzd3dHUOGDPnkOikpKdIlIgAgLi4ut8ojIiKiPCBHF858+/Yt1q9fj2bNmqFo0aLw9/dHmzZtcPPmTU3XpyIyMhI2NjYqbTY2NoiLi0NSUlKW6/j6+sLMzEy62dvb52qNREREpF1qh5uOHTvC2toaQ4cORYkSJXDkyBGEh4dj6tSpKFOmTG7U+FnGjBmD2NhY6fbkyRNtl0RERES5SO3dUrq6utiyZQvc3d2hq6ursuzGjRuoUKGCxor7mK2tLaKiolTaoqKiYGpqCmNj4yzXMTQ0hKGhYa7VRERERHmL2uFm/fr1Kvfj4+OxceNGrFixAhcvXvysQ8P/S/Xq1fHXX3+ptIWGhqJ69eq59pxERET0dcnRnBvg/TlqvLy8ULhwYcydOxcNGjTAmTNn1HqMhIQEXLlyBVeuXAHw/lDvK1eu4PHjxwDe71Ly9PSU+vfr1w8PHjzAqFGjcPv2bSxZsgRbtmzB0KFDc/oyiIiISGbUGrmJjIxEUFAQVq5cibi4OLRv3x4pKSnYuXMnypUrp/aTX7hwAfXr15fuDxs2DADg5eWFoKAgPH/+XAo6AODo6Ii9e/di6NChWLBgAezs7LBixQoeBk5ERESSbIebli1b4tixY2jevDn8/f3RpEkT6OrqIiAgIMdPXq9ePQghPrk8q7MP16tXD5cvX87xcxIREZG8ZTvc7Nu3D4MGDUL//v1RqlSp3KyJiIiIKMeyPefmxIkTiI+PR5UqVVCtWjUsWrQIL1++zM3aiIiIiNSW7XDj5uaG5cuX4/nz5/D29samTZtQpEgRKJVKhIaGIj4+PjfrJCIiIsoWtY+WypcvH3r27IkTJ07g+vXrGD58OGbOnAlra2u0atUqN2okIiIiyrYcHwoOAKVLl8bs2bPx9OlTbNy4UVM1EREREeXYZ4WbDLq6uvDw8MDu3bs18XBEREREOaaRcENERESUVzDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs5Ilws3jxYjg4OMDIyAjVqlXDuXPnPtk3KCgICoVC5WZkZPQFqyUiIqK8TOvhZvPmzRg2bBgmTpyIS5cuoVKlSnB3d0d0dPQn1zE1NcXz58+lW0RExBesmIiIiPIyrYeb+fPno0+fPujRowfKlSuHgIAAmJiYYNWqVZ9cR6FQwNbWVrrZ2Nh8wYqJiIgoL9NquElNTcXFixfRqFEjqU1HRweNGjXC6dOnP7leQkICihcvDnt7e7Ru3Ro3b978ZN+UlBTExcWp3IiIiEi+tBpuXr58ifT09EwjLzY2NoiMjMxyndKlS2PVqlXYtWsX/vjjDyiVStSoUQNPnz7Nsr+vry/MzMykm729vcZfBxEREeUdWt8tpa7q1avD09MTLi4uqFu3LoKDg2FlZYVly5Zl2X/MmDGIjY2Vbk+ePPnCFRMREdGXpKfNJy9UqBB0dXURFRWl0h4VFQVbW9tsPYa+vj5cXV0RHh6e5XJDQ0MYGhp+dq1ERET0ddDqyI2BgQGqVKmCQ4cOSW1KpRKHDh1C9erVs/UY6enpuH79OgoXLpxbZRIREdFXRKsjNwAwbNgweHl5oWrVqvjhhx/g7++PxMRE9OjRAwDg6emJokWLwtfXFwAwZcoUuLm5wcnJCTExMZgzZw4iIiLQu3dvbb4MIiIiyiO0Hm46dOiAFy9eYMKECYiMjISLiwv2798vTTJ+/PgxdHT+N8D05s0b9OnTB5GRkbCwsECVKlVw6tQplCtXTlsvgYiIiPIQrYcbABgwYAAGDBiQ5bIjR46o3Pfz84Ofn98XqIqIiIi+Rl/d0VJERERE/4bhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkJU+Em8WLF8PBwQFGRkaoVq0azp0796/9t27dijJlysDIyAgVK1bEX3/99YUqJSIiorxO6+Fm8+bNGDZsGCZOnIhLly6hUqVKcHd3R3R0dJb9T506hU6dOqFXr164fPkyPDw84OHhgRs3bnzhyomIiCgv0nq4mT9/Pvr06YMePXqgXLlyCAgIgImJCVatWpVl/wULFqBJkyYYOXIkypYti6lTp6Jy5cpYtGjRF66ciIiI8iKthpvU1FRcvHgRjRo1ktp0dHTQqFEjnD59Ost1Tp8+rdIfANzd3T/Zn4iIiL4tetp88pcvXyI9PR02NjYq7TY2Nrh9+3aW60RGRmbZPzIyMsv+KSkpSElJke7HxsYCAOLi4j6n9E9SprzNlcelr0dubVvZxW2QuA2StuXGNpjxmEKI/+yr1XDzJfj6+mLy5MmZ2u3t7bVQDX0LzPy1XQF967gNkrbl5jYYHx8PMzOzf+2j1XBTqFAh6OrqIioqSqU9KioKtra2Wa5ja2urVv8xY8Zg2LBh0n2lUonXr1+jYMGCUCgUn/kK6ENxcXGwt7fHkydPYGpqqu1y6BvEbZC0jdtg7hFCID4+HkWKFPnPvloNNwYGBqhSpQoOHToEDw8PAO/Dx6FDhzBgwIAs16levToOHTqEIUOGSG2hoaGoXr16lv0NDQ1haGio0mZubq6J8ukTTE1N+UtNWsVtkLSN22Du+K8Rmwxa3y01bNgweHl5oWrVqvjhhx/g7++PxMRE9OjRAwDg6emJokWLwtfXFwAwePBg1K1bF/PmzUPz5s2xadMmXLhwAYGBgdp8GURERJRHaD3cdOjQAS9evMCECRMQGRkJFxcX7N+/X5o0/PjxY+jo/O+grho1amDDhg0YP348xo4di1KlSmHnzp2oUKGCtl4CERER5SEKkZ1px0TZkJKSAl9fX4wZMybTrkCiL4HbIGkbt8G8geGGiIiIZEXrZygmIiIi0iSGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYb+mYolUptl0BERF8Aww19MzIuwPry5UsAAK88Ql/axwGb2yBpw8fboRy/+DHc0DdlwYIF8PDwwP3796FQKLRdDn1jdHR0EBsbi5CQEADgNkhaoaOjg5iYGMyZMwdv3ryRvvjJifxeEdEHPv5mrK+vD2NjYxgYGGipIvqWKZVKzJs3D97e3tizZ4+2y6Fv2IEDBzB//nwsWrRI26XkCl4VnL4JcXFxMDU1BQDExsbCzMxMyxXRt0KpVKp8M7516xZWrlyJWbNmQVdXV4uV0bckPT1dZXtLS0vD5s2b0alTJ1luhww3JHtDhw5Feno6xowZg8KFC2u7HPoGxcTEICYmBvb29iofJB9/4BB9jo+D9MdevXqFkydPokaNGihUqJDULsftkLulSHY+zut2dnZYu3at7H556esghMCvv/6KatWq4dGjRyrLuE3S53j+/Dn++ecfvHjxAsD7uTT/Nl6xZcsWeHh44OjRoyrtctwOOXJDX7WMbxxCCCgUik9+c3nz5g0sLCy0UCHJzX99O86qT0REBMaPH4+goCBZfpDQl7d69WosXrwYT548QcmSJVGrVi3Mnj1bpU9WIzL+/v4YMGAA9PT0vmS5XxzDDX01MgIM8P6XVggBPT09PHv2DDt27ECPHj2QL18+AO93RVlYWGDChAmZ1iXKqQ9Dy+HDh/H48WM4OTmhRIkSKFKkiEqf2NhYKJXKTKFajrsA6Mvas2cP2rdvjyVLlsDExAQPHjzA7NmzUaNGDaxZswYFCxaU/ua9fPkS4eHhcHNzU3mMd+/eyTrgcLcU5VkZuTsuLg5JSUlQKBQ4cOAAwsPDoaurCz09PURERMDV1RX//POPFGwSExOhr68PPz8/vH79msGGNEIIIQWbX3/9Fd27d8fcuXPRt29fjBgxAufPnwfwftdASkoKJkyYgMqVK+PVq1cqj8NgQ5/r/PnzaN68Obp374727dtj1KhRCAkJwbVr19ClSxcA708zkJaWhnXr1qFGjRo4ceKEymPIOdgADDeUx0VGRqJixYo4evQoNmzYgCZNmuDvv/8G8H5XU/ny5dGmTRtMnz5dWidfvnwYNWoU7t27B0tLSwYb0oiM7Wju3Ln4448/sHHjRty4cQNt27bFn3/+ifHjx+P06dMAAAMDA7i6uqJhw4YwNzfXYtUkRw8fPsTz589V2r7//nvs3r0bFy9eRJ8+fQC8P/VFixYtMH369EwjN7IniPK4Hj16CFNTU6GjoyOWL18utaemporNmzeL9PR0qU2pVGqjRPpGREVFibZt24pVq1YJIYTYvXu3MDU1Ff369ROurq6iYcOG4syZM0II1W3x3bt3WqmX5CkkJETY2NiITZs2SW0Z29v69euFk5OTOH/+fKb10tLSvliN2saRG8qzMk4J7uPjg/j4eBgYGMDW1hbJyckA3n8rad++vcrETY7SUG6ytrbGqFGj0KRJE1y+fBk+Pj6YNm0ali5dip9++glnzpyBj48PLl68qLItclcUaVLZsmVRr149rFu3DocOHQLwv799Li4uiI6Oli4z8yG574r6EMMN5VkZocXe3h4nTpyAl5cXOnbsiF27diEpKSlTfzleH4W051Pbk6urKwoXLox9+/bB2dkZffv2BQBYWlrCzc0NLVu2hKur65cslb4x9vb26NevH2JiYuDn54fdu3dLywoXLgxHR0ctVpc3fDsxjr4a4v8nAD9//hxpaWkoVqwYrK2tUaNGDSQnJ6NXr14ICgpCixYtYGRkhICAADRq1AhOTk7aLp1kQnwweXjFihWIjo6GgYEBRowYIV26IyUlBc+ePcOjR49QunRpHDhwAK1atcLAgQP/9bQERJ8j42i7evXqYcmSJRg7dixGjx6NkJAQODs7Y8uWLVAoFPjxxx+1XapW8VBwypOCg4MxadIkREVFoXnz5mjTpg1atmwJAOjRowd27NiB4cOHIyoqCkuXLsX169dRrlw5LVdNcjNx4kT4+/vj+++/x7lz51CtWjWsW7cOtra2+PPPPzFt2jS8efMG+vr6EELg2rVr0NPT4xF6lCsytqvg4GAsWbIEBw4cwO3btxEWFoZFixbB3t4e5ubmWL9+PfT19b/p0w4w3FCec/PmTbi7u2Po0KEwMTHBxo0bYWhoCC8vL3Tt2hUAMHjwYFy6dAkpKSkIDAyEi4uLdosmWfhwtOXdu3fw8vLCwIED4erqikePHqF58+awtbXFjh07YGVlhb179yI8PBwJCQkYPXo09PT0vukPFNKMjBAjPjq3l66uLoKDg+Hp6Yn58+dLu0SB99urjo6Oyvb7Lc2x+RjDDeUpt2/fxtatW5GUlIQZM2YAAK5fv44JEyYgLi4OPXr0kAJOZGQk8uXLhwIFCmizZJKJD4PNrVu3EBcXh2XLlmHChAlwcHAA8P4Q3B9//BE2NjbYuXMnrKysVB6DwYY+14fb4cuXL6FQKFCwYEEA7//mVa5cGRMmTEC/fv2kdT4eKeTIIcMN5RFCCLx58wYtWrTA33//jZYtW2LdunXS8mvXrmHChAlISkpCx44d0aNHDy1WS3I2cuRIaVg/KioKwcHBaNq0qfRh8fDhQzRt2hRCCJw8eVLlAoREn+PDUDJ16lTs3LkTcXFxKFSoEKZPn44GDRrg2bNnKFq0qJYrzfs4243yBIVCAUtLS/j6+qJ8+fK4dOkSQkNDpeXOzs6YOnUq0tLSpF94Ik348KioPXv2YP/+/fj999+xZMkSODo6Yty4cbh69ap0xmxHR0fs2bMHLi4uvF4ZaVRGsJkyZQoWLFggnWqgUKFC6NKlC9asWZNptJCyxpEb0ppPDZ0ePXoUY8eOha2tLXx8fNCgQQNp2c2bN2FmZgY7O7svWSp9A4KDg3Hq1CkULFgQY8aMAQAkJCSgcuXKMDU1xYoVK1CpUqVM2yx3RZEmvXr1Co0bN4aPjw969uwptfft2xd//vknwsLCUKZMGe56+g8cuSGtyPjFPHXqFObPn4/ffvsNJ0+eRFpaGurWrYspU6YgMjISixYtwpEjR6T1ypcvz2BDGpeUlITffvsN8+fPx82bN6X2/Pnz49KlS4iPj4e3t7d0/agPMdiQJr179w4vX76URgUzTloaGBiIIkWKwM/PDwBPWPpfGG7oi/vwcMamTZvi5MmT2L17N8aOHYvp06cjNTUVDRs2xJQpU/Dq1StMnToVx48f13bZJGPGxsY4fvw4GjVqhIsXL2L37t1IT08H8L+Ac/v2bSxbtkzLlZKcZLXjxMbGBra2tli1ahUAwMjICKmpqQAAJycnhppsYrihLy5jxGbQoEGYP38+tm/fjq1bt+LixYvYvHkzxo8fLwWcX3/9Ffr6+jzjJmnMh3NshBDSB4ylpSU2bNgACwsLzJkzByEhIdKyfPnyITIyEoGBgVqpmeRHqVRKQeWff/5BdHQ03r59CwCYNGkSbt++LR0RlXHiyKdPn/JCrNnEOTf0xWT8MisUCixZsgRXrlxBYGAgHj58iEaNGqFWrVowNTXF1q1b4e3tjbFjx8LQ0BBv376FiYmJtssnGfjwMNuFCxfi6tWrePDgAYYMGYLKlSvDzs4OL168QOvWraGrq4uxY8fC3d1d5UzDnGNDn2P9+vVwc3NDyZIlAQBjxoxBSEgIIiIi0KhRI7Rq1QpdunTB8uXLMXXqVBQsWBAVKlTA/fv3ERMTI50okv4dww3lmowPkg/DyZUrV+Di4oK4uDg8efIETk5OaNKkCRwdHbFq1SrExsZKZxru3r07pk+fzolz9Nk+3obGjBmDlStXom/fvnj69ClOnz6N1q1bo2/fvnBycsKLFy/Qtm1bvHjxAkFBQXBzc9Ni9SQX+/btQ4sWLTB69GgMGTIE+/btw6hRo+Dv749Xr17h0qVLCAkJwW+//YZ+/frh+vXr8Pf3h46ODiwsLDBjxgyeKDK7cvWa4/TNe/DggejUqZP4+++/xZYtW4RCoRDnzp0TSqVSCCHE9evXRZkyZcTZs2eFEELcv39ftGjRQowdO1Y8fvxYm6WTzKSnpwshhFi3bp1wdHQUFy9eFEIIcfz4caFQKESpUqXE4MGDxYMHD4QQQjx//lz07dtXvHv3Tms1k/wsWrRI2NnZialTp4oBAwaI5cuXS8uePHkipkyZIhwcHMT+/fuzXD8tLe1LlfpV49gW5ark5GQcP34c3bt3x5UrV7B69Wp8//330i4qIQTevXuH06dPo3z58li7di0AYMSIETyHCH22bt26wcrKCvPnz4eOjg7S0tJgYGCAfv36oXLlyti5cyd69OiBFStWIDIyEtOmTYOOjg769OmDsmXLShOI+U2ZPldqaioMDAzg4+MDExMTjBkzBvHx8Zg2bZrUx87ODp6enjhw4AAuXLgAd3f3TBdg5S6pbNJ2uiL5yvimHBAQIHR0dESlSpXE5cuXVfrExsaK7t27i5IlSwoHBwdhZWUlfaMm+hyxsbFi8uTJwtLSUkyaNElqf/bsmYiKihLPnz8XVatWFfPmzZP6FylSRBQuXFgsWLBACCGkEUYiTfH19RXR0dFi/fr1wsTERDRr1kzcvXtXpU+HDh1E27ZttVShPPBoKcoVQgjo6OhACIEiRYpg3rx5ePfuHcaPH48TJ05I/UxNTTF37lwsWbIEEydOxNmzZ1G5cmUtVk5yEB8fD1NTU/Tv3x/jx4+Hv78/Jk6cCAAoUqQIrK2t8fz5c7x580aaT/Ps2TM0btwYEyZMgI+PDwCeS4Q+n/hgWuuaNWswdepU3Lt3D507d4afnx8uXbqEgIAA3LlzBwAQFxeHhw8folixYtoqWRY4vkUaJ/5/8ubhw4dx9OhRDBkyBC1btkSjRo3Qvn17zJw5E2PHjkWNGjUAvL8wZuPGjbVcNcnFqFGjsGzZMty/fx9WVlbo2rUrhBCYOnUqAGDy5MkA3gcgXV1dnDx5EkIIzJw5EyYmJtLht9wVRZqQEZAPHTqEy5cvIzAwUPrb17dvX6SlpWHy5MnYv38/KleujMTERKSmpmL27NnaLPvrp81hI5KfjGH8bdu2CTMzMzFmzBhx/vx5afm1a9dEuXLlRIsWLcQff/whJk2aJBQKhXjy5Al3AZBGXL16VdSpU0eULl1avHjxQgghRHR0tJg3b54wNzcXEyZMkPoOGDBAlCxZUtjZ2Qk3NzeRmpoqhODuKNKsI0eOiIoVK4qCBQuKnTt3CiGESElJkZavXLlS5M+fX1SuXFmsXbtWmsTOycM5x0PBSePOnTuHJk2aYNasWejTp4/UHhcXB1NTU9y6dQt9+vRBUlISYmNjsWXLFu6KIo04ffo0Xrx4gXLlyqFDhw5ISEiQrtz94sULrFu3DlOnTpUuSAi8Pz2BQqFAxYoVoaOjg3fv3nHSJn0W8dGpBxISEjBnzhwEBgaiWrVq2LhxI4yNjZGWlgZ9fX0AwPz583Hq1Cls3boVCoWCI4efieGGNG7RokXYsWMHDh06hNjYWBw+fBh//PEHbt26hREjRqBnz56Ijo5GbGwszMzMYG1tre2SSSY8PT3xzz//4ODBg3j06BF+/vlnxMfHZwo406ZNw4ABAzBlyhSV9fmBQpq0ePFi2NnZoXXr1khKSsLcuXOxY8cO1KtXDzNmzICRkZFKwMkIRR+HI1IfJxSTxtna2uLixYvw9fXFzz//jNWrV8PIyAjNmzdH7969cffuXVhbW6NUqVIMNqRRixcvxtOnT7Fo0SI4ODhg48aNMDMzQ82aNfHy5UtYWVmhW7dumDBhAqZNm4aVK1eqrM9gQ5ry4sULHD58GL/88gv2798PY2NjDBs2DC1atMCpU6cwbtw4JCcnQ19fH+/evQMABhsN4sgNfZaMX8SEhATkz58fABAVFYWFCxdiy5YtaNCgAbp3744ffvgBUVFRaNWqFYKCglC+fHktV05ykzHq8vvvv+Py5cuYP38+LCwscPv2bXh6eiI2NlYawYmMjMTRo0fx008/cRcUacTH56MBgKtXr+L333/HwYMHERAQgKZNmyIxMRGzZ8/GwYMHUbZsWSxZskS6dhRpDkdu6LMoFArs3bsXnTp1Qr169RAUFAQ9PT1MmzYNZ8+eRUBAANzc3KCjo4OFCxciMTGRozWUKzJGXerVq4djx45h7969AIDSpUtj3bp1sLCwQJ06dRAVFQVbW1t06NABenp60rdmos+REWwiIyOltkqVKmHw4MGoX78++vXrh/379yNfvnwYNWoUfvjhB+jo6Ei7pEjDtDSRmWTi5MmTwsjISIwcOVI0adJEODs7C29vbxEeHi71CQsLE3379hWWlpaZTuJHlFMZJ4nMSkBAgPjuu+/EnTt3pLY7d+4IBwcH0bFjxy9RHn0jPtwON23aJEqUKKFyhKgQQly5ckW0bt1aFCtWTBw5ckQIIURSUpJ0VN6/bcuUMxy5oRyLiIhAaGgopk+fjtmzZ2Pfvn3o27cvrl27Bl9fXzx48ACJiYk4ffo0oqOjcfToUbi4uGi7bJKBD3cBnDt3DqdOncLRo0el5a1atUK1atUQFhYmtX333Xc4duwY/vjjjy9eL8lTSkqKtB2mpqaiZMmSKFOmDHx8fHDx4kWpX6VKleDh4YEnT56gcePGOHXqFIyMjKQ5Nh/vzqLPx58oZcuiRYvw119/Sffv3LmDDh06YNWqVTAyMpLafXx80KVLF9y8eROzZ89GTEwMRo4ciTVr1qBChQraKJ1k5sMPg7Fjx6J79+7o2bMnvLy80KFDB8TFxaFw4cLSfIa0tDRpXXt7e+jq6iI9PV1b5ZNM7Nu3D+vWrQMA9OnTBw0aNEDVqlUxfPhw2NrawtvbGxcuXJD6FytWDB07dsS8efNQrVo1qZ2Th3OJtoeOKO97+PCh6Ny5s7h3755K+6+//iqsra1F27ZtpZOlZVi6dKkoXbq0GDRoEE9ERbli7ty5omDBguLs2bMiPT1dzJgxQygUCnHixAmpT82aNYW3t7cWqyS56tSpk3BwcBDu7u6iUKFC4urVq9Kyw4cPCw8PD1GhQgWxb98+8fDhQ+Hh4SGGDx8u9eHV5nMXww1lS2JiohBCiDNnzoht27ZJ7RMmTBAVK1YU48ePF1FRUSrrLF++XDx8+PBLlknfCKVSKby8vERgYKAQQojt27cLc3NzERAQIIQQIj4+XgghxL59+0SrVq3EtWvXtFYryZeLi4tQKBQqF2bNcPz4cdGtWzehUCjEd999J5ydnaUvejwDdu7jMZCULcbGxoiJiYGvry+ePXsGXV1deHh4YPLkyUhLS8PevXshhMDgwYNhZWUFAOjdu7eWqya5Sk5OxtmzZ1GvXj0cOXIEXl5emDNnDry9vfHu3TvMnj0b1atXh5ubG6ZMmYJz586hYsWK2i6bZCI1NRXJyclwcnJCsWLFsHnzZhQtWhQdO3aUTolRq1YtVKtWDX369EFaWhrq1q0LXV1dngH7C+GcG8oWhUIBc3NzDB8+HI6OjvD390dwcDAAYMaMGWjSpAlCQ0MxY8YMvHz5UsvVkpxcu3YNT58+BQAMHToUR48ehbGxMTp37ow//vgDzZo1g5+fn3TByzdv3uDChQu4c+cOLCwssG7dOhQvXlybL4FkxsDAAKampti6dSt27dqF77//HrNnz8amTZsQHx8v9UtOTkbt2rXRoEEDaa4Xg82XwXBD2SLe78JE7dq1MXToUFhYWOD3339XCThubm64fPkyBM8LSRoghMDdu3dRv359rFq1Cv369cOCBQtgYWEBAHBzc0NERASqVauG6tWrAwD++ecfdO/eHTExMRgwYAAAoGTJkmjUqJHWXgfJjxACSqVSur9mzRrUqFEDfn5+WLt2LR4/fowGDRqgXbt2Un+AZ8D+kniGYsqWjLO/xsbGwsTEBNeuXcP06dPx5s0bDB48GB4eHgDen3I8Y7cUkSYsX74co0aNQnJyMnbt2oXGjRtLZ8bevHkzpkyZAiEE9PT0YGxsDKVSiVOnTkFfX5/XiqLP9vr1a1haWqq0ZWx/W7duRWhoKAIDAwEAffv2xZEjR5Ceng5LS0ucPHmSZx/WEo7c0H969+4ddHV18ejRI9SrVw8HDhxAlSpVMGLECFhZWWHy5MnYs2cPADDYkMZkfDO2t7eHoaEhTE1NcebMGTx69Eg6fLZDhw5Yu3YtpkyZgvbt22P06NE4c+aMdL0eBhv6HAsWLMD333+vsqsJgBRsunfvjkqVKkntgYGBWLZsGRYuXIgzZ87AwMCAZ8DWFu3MY6a86lOz+MPDw4WNjY3o3bu3yiGMR44cEd26dROPHj36UiWSzH28DaampoqkpCSxdOlSUbRoUTF27Nj/3N54mC19rmXLlglDQ0OxYcOGTMseP34sKlasKBYtWiS1ZbXNcTvUHu6WIon4/6HW06dP49atWwgPD4enpycKFy6MNWvW4MKFC1izZk2mK9cmJyernMiPKKc+PPPw69evER8frzIZ2N/fH3PnzkWvXr3Qo0cPODg4oGXLlhg3bhzc3Ny0VTbJzPLlyzFw4ECsW7cO7dq1Q0xMDBITE5GcnAxra2sUKFAA9+7dQ6lSpbRdKn0Cww2p2L59O/r27StdYPDFixfo0KEDRo8ejQIFCmi7PJKxD4PNlClTcODAAdy4cQPt27dHmzZt0LRpUwDvA46/vz8qVKiAV69e4fHjx3j06BEvQEga8eDBAzg5OaF9+/bYtGkTbty4gV9++QUvXrxAREQE6tevj/79+6NFixbaLpX+BY9JI8mNGzcwdOhQzJs3D927d0dcXBzMzc1hbGzMYEO5LiPYTJgwAYGBgZgzZw4cHBzQr18/3Lt3DzExMejUqROGDBmCQoUK4erVq0hOTsbx48elq3vzMFv6XFZWVpg1axYmTJiAESNG4MCBA6hduzZat26NuLg4bNu2DePHj0ehQoU4WpiXaXOfGGnP4cOHxf379zO1Va9eXQghxK1bt0Tx4sVF7969peX379/nPmTKVYcPHxbly5cXx44dE0IIcerUKWFgYCDKlSsnqlWrJrZu3Sr1/fCyHrzEB2lScnKymDt3rtDR0RE9e/YUqamp0rILFy6I0qVLi8WLF2uxQvovPFrqGyOEwOXLl9G0aVMsXboUERER0rJnz55BCIGEhAQ0adIEjRs3xrJlywAAoaGhWLp0Kd68eaOt0kmGxEd7xYsWLYr+/fujdu3aOHDgAFq0aIHAwECEhobi/v37+P3337Fy5UoAUBml4YgNaZKhoSH69euH7du3o3fv3tDX15e21SpVqsDIyAhPnjzRcpX0bxhuvjEKhQKurq6YN28etmzZgqVLl+LBgwcAgObNmyMqKgqmpqZo3rw5AgMDpV0FISEhuHbtGg+tJY1RKpXSpPQHDx4gMTERpUqVQqdOnZCcnIwFCxZg0KBB6NatG4oUKYLy5csjPDwct27d0nLl9C3Ily8fmjZtKp0gMmNbjY6OhrGxMcqXL6/N8ug/8OvONyZjXoKPjw8AYM6cOdDV1UXv3r3h6OiI3377DTNmzMC7d+/w9u1bhIeHY+PGjVixYgVOnDghnR2W6HN8OHl4woQJOH36NEaOHIn69evD0tISiYmJeP78OUxMTKCjo4OUlBQ4ODhg1KhRaNKkiZarJzkSHxwBmsHQ0FD6f3p6Ol6+fIk+ffpAoVCgU6dOX7pEUgPDzTcmY+TlwIED0NHRQVpaGvz9/ZGcnIzRo0ejffv2SEpKwowZM7Bt2zbY2NjAwMAAYWFhqFChgparJ7n4MNgsW7YMgYGBcHV1lY54SklJgaWlJU6cOCFNGn716hVWrVoFHR0dlXBElBMRERF4/fo1ChYsCFtb2389k3BaWhrWrVuHjRs34vXr1zhz5ox0rSiOZudNPBT8GxQSEiJdbDBfvny4d+8efv/9d/zyyy8YPXo0rKysEB8fj6NHj8LBwQHW1tawtrbWdtn0lfs4kNy9exceHh6YNWsWWrZsmanf+fPnMX78eCQkJMDS0hLBwcHQ19dnsKHPtnbtWsybNw/R0dEoVKgQBg4cKI3IZPh4OwsNDcXNmzcxYMAAHp33FWC4+cYolUp06dIFCoUCGzZskNoXLlyIUaNGwcfHB7/88gtKlCihxSpJbtq2bYuxY8eiatWqUtuVK1fQpEkTHD16FKVLl87yxJDJyckQQsDIyAgKhYIfKPTZ1q5dCx8fH+nSCjNmzMCDBw9w8uRJadvKCDYxMTE4cOAA2rdvr/IYHLHJ+/j15xuT8U0kY/g/NTUVADBw4EB4e3tj9erV+P3331WOoiL6XGZmZnB2dlZpMzIywps3b3Djxg2pLeN6UqdPn8b27duho6MDY2NjKBQKKJVKBhv6LBcuXMDUqVOxaNEi9OzZExUrVsTQoUPh5OSEU6dO4ebNm4iLi5N22a9Zswa//PIL/vjjD5XHYbDJ+xhuvhH//POP9P/SpUvjzz//RHR0NAwMDJCWlgYAsLOzg4mJCcLCwmBsbKytUklGnj17BgBYvXo1DAwM8Pvvv+PAgQNITU2Fk5MTOnTogDlz5uDgwYNQKBTQ0dFBeno6pk+fjrCwMJV5ENwVRZ8rJSUFQ4YMQfPmzaW2SZMm4dChQ+jUqRM8PT3RsWNHvH79Gvr6+mjWrBlGjBjBycNfIe6W+gZcvXoVAwYMQOfOndG/f3+kpqaiQYMGePnyJY4cOQJbW1sAwOjRo1G+fHm0aNEClpaWWq6avnZ9+vQBAIwZM0bazens7IyXL19i06ZNqFOnDo4fPw4/Pz9cv34dXbp0gYGBAQ4dOoQXL17g0qVLHKkhjVIqlXjx4gVsbGwAAJ6enjh48CB2794Ne3t7HD16FNOmTcPo0aPRuXNnlTk43BX1deFXoW+AiYkJzM3NsW3bNgQFBcHAwADLli2DlZUVypYtCw8PDzRu3BgLFixA1apVGWxII5ydnbF//34sXboU4eHhAIBr166hdOnS6NKlC44dO4batWtjypQp8PT0xLp163D48GEUK1YMFy9elCZtEmmKjo6OFGwAYMSIETh79iyqVq0KGxsbNG3aFK9fv0ZUVFSmw8IZbL4uHLn5RoSHh2Ps2LGIjIxEnz590K1bN6Snp2Pu3LmIiIiAEAIDBw5EuXLltF0qyciqVaswYcIEdOzYEX369EHp0qUBAHXq1MHDhw+xfv161KlTBwDw9u1bmJiYSOty8jB9aU+fPkXXrl0xYsQIXhjzK8dwI1OXLl3C8+fPVfYth4eHY/z48Xj06BEGDhyILl26aLFCkrMPD6NduXIlJkyYgE6dOmUKOBEREVi7di2qV6+uMr8mqxOqEanjw20o4/8Z/7548QJWVlYq/RMTE9GpUyfExsbi8OHDHKn5yjHcyFB8fDyaN28OXV1djBo1Ck2bNpWWPXr0CE2aNIGJiQl69+6NX375RYuVktx86hw0y5cvx+TJk9GhQwf07dtXCjgNGjTAyZMncebMGbi6un7pckmmstoOM9qCg4OxceNGLFiwAEWKFEFSUhJ27dqFdevW4dmzZzh//jz09fU5x+Yrxzk3MpKRUwsUKIDZs2dDT08PixYtwt69e6U+Dg4OqF+/PiIjI3Ho0CHExMRoqVqSmw8/UE6dOoWwsDBcvXoVwPvJxb/99hs2bdqEwMBA3LlzBwBw+PBh9O7dO9Nh4kQ5deLECemilsOGDcPMmTMBvJ9vs3nzZnh6eqJRo0YoUqQIgPcXXX348CFKlCiBCxcuQF9fH+/evWOw+cpx5EYGMoZaM75pZHzInD17Fr/++ivy5cuH/v37S7uohg8fjhIlSqBt27YoXLiwlqsnOfhwF8CwYcOwefNmJCQkwM7ODsWKFcO+ffsAAMuWLcO0adPQsWNHeHl5qVzSg9+U6XMIIRAbGwtra2s0bdoUhQoVQnBwMI4fP44KFSogJiYGbm5u8PHxwcCBA6V1PvzbCXA7lAuGm69cxi9nWFgYdu/ejdevX6NWrVpo164dzM3NcebMGfz2229ISUlBiRIlYGJigs2bN+Pq1auws7PTdvkkAx8GmwMHDmDIkCEIDAyEubk5/v77b0ycOBH58uXDhQsXALyfg+Pt7Q1/f38MGDBAm6WTDEVHR6NEiRJIT0/H9u3b0axZM2lZVnNtspqbQ18/7pb6yikUCuzYsQMtW7bE27dv8fbtW6xbtw79+/fH69ev4ebmhrlz56Ju3boIDw/HgwcPcPjwYQYb0piMD4Pdu3dj06ZNaNSoEWrVqoUKFSrg559/xtq1a5GQkID+/fsDAHr16oVdu3ZJ94k0JSUlBZGRkTAxMYGuri5WrVolnYYAAAoVKiT9P+Ns2B+GGQYb+eDIzVfuwoUL6NixI3799Vf07t0bERERqFy5MoyNjeHi4oK1a9fC0tJSulbPx4fbEmnC69ev0aJFC1y9ehX169fHnj17VJaPHTsWJ0+exF9//YV8+fJJ7dwFQJ/rU5PYHz16BGdnZ9SvXx/z589HyZIltVAdaQtHbr4ivr6+GDdunPSNA3h/ens3Nzf07t0bjx49QsOGDeHh4YHx48fj/Pnz+OWXX/D69WsYGRkBAIMNacSH2yAAWFpaYs2aNfjxxx9x+fJlrF69WmV5qVKl8OrVKyQlJam0M9jQ5/gw2Bw5cgQbNmzA1atX8ezZMzg4OODkyZMICwvDqFGjpEnsbdq0wcKFC7VZNn0BHLn5iixcuBCDBw/GjBkzMGrUKOmX+tatWyhdujRat24tfcgolUq4uLggPDwczZs3x+bNm3ltHtKIDz9Q7t+/D4VCARMTE9ja2uLhw4fw8fFBYmIi2rVrB29vb0RFRcHLywtGRkbYs2cPh/5J40aMGIE1a9ZAT08P+fPnh62tLfz8/FC1alVcv34d9evXh4ODA1JTU/Hu3TtcvXpVungwyZSgr4JSqRRCCLF8+XKho6Mjpk6dKtLS0qTlT548EWXLlhV79uwRQgjx+vVr0alTJ7Fw4ULx9OlTrdRM8pOxHQohxMSJE0XFihVFmTJlROHChUVgYKAQQojw8HDRrFkzYWRkJEqXLi3atGkj3N3dRVJSkhBCiPT0dK3UTvLx4XYYGhoqKlWqJI4fPy5ev34tdu3aJdq0aSOcnJzEpUuXhBBC3Lt3T0yZMkVMnz5d+rv54d9Pkh+Gm6+AUqmUfpmVSqX4448/hI6Ojpg2bZr0QREdHS1cXFyEt7e3ePTokRg7dqz4/vvvRVRUlDZLJ5maMmWKsLKyEiEhISIhIUG0adNGmJubi5s3bwohhHjw4IFo3ry5cHFxEX5+ftJ6ycnJWqqY5GjNmjViwIABom/fvirt58+fF02aNBFeXl4iISFBCKEaiBhs5I/7Kb4SCoUCBw8exPDhw1GlShXpmj0zZ86EEAIWFhbo0qULjh49Cjc3N6xduxYBAQGwtrbWdukkAx/OsVEqlTh37hz8/PzQuHFjhIaG4siRI5gxYwbKlSuHtLQ0ODo6Yt68ebCxscHevXsRHBwMADA0NNTWSyAZEB/Noti5cycWL16MK1euICUlRWqvWrUqateujRMnTiA9PR2A6pFQvGbZN0Db6YqyZ/v27cLY2FhMnTpVnD9/XgghRGBgoLSLSgghUlJSxM2bN0VoaKh48uSJNsslmZowYYKYOXOmKFq0qLhz544ICwsT+fPnF0uXLhVCCPH27Vsxbtw48ejRIyGEEHfv3hUtWrQQVatWFcHBwdosnb5yH468rF+/Xqxdu1YIIcSAAQOEubm5WLx4sYiNjZX6hISEiDJlykjbIn1bGG6+Anfu3BGOjo5iyZIlmZYtW7ZM2kVFpGkfzo/ZtGmTsLe3Fzdu3BBdu3YV7u7uwsTERKxcuVLq8+zZM1G7dm2xdu1aad1bt26Jn3/+WURERHzx+kkePtwOb9y4IVxdXUWlSpXErl27hBBCeHl5iVKlSonp06eL8PBwER4eLho2bCjq1q2rEoro28Gxua/A48ePoa+vr3KmzYwjVvr27Yt8+fKhW7duMDQ0xIgRI7RYKclNxlFRR48exZEjRzB8+HCUL19eOjlkw4YN0bNnTwDvL9jau3dv6OrqonPnztDR0YFSqUSZMmWwYcMGHp1COZaxHY4cORIPHz6EsbExbt++jaFDh+Ldu3cICgpCz549MX78eCxcuBA1a9ZE/vz5sXnzZigUik+eC4fki+HmK5CQkKByfhClUintPz5y5AiqVKmCzZs3q1ynh0hTIiMj0atXL0RHR2Ps2LEAgH79+uH+/fs4fPgwXF1dUapUKTx+/BjJyck4f/48dHV1VU7QxzkO9LmCgoKwYsUKHDp0CI6OjkhJSYGXlxd8fX2ho6ODVatWwcTEBFu2bEGTJk3QsWNHGBoaIjU1FQYGBtoun74wRtmvQKVKlfDy5UsEBgYCeP8tJiPc7Nq1Cxs2bEDbtm1RtmxZbZZJMmVra4vg4GDY2Njgzz//xMWLF6Grq4s5c+ZgypQpaNCgAWxtbdGhQ4dPXlWZ57ahzxUeHo4KFSrAxcUFZmZmsLW1xapVq6Crq4uhQ4dix44dWLRoERo1aoT58+dj9+7diI+PZ7D5RvHr1FfA0dERixYtQr9+/ZCWlgZPT0/o6uoiKCgIQUFBOH36NM/0SrnK2dkZ27dvh5eXFwICAjBw4EA4OzujVatWaNWqlUrf9PR0jtSQxoj/v5iloaEhkpOTkZqaCiMjI6SlpaFo0aLw9fVFixYt4O/vD2NjY2zYsAGdO3fGiBEjoKenh/bt22v7JZAW8AzFXwmlUont27fD29sb+fLlg5GREXR1dbFx40a4urpquzz6Rly+fBm9e/dGlSpVMHjwYJQvX17bJdE34vr163B1dcVvv/2GiRMnSu0hISFYvnw53rx5g/T0dBw5cgQA0KNHD/z2228oUaKEliombWK4+cr8888/iIiIgEKhgKOjI2xsbLRdEn1jLl++DG9vbxQvXhyzZ8+Go6Ojtkuib0RQUBD69u2LIUOGoEOHDrCwsMCgQYNQo0YNtGnTBuXLl8fevXvRtGlTbZdKWsZwQ0RqO3fuHAICArBixQoehUJf1Pbt2/HLL7/AwMAAQghYW1vj1KlTiIqKwo8//oht27bB2dlZ22WSljHcEFGOZMyF4GG29KU9e/YMT548QVpaGmrWrAkdHR2MGTMGO3fuRFhYGGxtbbVdImkZww0R5VhGwCHSlps3b2LWrFn466+/cPDgQbi4uGi7JMoDeEgDEeUYgw1p07t375Camgpra2scPXqUE9xJwpEbIiL6qqWlpfEM2KSC4YaIiIhkhbMAiYiISFYYboiIiEhWGG6IiIhIVhhuiIiISFYYbohI9o4cOQKFQoGYmJhsr+Pg4AB/f/9cq4mIcg/DDRFpXffu3aFQKNCvX79My3x8fKBQKNC9e/cvXxgRfZUYbogoT7C3t8emTZuQlJQktSUnJ2PDhg0oVqyYFisjoq8Nww0R5QmVK1eGvb09goODpbbg4GAUK1YMrq6uUltKSgoGDRoEa2trGBkZoVatWjh//rzKY/3111/47rvvYGxsjPr16+PRo0eZnu/EiROoXbs2jI2NYW9vj0GDBiExMTHXXh8RfTkMN0SUZ/Ts2ROrV6+W7q9atQo9evRQ6TNq1Chs374da9aswaVLl+Dk5AR3d3e8fv0aAPDkyRO0bdsWLVu2xJUrV9C7d2/8+uuvKo9x//59NGnSBD/99BOuXbuGzZs348SJExgwYEDuv0giynUMN0SUZ3Tt2hUnTpxAREQEIiIicPLkSXTt2lVanpiYiKVLl2LOnDlo2rQpypUrh+XLl8PY2BgrV64EACxduhQlS5bEvHnzULp0aXTp0iXTfB1fX1906dIFQ4YMQalSpVCjRg38/vvvWLt2LZKTk7/kSyaiXMALZxJRnmFlZYXmzZsjKCgIQgg0b94chQoVkpbfv38faWlpqFmzptSmr6+PH374Abdu3QIA3Lp1C9WqVVN53OrVq6vcv3r1Kq5du4b169dLbUIIKJVKPHz4EGXLls2Nl0dEXwjDDRHlKT179pR2Dy1evDhXniMhIQHe3t4YNGhQpmWcvEz09WO4IaI8pUmTJkhNTYVCoYC7u7vKspIlS8LAwAAnT55E8eLFAby/IvT58+cxZMgQAEDZsmWxe/dulfXOnDmjcr9y5cr4+++/4eTklHsvhIi0hnNuiChP0dXVxa1bt/D3339DV1dXZVm+fPnQv39/jBw5Evv378fff/+NPn364O3bt+jVqxcAoF+/frh37x5GjhyJO3fuYMOGDQgKClJ5nNGjR+PUqVMYMGAArly5gnv37mHXrl2cUEwkEww3RJTnmJqawtTUNMtlM2fOxE8//YRu3bqhcuXKCA8PR0hICCwsLAC83620fft27Ny5E5UqVUJAQABmzJih8hjOzs44evQo7t69i9q1a8PV1RUTJkxAkSJFcv21EVHuUwghhLaLICIiItIUjtwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs/B+XLE52CERTBAAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "## calculate avg response time\n", - "unique_models = set(unique_result[\"response\"]['model'] for unique_result in result[0][\"results\"])\n", - "model_dict = {model: {\"response_time\": []} for model in unique_models}\n", - "for iteration in result:\n", - " for completion_result in iteration[\"results\"]:\n", - " model_dict[completion_result[\"response\"][\"model\"]][\"response_time\"].append(completion_result[\"response_time\"])\n", - "\n", - "avg_response_time = {}\n", - "for model, data in model_dict.items():\n", - " avg_response_time[model] = sum(data[\"response_time\"]) / len(data[\"response_time\"])\n", - "\n", - "models = list(avg_response_time.keys())\n", - "response_times = list(avg_response_time.values())\n", - "\n", - "plt.bar(models, response_times)\n", - "plt.xlabel('Model', fontsize=10)\n", - "plt.ylabel('Average Response Time')\n", - "plt.title('Average Response Times for each Model')\n", - "\n", - "plt.xticks(models, [model[:15]+'...' if len(model) > 15 else model for model in models], rotation=45)\n", - "plt.show()" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/LiteLLM_Azure_and_OpenAI_example.ipynb b/cookbook/LiteLLM_Azure_and_OpenAI_example.ipynb deleted file mode 100644 index 7df1c47eb11..00000000000 --- a/cookbook/LiteLLM_Azure_and_OpenAI_example.ipynb +++ /dev/null @@ -1,422 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "BmX0b5Ueh91v" - }, - "source": [ - "# LiteLLM - Azure OpenAI + OpenAI Calls\n", - "This notebook covers the following for Azure OpenAI + OpenAI:\n", - "* Completion - Quick start\n", - "* Completion - Streaming\n", - "* Completion - Azure, OpenAI in separate threads\n", - "* Completion - Stress Test 10 requests in parallel\n", - "* Completion - Azure, OpenAI in the same thread" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "iHq4d0dpfawS" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "mnveHO5dfcB0" - }, - "outputs": [], - "source": [ - "import os" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "eo88QUdbiDIE" - }, - "source": [ - "## Completion - Quick start" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "5OSosWNCfc_2", - "outputId": "c52344b1-2458-4695-a7eb-a9b076893348" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Openai Response\n", - "\n", - "{\n", - " \"id\": \"chatcmpl-7yjVOEKCPw2KdkfIaM3Ao1tIXp8EM\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1694708958,\n", - " \"model\": \"gpt-3.5-turbo-0613\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I'm an AI, so I don't have feelings, but I'm here to help you. How can I assist you?\"\n", - " },\n", - " \"finish_reason\": \"stop\"\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"prompt_tokens\": 13,\n", - " \"completion_tokens\": 26,\n", - " \"total_tokens\": 39\n", - " }\n", - "}\n", - "Azure Response\n", - "\n", - "{\n", - " \"id\": \"chatcmpl-7yjVQ6m2R2HRtnKHRRFp6JzL4Fjez\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1694708960,\n", - " \"model\": \"gpt-35-turbo\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"finish_reason\": \"stop\",\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Hello there! As an AI language model, I don't have feelings but I'm functioning well. How can I assist you today?\"\n", - " }\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"completion_tokens\": 27,\n", - " \"prompt_tokens\": 14,\n", - " \"total_tokens\": 41\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "# openai configs\n", - "os.environ[\"OPENAI_API_KEY\"] = \"\"\n", - "\n", - "# azure openai configs\n", - "os.environ[\"AZURE_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n", - "os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n", - "\n", - "\n", - "# openai call\n", - "response = completion(\n", - " model = \"gpt-3.5-turbo\",\n", - " messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - ")\n", - "print(\"Openai Response\\n\")\n", - "print(response)\n", - "\n", - "\n", - "\n", - "# azure call\n", - "response = completion(\n", - " model = \"azure/your-azure-deployment\",\n", - " messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - ")\n", - "print(\"Azure Response\\n\")\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "dQMkM-diiKdE" - }, - "source": [ - "## Completion - Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "uVvJDVn4g1i1" - }, - "outputs": [], - "source": [ - "import os\n", - "from litellm import completion\n", - "\n", - "# openai configs\n", - "os.environ[\"OPENAI_API_KEY\"] = \"\"\n", - "\n", - "# azure openai configs\n", - "os.environ[\"AZURE_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n", - "os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n", - "\n", - "\n", - "# openai call\n", - "response = completion(\n", - " model = \"gpt-3.5-turbo\",\n", - " messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " stream=True\n", - ")\n", - "print(\"OpenAI Streaming response\")\n", - "for chunk in response:\n", - " print(chunk)\n", - "\n", - "# azure call\n", - "response = completion(\n", - " model = \"azure/your-azure-deployment\",\n", - " messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " stream=True\n", - ")\n", - "print(\"Azure Streaming response\")\n", - "for chunk in response:\n", - " print(chunk)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4xrOPnt-oqwm" - }, - "source": [ - "## Completion - Azure, OpenAI in separate threads" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "V5b5taJPjvC3" - }, - "outputs": [], - "source": [ - "import os\n", - "import threading\n", - "from litellm import completion\n", - "\n", - "# Function to make a completion call\n", - "def make_completion(model, messages):\n", - " response = completion(\n", - " model=model,\n", - " messages=messages\n", - " )\n", - "\n", - " print(f\"Response for {model}: {response}\")\n", - "\n", - "# openai configs\n", - "os.environ[\"OPENAI_API_KEY\"] = \"\"\n", - "\n", - "# azure openai configs\n", - "os.environ[\"AZURE_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n", - "os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n", - "\n", - "# Define the messages for the completions\n", - "messages = [{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n", - "\n", - "# Create threads for making the completions\n", - "thread1 = threading.Thread(target=make_completion, args=(\"gpt-3.5-turbo\", messages))\n", - "thread2 = threading.Thread(target=make_completion, args=(\"azure/your-azure-deployment\", messages))\n", - "\n", - "# Start both threads\n", - "thread1.start()\n", - "thread2.start()\n", - "\n", - "# Wait for both threads to finish\n", - "thread1.join()\n", - "thread2.join()\n", - "\n", - "print(\"Both completions are done.\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lx8DbMBqoAoN" - }, - "source": [ - "## Completion - Stress Test 10 requests in parallel\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "pHYANOlOkoDh" - }, - "outputs": [], - "source": [ - "import os\n", - "import threading\n", - "from litellm import completion\n", - "\n", - "# Function to make a completion call\n", - "def make_completion(model, messages):\n", - " response = completion(\n", - " model=model,\n", - " messages=messages\n", - " )\n", - "\n", - " print(f\"Response for {model}: {response}\")\n", - "\n", - "# Set your API keys\n", - "os.environ[\"OPENAI_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_API_KEY\"] = \"\"\n", - "os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n", - "os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n", - "\n", - "# Define the messages for the completions\n", - "messages = [{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n", - "\n", - "# Create and start 10 threads for making completions\n", - "threads = []\n", - "for i in range(10):\n", - " thread = threading.Thread(target=make_completion, args=(\"gpt-3.5-turbo\" if i % 2 == 0 else \"azure/your-azure-deployment\", messages))\n", - " threads.append(thread)\n", - " thread.start()\n", - "\n", - "# Wait for all threads to finish\n", - "for thread in threads:\n", - " thread.join()\n", - "\n", - "print(\"All completions are done.\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yB2NDOO4oxrp" - }, - "source": [ - "## Completion - Azure, OpenAI in the same thread" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "HTBqwzxpnxab", - "outputId": "f3bc0efe-e4d5-44d5-a193-97d178cfbe14" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "OpenAI Response: {\n", - " \"id\": \"chatcmpl-7yjzrDeOeVeSrQ00tApmTxEww3vBS\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1694710847,\n", - " \"model\": \"gpt-3.5-turbo-0613\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Hello! I'm an AI, so I don't have feelings, but I'm here to help you. How can I assist you today?\"\n", - " },\n", - " \"finish_reason\": \"stop\"\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"prompt_tokens\": 13,\n", - " \"completion_tokens\": 29,\n", - " \"total_tokens\": 42\n", - " }\n", - "}\n", - "Azure OpenAI Response: {\n", - " \"id\": \"chatcmpl-7yjztAQ0gK6IMQt7cvLroMSOoXkeu\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1694710849,\n", - " \"model\": \"gpt-35-turbo\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"finish_reason\": \"stop\",\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"As an AI language model, I don't have feelings but I'm functioning properly. Thank you for asking! How can I assist you today?\"\n", - " }\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"completion_tokens\": 29,\n", - " \"prompt_tokens\": 14,\n", - " \"total_tokens\": 43\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "import os\n", - "from litellm import completion\n", - "\n", - "# Function to make both OpenAI and Azure completions\n", - "def make_completions():\n", - " # Set your OpenAI API key\n", - " os.environ[\"OPENAI_API_KEY\"] = \"\"\n", - "\n", - " # OpenAI completion\n", - " openai_response = completion(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n", - " )\n", - "\n", - " print(\"OpenAI Response:\", openai_response)\n", - "\n", - " # Set your Azure OpenAI API key and configuration\n", - " os.environ[\"AZURE_API_KEY\"] = \"\"\n", - " os.environ[\"AZURE_API_BASE\"] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n", - " os.environ[\"AZURE_API_VERSION\"] = \"2023-05-15\"\n", - "\n", - " # Azure OpenAI completion\n", - " azure_response = completion(\n", - " model=\"azure/your-azure-deployment\",\n", - " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}]\n", - " )\n", - "\n", - " print(\"Azure OpenAI Response:\", azure_response)\n", - "\n", - "# Call the function to make both completions in one thread\n", - "make_completions()\n" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/LiteLLM_Bedrock.ipynb b/cookbook/LiteLLM_Bedrock.ipynb deleted file mode 100644 index eed6036392d..00000000000 --- a/cookbook/LiteLLM_Bedrock.ipynb +++ /dev/null @@ -1,310 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "fNkMBurtxawJ" - }, - "source": [ - "# LiteLLM Bedrock Usage\n", - "Important Note: For Bedrock Requests you need to ensure you have `pip install boto3>=1.28.57`, boto3 supports bedrock from `boto3>=1.28.57` and higher " - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "htAufI28xeSy" - }, - "source": [ - "## Pre-Requisites" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "jT5GbPjAuDTp" - }, - "outputs": [], - "source": [ - "!pip install litellm\n", - "!pip install boto3>=1.28.57 # this version onwards has bedrock support" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "H4Vu4er2xnfI" - }, - "source": [ - "## Set Bedrock/AWS Credentials" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "id": "CtTrBthWxp-t" - }, - "outputs": [], - "source": [ - "import os\n", - "os.environ[\"AWS_ACCESS_KEY_ID\"] = \"\" # Access key\n", - "os.environ[\"AWS_SECRET_ACCESS_KEY\"] = \"\" # Secret access key\n", - "os.environ[\"AWS_REGION_NAME\"] = \"\"" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "ycRK9NUdx1EI" - }, - "source": [ - "## Anthropic Requests" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "tgkuoHa5uLOy", - "outputId": "27a78e86-c6a7-4bcc-8559-0813cb978426" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Claude instant 1, response\n", - "{\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" I'm doing well, thanks for asking!\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-4f2e64a1-56d2-43f2-90d3-60ffd6f5086d\",\n", - " \"created\": 1696256761.3265705,\n", - " \"model\": \"anthropic.claude-instant-v1\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 11,\n", - " \"completion_tokens\": 9,\n", - " \"total_tokens\": 20\n", - " },\n", - " \"finish_reason\": \"stop_sequence\"\n", - "}\n", - "Claude v2, response\n", - "{\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" I'm doing well, thanks for asking!\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-34f59b33-f94e-40c2-8bdb-f4af0813405e\",\n", - " \"created\": 1696256762.2137017,\n", - " \"model\": \"anthropic.claude-v2\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 11,\n", - " \"completion_tokens\": 9,\n", - " \"total_tokens\": 20\n", - " },\n", - " \"finish_reason\": \"stop_sequence\"\n", - "}\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "response = completion(\n", - " model=\"bedrock/anthropic.claude-instant-v1\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - ")\n", - "print(\"Claude instant 1, response\")\n", - "print(response)\n", - "\n", - "\n", - "response = completion(\n", - " model=\"bedrock/anthropic.claude-v2\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - ")\n", - "print(\"Claude v2, response\")\n", - "print(response)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "HnM-HtM3yFMT" - }, - "source": [ - "## Anthropic Requests - With Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "_JZvg2yovRsU" - }, - "outputs": [], - "source": [ - "from litellm import completion\n", - "\n", - "response = completion(\n", - " model=\"bedrock/anthropic.claude-instant-v1\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " stream=True,\n", - ")\n", - "print(\"Claude instant 1, response\")\n", - "for chunk in response:\n", - " print(chunk)\n", - "\n", - "\n", - "response = completion(\n", - " model=\"bedrock/anthropic.claude-v2\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " stream=True\n", - ")\n", - "print(\"Claude v2, response\")\n", - "print(response)\n", - "for chunk in response:\n", - " print(chunk)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "zj1U1mh9zEhP" - }, - "source": [ - "## A121 Requests" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "6wK6MZLovU7r", - "outputId": "4cf80c04-f15d-4066-b4c7-113b551538de" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "J2 ultra response\n", - "{\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"\\nHi, I'm doing well, thanks for asking! How about you?\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-f2de678f-0e70-4e36-a01f-8b184c2e4d50\",\n", - " \"created\": 1696257116.044311,\n", - " \"model\": \"ai21.j2-ultra\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 6,\n", - " \"completion_tokens\": 16,\n", - " \"total_tokens\": 22\n", - " }\n", - "}\n", - "J2 mid response\n", - "{\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"\\nGood. And you?\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-420d6bf9-36d8-484b-93b4-4c9e00f7ce2e\",\n", - " \"created\": 1696257116.5756805,\n", - " \"model\": \"ai21.j2-mid\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 6,\n", - " \"completion_tokens\": 6,\n", - " \"total_tokens\": 12\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "response = completion(\n", - " model=\"bedrock/ai21.j2-ultra\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - ")\n", - "print(\"J2 ultra response\")\n", - "print(response)\n", - "\n", - "response = completion(\n", - " model=\"bedrock/ai21.j2-mid\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - ")\n", - "print(\"J2 mid response\")\n", - "print(response)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Y5gGZIwzzSON" - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/LiteLLM_CometAPI.ipynb b/cookbook/LiteLLM_CometAPI.ipynb deleted file mode 100644 index 0a7ab581ae3..00000000000 --- a/cookbook/LiteLLM_CometAPI.ipynb +++ /dev/null @@ -1,474 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "iFEmsVJI_2BR" - }, - "source": [ - "# LiteLLM CometAPI Cookbook" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "cBlUhCEP_xj4" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: litellm in /Users/xmx/.miniforge3/lib/python3.12/site-packages (1.78.2)\n", - "Requirement already satisfied: aiohttp>=3.10 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.11.18)\n", - "Requirement already satisfied: click in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.3.0)\n", - "Requirement already satisfied: fastuuid>=0.13.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.13.3)\n", - "Requirement already satisfied: httpx>=0.23.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.28.1)\n", - "Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.6.1)\n", - "Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.1.6)\n", - "Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (4.25.1)\n", - "Requirement already satisfied: openai>=2.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n", - "Requirement already satisfied: pydantic<3.0.0,>=2.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (2.11.10)\n", - "Requirement already satisfied: python-dotenv>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.1.1)\n", - "Requirement already satisfied: tiktoken>=0.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.12.0)\n", - "Requirement already satisfied: tokenizers in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.22.1)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (2.6.1)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.4.0)\n", - "Requirement already satisfied: attrs>=17.3.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (25.3.0)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.6.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (6.6.3)\n", - "Requirement already satisfied: propcache>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (0.3.1)\n", - "Requirement already satisfied: yarl<2.0,>=1.17.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.20.0)\n", - "Requirement already satisfied: anyio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (4.11.0)\n", - "Requirement already satisfied: certifi in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (2025.10.5)\n", - "Requirement already satisfied: httpcore==1.* in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (1.0.9)\n", - "Requirement already satisfied: idna in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (3.10)\n", - "Requirement already satisfied: h11>=0.16 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpcore==1.*->httpx>=0.23.0->litellm) (0.16.0)\n", - "Requirement already satisfied: zipp>=3.20 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from importlib-metadata>=6.8.0->litellm) (3.21.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jinja2<4.0.0,>=3.1.2->litellm) (3.0.3)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2025.9.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.36.2)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.27.1)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.9.0)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (0.11.0)\n", - "Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.3.1)\n", - "Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.67.1)\n", - "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.15.0)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.33.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (2.33.2)\n", - "Requirement already satisfied: typing-inspection>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.4.2)\n", - "Requirement already satisfied: regex>=2022.1.18 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tiktoken>=0.7.0->litellm) (2025.9.18)\n", - "Requirement already satisfied: requests>=2.26.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tiktoken>=0.7.0->litellm) (2.32.2)\n", - "Requirement already satisfied: huggingface-hub<2.0,>=0.16.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tokenizers->litellm) (0.25.2)\n", - "Requirement already satisfied: filelock in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (3.15.4)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (2025.9.0)\n", - "Requirement already satisfied: packaging>=20.9 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (25.0)\n", - "Requirement already satisfied: pyyaml>=5.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (6.0.3)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken>=0.7.0->litellm) (3.4.0)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken>=0.7.0->litellm) (1.26.20)\n" - ] - } - ], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Completion" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "p-MQqWOT_1a7" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ['COMETAPI_KEY'] = \"Your_CometAPI_Key_Here\"\n", - "api_key = os.getenv('COMETAPI_KEY')" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Ze8JqMqWAARO", - "outputId": "64f3e836-69fa-4f8e-fb35-088a913bbe98" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "ModelResponse(id='msg_017L3DDDit8AkEgHRe2DQBc9', created=1760589916, model='claude-sonnet-4-5-20250929', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='I\\'ll create a simple Python script that says hi.\\n\\n\\nhello.py\\n#!/usr/bin/env python3\\n\"\"\"\\nA simple script that says hi!\\n\"\"\"\\n\\ndef say_hi(name=None):\\n \"\"\"Say hi to someone, or just say hi generally.\"\"\"\\n if name:\\n print(f\"Hi, {name}!\")\\n else:\\n print(\"Hi!\")\\n\\nif __name__ == \"__main__\":\\n # Say hi generally\\n say_hi()\\n \\n # Say hi to someone specific\\n say_hi(\"World\")\\n\\n\\n\\nI\\'ve created a simple Python script called `hello.py` that:\\n\\n1. Defines a `say_hi()` function that can optionally take a name parameter\\n2. Prints \"Hi!\" if no name is provided\\n3. Prints \"Hi, [name]!\" if a name is provided\\n4. Demonstrates both usages when run\\n\\nYou can run it with:\\n```bash\\npython hello.py\\n```\\n\\nThis will output:\\n```\\nHi!\\nHi, World!\\n```\\n\\nWould you like me to create versions in other programming languages, or modify this in any way?', role='assistant', tool_calls=None, function_call=None, provider_specific_fields=None), provider_specific_fields={})], usage=Usage(completion_tokens=290, prompt_tokens=26, total_tokens=316, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=None, text_tokens=None, image_tokens=None, cached_tokens_details={})))" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from litellm import completion\n", - "response = completion(\n", - " model=\"cometapi/claude-sonnet-4-5-20250929\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "-LnhELrnAM_J", - "outputId": "d51c7ab7-d761-4bd1-f849-1534d9df4cd0" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "ModelResponse(id='chatcmpl-CRA9Uo6nsQ9C7kMJv1J4kyNDFJym7', created=1760589916, model='gpt-5-chat-latest', object='chat.completion', system_fingerprint='fp_2da73a467a', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Sure! I can help you write a simple code that prints out \"Hi\" in different programming languages. \\n\\nHere’s an example in **Python**:\\n\\n```python\\n# Simple Python program to say \"Hi\"\\nprint(\"Hi\")\\n```\\n\\nExample in **JavaScript**:\\n\\n```javascript\\n// Simple JavaScript program to say \"Hi\"\\nconsole.log(\"Hi\");\\n```\\n\\nExample in **C**:\\n\\n```c\\n#include \\n\\nint main() {\\n printf(\"Hi\\\\n\");\\n return 0;\\n}\\n```\\n\\nExample in **Java**:\\n\\n```java\\npublic class SayHi {\\n public static void main(String[] args) {\\n System.out.println(\"Hi\");\\n }\\n}\\n```\\n\\nWhich language would you like me to focus on, or do you want me to make it interactive so the program greets the user by name?', role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}, annotations=[]), provider_specific_fields={})], usage=Usage(completion_tokens=174, prompt_tokens=12, total_tokens=186, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=0, cached_tokens=0, text_tokens=None, image_tokens=None)))" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "response = completion(\n", - " model=\"cometapi/gpt-5-chat-latest\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "dJBOUYdwCEn1", - "outputId": "ffa18679-ec15-4dad-fe2b-68665cdf36b0" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "ModelResponse(id='02176058998406949c23b3bf3d52941de23f13565a086747738f0', created=1760589991, model='deepseek-v3.2-exp', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='Here are several ways to say \"hi\" in different programming languages:\\n\\n## Python\\n```python\\nprint(\"Hi!\")\\n```\\n\\n## JavaScript (Browser)\\n```javascript\\nconsole.log(\"Hi!\");\\n// or\\nalert(\"Hi!\");\\n```\\n\\n## JavaScript (Node.js)\\n```javascript\\nconsole.log(\"Hi!\");\\n```\\n\\n## Java\\n```java\\npublic class Hello {\\n public static void main(String[] args) {\\n System.out.println(\"Hi!\");\\n }\\n}\\n```\\n\\n## C\\n```c\\n#include \\n\\nint main() {\\n printf(\"Hi!\\\\n\");\\n return 0;\\n}\\n```\\n\\n## C++\\n```cpp\\n#include \\n\\nint main() {\\n std::cout << \"Hi!\" << std::endl;\\n return 0;\\n}\\n```\\n\\n## C#\\n```csharp\\nusing System;\\n\\nclass Program {\\n static void Main() {\\n Console.WriteLine(\"Hi!\");\\n }\\n}\\n```\\n\\n## PHP\\n```php\\n\\n```\\n\\n## Ruby\\n```ruby\\nputs \"Hi!\"\\n```\\n\\n## Go\\n```go\\npackage main\\n\\nimport \"fmt\"\\n\\nfunc main() {\\n fmt.Println(\"Hi!\")\\n}\\n```\\n\\n## Rust\\n```rust\\nfn main() {\\n println!(\"Hi!\");\\n}\\n```\\n\\n## Swift\\n```swift\\nprint(\"Hi!\")\\n```\\n\\n## Kotlin\\n```kotlin\\nfun main() {\\n println(\"Hi!\")\\n}\\n```\\n\\n## HTML (webpage)\\n```html\\n\\n\\n\\n Hi Page\\n\\n\\n

Hi!

\\n\\n\\n```\\n\\nThe Python version is probably the simplest if you\\'re just getting started!', role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}), provider_specific_fields={})], usage=Usage(completion_tokens=347, prompt_tokens=10, total_tokens=357, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=None, audio_tokens=None, reasoning_tokens=0, rejected_prediction_tokens=None, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None)), service_tier='default')" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "response = completion(\n", - " model=\"cometapi/deepseek-v3.2-exp\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Streaming" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Streaming Responses" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "I'm\n", - " doing\n", - " well\n", - " —\n", - " thanks\n", - " for\n", - " asking\n", - "!\n", - " How\n", - " can\n", - " I\n", - " help\n", - " you\n", - " today\n", - "?\n", - "\n" - ] - } - ], - "source": [ - "messages = [{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}]\n", - "response = completion(model=\"cometapi/gpt-5-mini\", messages=messages, stream=True)\n", - "for part in response:\n", - " print(part.choices[0].delta.content or \"\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Async Completion" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ModelResponse(id='chatcmpl-CRAAkmfczlmCEnCM55D9CKexbRenn', created=1760589994, model='gpt-5-mini-2025-08-07', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"I'm doing well, thanks — how are you? How can I help today?\", role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}, annotations=[]), provider_specific_fields={'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'protected_material_code': {'filtered': False, 'detected': False}, 'protected_material_text': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}})], usage=Usage(completion_tokens=26, prompt_tokens=12, total_tokens=38, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=0, cached_tokens=0, text_tokens=None, image_tokens=None)), prompt_filter_results=[{'prompt_index': 0, 'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'jailbreak': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}}])\n" - ] - } - ], - "source": [ - "from litellm import acompletion\n", - "import asyncio\n", - "\n", - "async def test_get_response():\n", - " user_message = \"Hello, how are you?\"\n", - " messages = [{\"content\": user_message, \"role\": \"user\"}]\n", - " response = await acompletion(model=\"cometapi/gpt-5-mini\", messages=messages)\n", - " return response\n", - "\n", - "response = await test_get_response()\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Async Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "test acompletion + streaming\n", - "response: \n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='Hi', role='assistant', function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' —', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' I', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='’m', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' doing', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' well', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=',', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' thanks', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='!', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' How', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' are', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' you', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='?', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' What', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' can', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' I', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' help', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' you', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' with', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' today', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='?', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason='stop', index=0, delta=Delta(provider_specific_fields=None, content=None, role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None)\n" - ] - } - ], - "source": [ - "from litellm import acompletion\n", - "import asyncio, os, traceback\n", - "\n", - "async def completion_call():\n", - " try:\n", - " print(\"test acompletion + streaming\")\n", - " response = await acompletion(\n", - " model=\"cometapi/gpt-5-mini\", \n", - " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}], \n", - " stream=True\n", - " )\n", - " print(f\"response: {response}\")\n", - " async for chunk in response:\n", - " print(chunk)\n", - " except:\n", - " print(f\"error occurred: {traceback.format_exc()}\")\n", - " pass\n", - "\n", - "await completion_call()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Embedding" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "EmbeddingResponse(model='text-embedding-3-small', data=[{'object': 'embedding', 'index': 0, 'embedding': [-0.018048199, 0.0047550877, -0.013976435, -0.021936804, -0.038773336, -0.03708264, 0.03854791, -0.0007172257, 0.026473511, -0.0027438616, -0.019823432, -0.011947598, -0.013426959, -0.0059914105, 0.020485623, 0.04269012, -0.028276922, -0.015216281, -0.03325039, 0.045057096, 0.0037477135, 0.015793936, -0.005188329, -0.0071713766, 0.008446445, 0.0070938864, 0.0027632343, 0.025656339, 0.022091785, -0.026797561, -0.029840818, -0.0542714, 0.017907308, -0.03454659, -0.014582269, 0.0429719, 0.03575826, 0.007939235, -0.010123054, -0.029587213, 0.018921727, 0.022556728, 0.019598005, 0.008819807, 0.01655475, 0.043310042, -0.034321167, 0.004441604, 0.032686826, 0.047226828, -0.0043253684, 0.006681779, 0.008995921, 0.06593721, 0.0066078105, -0.023190739, -0.00777721, 0.049875587, -0.028023317, -0.019386668, -0.0013252605, 0.009214303, -0.0055828253, -0.007432026, -0.01199691, 0.0054384116, -0.024247425, 0.047255006, -0.013743965, 0.012799991, 0.009883538, -0.005579303, -0.012616833, -0.014709071, -0.004473305, -0.022204498, 0.010454148, -0.02202134, 0.034631126, 0.0097567355, 0.041478455, -0.010750021, -0.0005340668, -0.041450277, -0.04516981, 0.014709071, -0.06142869, 0.021640932, -0.0008752884, -0.0012671428, -0.045254346, -0.004216178, -0.028220564, -0.011074071, 0.010693664, -0.0029481545, -0.039590508, -0.030178957, -0.019062618, -0.03539194, 0.019978413, 0.019161243, -0.025924034, 0.0058329077, -0.029305428, 0.028840488, 0.02731886, -0.0008048426, -0.056582022, 0.003043256, -0.08459125, -0.012560476, 0.00081100664, 0.0015771041, 0.015188103, -0.0049206354, 0.046888687, -0.027107522, 0.019245777, -0.011412211, 0.039336905, -0.018611765, 0.0429719, -0.04302826, -0.018132735, -0.0074883825, -0.02035882, -0.073038146, -0.042380158, 0.04485985, 0.03361671, -0.033842135, -0.0017259207, -0.0017091898, -0.049199305, -0.024148801, -0.044803493, 0.040943068, -0.03093977, 0.045057096, -0.0186963, -0.014053926, -0.009848315, -0.0070480965, -0.0060054995, -0.02813603, -0.022105874, 0.010235767, -0.004476827, 0.027220234, -0.026290352, 0.0216832, -0.05559578, 0.042464696, 0.0253182, -0.0031594916, 0.02266944, -0.030798879, -0.02896729, -0.0005891025, 0.004061197, -0.042295624, -0.008383043, 0.017146494, -0.025797231, -0.046522368, 0.044127215, 0.021105545, -0.04302826, 0.024430584, -0.014934498, -0.01556851, -0.075743265, -0.022415835, -0.028586883, 0.032743182, 0.03539194, -0.0034483192, -0.04669144, 0.051932603, 0.021626843, 0.03127791, 0.015610777, -0.016470214, -0.0056744046, -0.012743635, 0.060132485, 0.017428277, -0.0039942735, -0.017118316, 0.025106862, 0.008974788, 0.018513141, 0.0016035212, -0.0049593803, 0.0017514573, 0.031221554, -0.034856554, -0.011461522, 0.04621241, 0.044239927, 0.034715664, -0.0121941585, -0.012053267, -0.08526753, -0.011813751, -0.025769053, 0.0125182085, -0.0046670306, 0.038266126, 0.1187997, 0.005787118, -0.030038064, -0.054553185, -0.041506633, -0.024078354, 0.0071854657, -0.013391736, 0.03192601, -0.059625275, 0.0023458432, 0.027924692, 0.09163582, 0.030967949, 0.017639615, 0.01489223, 0.029559033, 0.042943723, 0.0003306547, 0.0047198646, 0.029897174, 0.00012603184, 0.0046811197, -0.0427183, 0.01789322, 0.018175002, -0.0081857955, 0.02581132, 0.009890582, 0.03840702, -0.094453655, -0.01076411, 0.06858598, 0.041647524, 0.033532172, 0.007467249, -0.008235107, -0.030967949, 0.0151317455, 0.027361127, 0.011834885, 0.008707094, 0.008178751, -0.022458103, 0.02844599, 0.003605061, -0.02399382, 0.05212985, 0.06041427, -0.023317542, -0.013335379, -0.044099037, -0.040802173, -0.0047656544, -0.023909286, 0.017315563, 0.017428277, 0.00736158, -0.0016070436, -0.055454887, -0.038012523, -0.020626513, 0.018273626, 0.03260229, -0.016991513, 0.038463376, -0.022458103, -0.0109613575, -0.021810003, 0.04846667, -0.042521052, 0.008601425, -0.019259866, -0.0040048403, -0.03308132, -0.02499415, 0.026783472, -0.032884073, 0.021824092, 0.013145176, -0.009186125, -0.01769597, 0.03240504, -0.015343083, -0.012539342, 0.03578644, -0.012299826, 0.011898286, 0.035730083, 0.058441788, 0.032010544, 0.048720278, 0.012926794, -0.0015207474, -0.03313768, 0.014540002, 0.020189751, 0.00029058868, 0.011531969, -0.022514459, 0.019752987, -0.037956167, 0.005272864, -0.042295624, -0.08521117, -0.03494109, 0.053313337, 0.029981708, -0.008150573, -0.053200625, 0.059681635, -0.035476476, 0.034828376, 0.00087881065, 0.025712697, 0.018668123, 0.03212326, 0.008474623, 0.017836861, 0.004910068, 0.016174342, -0.059681635, 0.04004136, 0.00753065, 0.008150573, -0.038012523, 0.0051178834, 0.012525253, 0.022119964, 0.030235313, 0.008242152, -0.01835816, -0.003150686, 0.010714797, -0.0033162334, -0.028882755, -0.06836055, 0.056159347, 0.013624207, 0.0077349427, 0.0066183778, 0.018583586, -0.008883208, -0.046550546, 0.046945043, -0.07393985, -0.017343743, 0.029530855, -0.010982491, 0.008129438, 0.009700378, -0.024613742, -0.0030097943, 0.0078053884, -0.006438741, 0.04770586, -0.008221018, 0.01654066, 0.02498006, -0.015793936, -0.010827511, 0.02399382, 0.03192601, 0.022923045, -0.029192716, 0.006724046, -0.04601516, 0.038519733, 0.031531516, 0.019443026, 0.000109466084, 0.03525105, -0.027248414, -0.038125236, 0.011771483, -0.007467249, 0.010285079, 0.01670973, -0.007861745, 0.026868006, 0.052327096, 0.026374886, -0.03905512, 0.031193376, -0.053566944, 0.04257741, -0.004670553, 0.0168788, 0.035166513, -0.057878222, 0.07095295, -0.009749691, -0.0137510095, -0.02151413, -0.02431787, 0.010073741, -0.05176353, -0.02083785, 0.003959051, -0.02682574, 0.062104966, -0.011461522, 0.04170388, 0.0076363184, 0.026177637, 0.0144413775, 0.014821785, -0.00046890447, 0.0050544823, 0.00032228927, -0.038970586, -0.011355854, -0.056300238, -0.04302826, -0.003545182, 0.04021043, 0.0051108385, -0.048438493, 0.00252548, -0.07692675, -0.0012433673, 0.0054278444, 0.029305428, -0.016188432, -0.003263399, -0.046156053, 6.031917e-05, 0.060977835, -0.016611107, -0.010637308, -0.012602744, 0.016442036, -0.051509928, -0.016991513, 0.0019407802, 0.019161243, 0.045282524, 0.031869654, -0.036941748, -0.035814617, -0.017850952, -0.027192056, -0.049734693, -0.020964652, 0.0228526, -0.025050506, 0.023472521, 0.025740875, -0.017738238, -0.009813092, -0.030883415, -0.012405495, -0.03277136, -0.029502677, 0.016780175, -0.04421175, -0.0020816717, 0.010341435, 0.059230782, -0.041901127, -0.04119667, 0.025924034, 0.02334572, -0.0008435878, 0.020654691, -0.022753974, 0.010700708, -0.013856677, -0.0121941585, -0.011391076, 0.006590199, 0.0050227814, -0.007960369, 0.0008418266, -0.0198657, 0.10781016, -0.0384352, -0.019147152, 0.0057237167, -0.0038181592, -0.047424074, -0.009341106, 0.018499052, -0.016906979, 0.005642704, -0.01837225, -0.038125236, -0.024895526, -0.010285079, -0.055708494, 0.014173684, -0.019724809, 0.00024215724, 0.04500074, 0.048804812, 0.009777869, -0.006572588, 0.008277375, 0.012328005, -0.012609788, 0.026079014, -0.012990195, 0.017963665, -0.007312268, -0.0015682983, 0.05446865, -0.01258161, 0.00035376972, -0.011299497, -0.036321826, -0.0071854657, 0.012969062, 0.026558045, -0.051819887, -0.0029146927, -0.044606246, -0.010383703, -0.03919601, 0.013624207, 0.0030978515, 0.0121941585, -0.0022225631, 0.0512845, -0.0029780937, -0.025191398, -0.015751667, -0.021006921, 0.0039520063, 0.04418357, 0.020570157, 0.00083390146, 0.020541979, -0.004807922, -0.0114263, -0.036152754, 0.018428607, -0.032658648, -0.002035882, 0.013828499, 0.03144698, -0.0003275727, -0.029756282, 0.008488712, 0.0041879993, 0.027826069, 0.0007273523, -0.018949905, -0.0029023646, 0.007861745, 0.011398122, 0.0125322975, 0.014976765, 0.006318983, -0.0066536004, -0.042915545, 0.025867676, -0.015272637, 0.034602948, 0.050241902, 0.014582269, 0.005987888, 0.015244459, -0.050213724, -0.003212326, 0.01315222, -0.022866689, -0.004772699, 0.035673723, -0.024796901, -0.00699174, -0.002072866, 0.022077696, 0.021147812, 0.005093227, -0.039618686, -0.0049241576, 0.012264604, -0.062104966, -0.0022613083, -0.004339458, 0.065486364, -0.0033320836, 0.02944632, 0.017498722, 0.0033039053, -0.020260196, -0.0154980635, -0.05460954, -0.03626547, 0.0072629564, 0.0028900367, 1.2954037e-05, 0.01769597, -0.0045930627, 0.022260854, 0.0027192058, -0.0010566862, -0.0005212985, 0.012158935, -0.0017312041, -0.035110157, -0.0036032998, -0.02317665, -0.01639977, 0.010327346, 0.018259536, -0.011095204, -0.00061904197, -0.023134382, -0.011989865, 0.0025924034, 0.0056708823, 0.03110884, -0.013462181, -0.021105545, 0.010376658, -0.010017385, -0.025106862, 0.026093103, 0.018456785, -0.02134506, 0.0066993902, 0.011891241, -0.010017385, -0.012687278, -0.017132405, 0.04717047, 0.012475941, -0.018752657, -0.008657781, 0.005276386, -0.02582541, 0.02913636, -0.0193444, -0.01101067, 0.029305428, 0.011736261, 0.043140974, 0.02135915, 0.00089422066, 0.009827181, 0.013638296, 0.013884856, -0.014004614, 0.010285079, 0.008108305, -0.04035132, -0.02978446, 0.008481667, -0.022289034, 0.01621661, -0.0057941624, -0.019090796, -0.01852723, -0.022923045, 0.0077208537, -0.039985005, -0.017428277, -0.009460864, 0.018301804, 0.0014397348, 0.04815671, -0.012187114, 0.018879458, 0.021739556, 0.018414518, -0.013462181, -0.06368295, 0.0057096276, 0.013088819, 0.0061640027, 0.031193376, -0.008728228, -0.019245777, 0.010735931, 0.012454808, 0.0397314, -0.017597347, 0.012278693, 0.0130465515, -0.025473181, -0.03215144, -0.0053292206, -0.0027068777, 0.014068015, -0.028079674, 0.016498392, 0.015159924, -0.009207259, -0.02334572, -0.0013710503, 0.008488712, 0.0012231142, 0.0020464489, -0.025149131, -0.021063277, -0.014427288, -0.035222873, 0.051030897, 0.016103897, 0.0063401167, -0.03093977, -0.004684642, -0.0070199184, -0.008495756, -0.0038674714, -0.012222337, -0.022556728, 0.0036015387, -0.040943068, -0.011362898, 0.016794264, 0.017766416, -0.014194817, 0.0011755633, -0.039759576, 0.011384032, -0.0006318103, 0.008298509, 0.04449353, 0.0004838742, 0.016935157, -0.011341765, 0.016864711, -0.00027892113, 0.0009140335, -0.031306088, -0.049452912, -0.0068367594, -0.00011216283, -0.005079138, -0.014420244, 0.01803411, 0.03984411, -0.026276262, -0.0011077593, -0.00063313113, -0.006301372, -0.019992502, 0.0064316965, -0.024289692, 0.0120039545, 0.0068649375, -0.017724149, -0.015667133, -0.0036490895, -0.007953324, 0.024627833, 0.024402406, 0.021810003, -0.015977094, 0.010524594, -0.0060054995, 0.0414221, -0.048551206, 0.01472316, 0.015427617, 0.0029217373, 0.012666144, 0.0048995013, 0.007326357, -0.04187295, -0.0064176074, -0.00674518, 0.0047762212, -0.053059734, -0.09541172, 0.022063607, 0.029530855, 0.01556851, 0.011292453, -0.0038709936, -0.0055370354, -0.016005272, -0.0035170037, -0.0572583, 0.038632445, 0.007981502, -0.005434889, -0.023895197, 0.0021380284, -0.0015084195, 0.016117986, 0.005434889, -0.014694982, -0.007104453, 0.011595369, -0.055229463, 0.0036455672, 0.0027104, -0.010052607, -0.023697948, -0.016315235, -0.002757951, 0.039505973, 0.011095204, 0.0002681341, 0.058948997, -0.0074883825, 0.0050122146, 0.040604927, 0.012912705, -0.025078684, 0.040464036, -0.008925476, -0.00876345, -0.040633105, -0.009024099, 0.024796901, 0.03592733, 0.03626547, -0.029474499, -0.00055431994, 0.0010839839, 0.016737908, 0.013286067, -0.005441934, 0.0059420983, -0.0121941585, 0.015089478, -0.010186454, -0.03477202, -0.0076363184, -0.0087141385, 0.0018439173, 0.028065585, -0.022331301, 0.0029516767, -0.045789734, 0.0010672531, 0.018287715, -0.015948916, 0.04849485, 0.0057589393, 0.0066219, 0.002196146, -0.047255006, 0.012116668, 0.02085194, 0.025924034, -0.0036737456, -0.02877004, 0.016906979, -0.037336245, -0.016258877, 0.010883868, -0.003765325, -0.0049523357, -0.002613537, -0.03263047, 0.023204828, 0.0049946033, -0.007692675, -0.034236632, 0.034095738, 0.020133393, 0.019259866, -0.014103238, 0.024599653, 0.005889264, 0.02430378, 0.0111233825, -0.018780835, -0.00040550332, 0.020232018, 0.03806888, 0.009890582, 0.032376863, 0.031052483, 0.01871039, 0.03891423, -0.0009739124, 0.002759712, 0.017498722, -0.01158128, -0.0045578396, 0.02744566, 0.06497915, 0.024853257, 0.004709298, 0.016667463, -0.00066263025, -0.018132735, -0.013138131, -0.01124314, -0.0125182085, -0.0038111147, 0.03361671, -0.007270001, 0.0012011, -0.01771006, -0.00039999973, 0.024021998, 0.0027896515, 0.0024744067, 0.0013965869, -0.05939985, 0.0014150789, -0.0052517303, 0.052524347, 0.015779847, -0.03327857, 0.042633764, 0.0059420983, -0.023387987, 0.0039097387, -0.028023317, -0.011863063, 0.004378203, 0.02052789, -0.063626595, -0.014864052, 0.014293442, -0.00015938349, -0.007932191, -0.0010954313, 0.023528878, -0.007467249, 0.0059667546, 0.017132405, 0.005730761, -0.00020495309, -0.032038722, 0.0036631785, 0.042915545, -0.029925352, 0.015667133, 0.018935816, -0.0072065997, 0.01556851, -0.025473181, 0.017625526, -0.0026698937, -0.007446115, -0.008622559, -0.043422755, -0.020133393, -0.0039801844, 0.01489223, -0.021655021, 0.015357172, -0.03640636, -0.005663838, -0.028530527, 0.0022648307, -0.00043015933, 0.043591827, -0.015526242, 0.011870108, -0.02530411, -0.016315235, -0.00032316984, -0.030150779, -0.0052552526, 0.020372909, 0.0075024716, 0.0104330145, -0.00055608107, -0.026248084, -0.015202192, -0.03341946, 0.031559695, -0.0012046222, 0.07185466, -0.039590508, 0.022979401, 0.05810365, 0.014025748, -0.029756282, -0.022866689, 0.0073897582, 0.037618026, -0.004180955, -0.0051566283, 0.009728557, -0.03604004, 0.040633105, 0.0026963109, -0.0054172776, 0.034095738, -0.00595971, 0.040943068, -0.031390622, 0.055962097, 0.02117599, -0.012912705, -0.019626183, 0.055877563, 0.017343743, -0.0035416598, 0.013257889, -0.0186963, 0.01656884, -0.06396473, -0.0055405577, 0.020767406, -0.0046564634, 0.045085277, -0.009221348, 0.013645341, 0.008777539, 0.004730432, -0.018625854, -0.011067026, 0.021500042, -0.015047211, 0.004600107, -0.0014344514, -0.0023740216, -0.016188432, 0.006209792, 0.0011993388, 0.004180955, -0.017160583, 0.014497734, 0.015371261, 0.018259536, -0.028333278, -0.008390088, 0.041929305, 0.003923828, 0.02550136, -0.003300383, -0.008058993, -0.010418925, 0.058216363, 0.01885128, -0.02020384, 0.002858336, -0.009806047, -0.022274945, 0.0070445742, 0.026670758, 0.008213974, -0.035307407, -0.027713355, 0.042915545, -0.039675042, -0.0029217373, 0.012053267, -0.003853382, 0.01133472, -0.010073741, 0.005878697, 0.0070938864, -0.035673723, 0.024205158, 0.005896309, 0.030573452, 0.02416289, -0.0072911344, 0.01738601, 0.017005602, -0.02846008, 0.0030344503, 0.018794924, -0.0148076955, -0.0344057, 0.025430914, 0.033503994, -0.0050580045, 0.0077138087, 0.03243322, 0.01372283, -0.005441934, 0.0073404466, -0.0007832686, -0.04767768, 0.0070480965, 0.015145835, 0.026233995, -0.01670973, -0.019513471, -0.014849963, 0.007953324, -0.0032176094, 0.006572588, -0.0012477703, 0.004230267, 0.004476827, -0.021810003, -0.030009886, -0.019273955, -0.0030414949, -0.002918215, 0.060639694, 0.024641922, 0.010327346, 0.026558045, 0.018921727, -0.025867676, -0.016117986, 0.023881108, 0.025360467, 0.009770825, 0.03792799, -0.022429924, 0.033363104, -0.0018914682, 0.04040768, 0.018484963, 0.0070199184, -0.017583257, 0.016258877, 0.010954313, -0.008939565, -0.024148801, -0.02498006, -0.007889924, 0.02748793, 0.0307707, 0.029756282, 0.0051425393, 0.0045719286, -0.03046074, 0.013596028, 0.025684519, -0.0033197557, 0.006967084, 0.03677268, 0.0120039545, -0.0032792494, -0.0032211316, -0.02399382, -0.026924362, -0.013920079, -0.0042197, 0.025346378, -0.0027015943, -0.016991513, 0.0031594916, -0.007579962, 0.018978084, 0.017681882, 0.0126591, 0.028939111, 0.008833896, 0.10183637, 0.0059632324, -0.05196078, -0.023697948, 0.011045893, -0.008777539, -0.013807366, 0.019273955, -0.025346378, 0.0074742935, 0.009961028, -0.010813422, 0.018597675, 0.009636978, 0.014948587, 0.024064265, -0.008693005, -0.020570157, 0.014194817, -0.026219906, -0.02299349, 0.011067026, 0.032066904, 0.013391736, -0.05148175, -0.009489042, -0.03062981, -0.0012847543, 0.07286908, 0.026529867, -0.00025008238, 0.013638296, 0.016089808, 0.018654034, -0.0020394044, -0.024543297, 0.0147795165, -0.009601755, 0.0018791402, -0.040520392, -0.003360262, 0.02216223, 0.0137650985, 0.0059914105, 0.0048361, -0.0009844792, 0.016977424, -0.00934815, 0.024233336, -0.013088819, -0.017555078, -0.0050263037, 0.010595039, -0.027516108, 0.0071537653, -0.023247095, -0.0017655464, -0.015948916, 0.058160007, -0.025966302, 0.0121941585, -0.012384362, -0.0015612538, 0.009946939, 0.00628376, 0.011327676, 0.0109613575, 0.008601425, -0.018329982, 0.055680316, -0.012778858, -0.0100807855, -0.011067026, -0.0036490895, -0.01356785, 0.0073193125, -0.014272308, -0.027403394, -0.030742522, 0.02862915, 0.03062981, -0.014596358, 0.021697288, 0.0042408337, 0.027572464, 0.0019601528, -0.037138995, -0.031306088, 0.041929305, 0.017738238, 0.004857234, 0.008256241, 0.0118278405, 0.021753646, -0.00160176, -0.0018333505, -0.0047374764, 0.042239267, 0.0058329077, -0.026459422, 0.015075389, 0.021147812, -0.005212985, 0.01281408, 0.017738238, 0.008242152, -0.020372909, -0.011081115, -0.011017715, 0.007706764, 0.01834407, 0.01954165, 0.037477136, -0.010278034, 0.015808025, 0.00031590514, -0.017681882, -0.008967743, -0.020612424, -0.025416825, -0.0037970257, -0.029868996, 0.01720285, -0.0144554665, 0.026727116, 0.00414221, 0.0040154075, 0.05838543, 0.0005622451, -0.025219576, 0.004180955, -0.002932304, -0.0090663675, 0.011574236, 0.02450103, -0.012553431, -0.020612424, -0.032095082, 0.015526242, 0.008974788, 0.0053151315, -0.0003112821, -0.017935487, -0.0076222294, 0.03358853, 0.029474499, -0.011496745, -0.012835215, -0.020739228, -0.012482986, -0.037871633, 0.0052517303, -0.012926794, -0.0025237189, 0.0020323596, 0.045113456, -0.04835396, -0.027755624, -0.0079955915, 0.007896968, 0.0072559114, 0.015047211, -0.0014573464, -0.014032792, 0.021091456, -0.0046071517, -0.0065232757, -0.02582541, -0.035870973, -0.015343083, 0.03254593, -0.028431902, -0.003286294, 0.014328664, 0.008840941, 0.015948916, 0.012835215, 0.019400757, -0.012342094, -0.010693664, 0.004772699, -0.03254593, 0.010707753, -0.016822444, -0.0032827717, 0.021246437, -0.04485985, -0.04384543, -0.015906649, -0.009707424, 0.02299349, 0.019513471, -0.010151232, 0.018963994, -0.0057976847, 0.05739919, -0.019922055, -0.029108182, -0.0106232185, 0.021077367, 0.0036455672, -0.026614401, 0.04497256, -0.04446535, -0.0004556959, -0.004578973, 0.003962573, -0.004910068, 0.015089478, -0.0301226, 0.007664497, 0.008375999, 0.031982366, 0.006135824, 0.02152822, -0.015469885, -0.007210122, 0.034715664, -0.01233505, 0.0004490916, -0.0144413775, -0.003150686, -0.02003477, -0.027924692, -0.0015850292, -0.009376328, -0.0035997776, -0.03240504, -0.010912046, 0.0031999978, 0.022303123, -0.008988877, 0.00024633997, -0.0035698381, 0.0070974086, -0.002599448, -0.042267445, -0.016935157, -0.0002481011, -0.041393917, 0.014483645, 0.019006262, -0.02813603, 0.0072030774, -7.3032425e-05, 0.01802002, -0.017188761, 0.015991183, 0.020401087, 0.03542012, 0.04469078, 0.04071764, 0.011095204, -0.031390622, -0.03254593, 0.014187773, 0.016272966, -0.009721513, -0.026388975, -0.014849963, -0.005642704, -0.022556728, 0.0064457855, -0.043450933, 0.010834555, -0.015977094, 0.020880118, -0.02385293, -0.054806788, 0.03789981, 0.0013516777, -0.026431242, -0.015540331, 0.016695641, -0.037167173, -0.021190079, 0.023881108, -0.0045860177, 0.0064105624, -0.007763121, -0.013053596, 0.024472851, -0.0004962022, -0.00976378, 0.060019772, -0.0057624616, -0.04384543, 0.010313257, 0.0076715415, 0.0025888812, -0.03589915, 0.008791628, -0.012785902, 0.01042597, 0.015653044, 0.04767768, -0.009869449, 0.0064457855, -0.010947268, -0.0077349427, -0.032715004, -0.023867019, -0.011327676, -0.00046274049, -0.036998104, 0.013913034, 0.012250515, -0.009996251, 0.021204168, 0.020091126, -0.003740669, -0.0049769916, -0.0140891485, 0.024064265, 0.0038815604, 0.025684519, 0.041788414, -0.013553761, 0.006681779, -0.0050826604, -0.018175002, 0.008228063, -0.006230926, -0.018907638, 0.0154839745, -0.028713685, -0.015047211, -0.019682541, 0.02516322, 0.040802173, 0.007213644, 0.011743305, -0.015963005, -0.03818159, 0.01191942, -0.031728763, -0.011863063, 0.023881108, 0.0053116092, -0.020992832, -0.017991843, -0.00405063, -0.017780505, -0.0057659843, 0.02978446, 0.031165197, 0.0014221234, 0.021316882, 0.026008569, -0.0018544842, -0.032658648, 0.028474169, 0.013109953, 0.018076377, 0.0007991189, -0.0042373114, 0.028910933, -0.0029358263, 0.021866359, 0.024472851, -0.002576553, -0.033532172, 0.01920351, -0.0095665315, -0.03093977, 0.0034817809, 0.018654034, -0.0074038478, 0.021443684, 0.0038604268, -0.02745975, 0.031587873, 0.0061146906, 0.022711707, -0.019795254, -0.016991513, -0.04471896, -0.007875834, -0.0034941088, -0.043789074, 0.021091456, 0.024909616, -0.013194487, -0.0042690123, 0.027896514, -0.018414518, -0.023303451, -0.025797231, -0.009524264]}], object='list', usage=Usage(completion_tokens=0, prompt_tokens=3, total_tokens=3, completion_tokens_details=None, prompt_tokens_details=None))\n" - ] - } - ], - "source": [ - "import litellm\n", - "\n", - "\n", - "async def main():\n", - " response = await litellm.aembedding(\n", - " model=\"cometapi/text-embedding-3-small\", # The model name must include prefix \"openai\" + the model name from ai/ml api\n", - " api_key=api_key, # your aiml api-key\n", - " api_base=\"https://api.cometapi.com/v1\", # 👈 the URL has changed from v2 to v1\n", - " input=\"Your text string\",\n", - " )\n", - " print(response)\n", - "\n", - "await main()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "EmbeddingResponse(model='text-embedding-3-small', data=[{'object': 'embedding', 'index': 0, 'embedding': [-0.018048199, 0.0047550877, -0.013976435, -0.021936804, -0.038773336, -0.03708264, 0.03854791, -0.0007172257, 0.026473511, -0.0027438616, -0.019823432, -0.011947598, -0.013426959, -0.0059914105, 0.020485623, 0.04269012, -0.028276922, -0.015216281, -0.03325039, 0.045057096, 0.0037477135, 0.015793936, -0.005188329, -0.0071713766, 0.008446445, 0.0070938864, 0.0027632343, 0.025656339, 0.022091785, -0.026797561, -0.029840818, -0.0542714, 0.017907308, -0.03454659, -0.014582269, 0.0429719, 0.03575826, 0.007939235, -0.010123054, -0.029587213, 0.018921727, 0.022556728, 0.019598005, 0.008819807, 0.01655475, 0.043310042, -0.034321167, 0.004441604, 0.032686826, 0.047226828, -0.0043253684, 0.006681779, 0.008995921, 0.06593721, 0.0066078105, -0.023190739, -0.00777721, 0.049875587, -0.028023317, -0.019386668, -0.0013252605, 0.009214303, -0.0055828253, -0.007432026, -0.01199691, 0.0054384116, -0.024247425, 0.047255006, -0.013743965, 0.012799991, 0.009883538, -0.005579303, -0.012616833, -0.014709071, -0.004473305, -0.022204498, 0.010454148, -0.02202134, 0.034631126, 0.0097567355, 0.041478455, -0.010750021, -0.0005340668, -0.041450277, -0.04516981, 0.014709071, -0.06142869, 0.021640932, -0.0008752884, -0.0012671428, -0.045254346, -0.004216178, -0.028220564, -0.011074071, 0.010693664, -0.0029481545, -0.039590508, -0.030178957, -0.019062618, -0.03539194, 0.019978413, 0.019161243, -0.025924034, 0.0058329077, -0.029305428, 0.028840488, 0.02731886, -0.0008048426, -0.056582022, 0.003043256, -0.08459125, -0.012560476, 0.00081100664, 0.0015771041, 0.015188103, -0.0049206354, 0.046888687, -0.027107522, 0.019245777, -0.011412211, 0.039336905, -0.018611765, 0.0429719, -0.04302826, -0.018132735, -0.0074883825, -0.02035882, -0.073038146, -0.042380158, 0.04485985, 0.03361671, -0.033842135, -0.0017259207, -0.0017091898, -0.049199305, -0.024148801, -0.044803493, 0.040943068, -0.03093977, 0.045057096, -0.0186963, -0.014053926, -0.009848315, -0.0070480965, -0.0060054995, -0.02813603, -0.022105874, 0.010235767, -0.004476827, 0.027220234, -0.026290352, 0.0216832, -0.05559578, 0.042464696, 0.0253182, -0.0031594916, 0.02266944, -0.030798879, -0.02896729, -0.0005891025, 0.004061197, -0.042295624, -0.008383043, 0.017146494, -0.025797231, -0.046522368, 0.044127215, 0.021105545, -0.04302826, 0.024430584, -0.014934498, -0.01556851, -0.075743265, -0.022415835, -0.028586883, 0.032743182, 0.03539194, -0.0034483192, -0.04669144, 0.051932603, 0.021626843, 0.03127791, 0.015610777, -0.016470214, -0.0056744046, -0.012743635, 0.060132485, 0.017428277, -0.0039942735, -0.017118316, 0.025106862, 0.008974788, 0.018513141, 0.0016035212, -0.0049593803, 0.0017514573, 0.031221554, -0.034856554, -0.011461522, 0.04621241, 0.044239927, 0.034715664, -0.0121941585, -0.012053267, -0.08526753, -0.011813751, -0.025769053, 0.0125182085, -0.0046670306, 0.038266126, 0.1187997, 0.005787118, -0.030038064, -0.054553185, -0.041506633, -0.024078354, 0.0071854657, -0.013391736, 0.03192601, -0.059625275, 0.0023458432, 0.027924692, 0.09163582, 0.030967949, 0.017639615, 0.01489223, 0.029559033, 0.042943723, 0.0003306547, 0.0047198646, 0.029897174, 0.00012603184, 0.0046811197, -0.0427183, 0.01789322, 0.018175002, -0.0081857955, 0.02581132, 0.009890582, 0.03840702, -0.094453655, -0.01076411, 0.06858598, 0.041647524, 0.033532172, 0.007467249, -0.008235107, -0.030967949, 0.0151317455, 0.027361127, 0.011834885, 0.008707094, 0.008178751, -0.022458103, 0.02844599, 0.003605061, -0.02399382, 0.05212985, 0.06041427, -0.023317542, -0.013335379, -0.044099037, -0.040802173, -0.0047656544, -0.023909286, 0.017315563, 0.017428277, 0.00736158, -0.0016070436, -0.055454887, -0.038012523, -0.020626513, 0.018273626, 0.03260229, -0.016991513, 0.038463376, -0.022458103, -0.0109613575, -0.021810003, 0.04846667, -0.042521052, 0.008601425, -0.019259866, -0.0040048403, -0.03308132, -0.02499415, 0.026783472, -0.032884073, 0.021824092, 0.013145176, -0.009186125, -0.01769597, 0.03240504, -0.015343083, -0.012539342, 0.03578644, -0.012299826, 0.011898286, 0.035730083, 0.058441788, 0.032010544, 0.048720278, 0.012926794, -0.0015207474, -0.03313768, 0.014540002, 0.020189751, 0.00029058868, 0.011531969, -0.022514459, 0.019752987, -0.037956167, 0.005272864, -0.042295624, -0.08521117, -0.03494109, 0.053313337, 0.029981708, -0.008150573, -0.053200625, 0.059681635, -0.035476476, 0.034828376, 0.00087881065, 0.025712697, 0.018668123, 0.03212326, 0.008474623, 0.017836861, 0.004910068, 0.016174342, -0.059681635, 0.04004136, 0.00753065, 0.008150573, -0.038012523, 0.0051178834, 0.012525253, 0.022119964, 0.030235313, 0.008242152, -0.01835816, -0.003150686, 0.010714797, -0.0033162334, -0.028882755, -0.06836055, 0.056159347, 0.013624207, 0.0077349427, 0.0066183778, 0.018583586, -0.008883208, -0.046550546, 0.046945043, -0.07393985, -0.017343743, 0.029530855, -0.010982491, 0.008129438, 0.009700378, -0.024613742, -0.0030097943, 0.0078053884, -0.006438741, 0.04770586, -0.008221018, 0.01654066, 0.02498006, -0.015793936, -0.010827511, 0.02399382, 0.03192601, 0.022923045, -0.029192716, 0.006724046, -0.04601516, 0.038519733, 0.031531516, 0.019443026, 0.000109466084, 0.03525105, -0.027248414, -0.038125236, 0.011771483, -0.007467249, 0.010285079, 0.01670973, -0.007861745, 0.026868006, 0.052327096, 0.026374886, -0.03905512, 0.031193376, -0.053566944, 0.04257741, -0.004670553, 0.0168788, 0.035166513, -0.057878222, 0.07095295, -0.009749691, -0.0137510095, -0.02151413, -0.02431787, 0.010073741, -0.05176353, -0.02083785, 0.003959051, -0.02682574, 0.062104966, -0.011461522, 0.04170388, 0.0076363184, 0.026177637, 0.0144413775, 0.014821785, -0.00046890447, 0.0050544823, 0.00032228927, -0.038970586, -0.011355854, -0.056300238, -0.04302826, -0.003545182, 0.04021043, 0.0051108385, -0.048438493, 0.00252548, -0.07692675, -0.0012433673, 0.0054278444, 0.029305428, -0.016188432, -0.003263399, -0.046156053, 6.031917e-05, 0.060977835, -0.016611107, -0.010637308, -0.012602744, 0.016442036, -0.051509928, -0.016991513, 0.0019407802, 0.019161243, 0.045282524, 0.031869654, -0.036941748, -0.035814617, -0.017850952, -0.027192056, -0.049734693, -0.020964652, 0.0228526, -0.025050506, 0.023472521, 0.025740875, -0.017738238, -0.009813092, -0.030883415, -0.012405495, -0.03277136, -0.029502677, 0.016780175, -0.04421175, -0.0020816717, 0.010341435, 0.059230782, -0.041901127, -0.04119667, 0.025924034, 0.02334572, -0.0008435878, 0.020654691, -0.022753974, 0.010700708, -0.013856677, -0.0121941585, -0.011391076, 0.006590199, 0.0050227814, -0.007960369, 0.0008418266, -0.0198657, 0.10781016, -0.0384352, -0.019147152, 0.0057237167, -0.0038181592, -0.047424074, -0.009341106, 0.018499052, -0.016906979, 0.005642704, -0.01837225, -0.038125236, -0.024895526, -0.010285079, -0.055708494, 0.014173684, -0.019724809, 0.00024215724, 0.04500074, 0.048804812, 0.009777869, -0.006572588, 0.008277375, 0.012328005, -0.012609788, 0.026079014, -0.012990195, 0.017963665, -0.007312268, -0.0015682983, 0.05446865, -0.01258161, 0.00035376972, -0.011299497, -0.036321826, -0.0071854657, 0.012969062, 0.026558045, -0.051819887, -0.0029146927, -0.044606246, -0.010383703, -0.03919601, 0.013624207, 0.0030978515, 0.0121941585, -0.0022225631, 0.0512845, -0.0029780937, -0.025191398, -0.015751667, -0.021006921, 0.0039520063, 0.04418357, 0.020570157, 0.00083390146, 0.020541979, -0.004807922, -0.0114263, -0.036152754, 0.018428607, -0.032658648, -0.002035882, 0.013828499, 0.03144698, -0.0003275727, -0.029756282, 0.008488712, 0.0041879993, 0.027826069, 0.0007273523, -0.018949905, -0.0029023646, 0.007861745, 0.011398122, 0.0125322975, 0.014976765, 0.006318983, -0.0066536004, -0.042915545, 0.025867676, -0.015272637, 0.034602948, 0.050241902, 0.014582269, 0.005987888, 0.015244459, -0.050213724, -0.003212326, 0.01315222, -0.022866689, -0.004772699, 0.035673723, -0.024796901, -0.00699174, -0.002072866, 0.022077696, 0.021147812, 0.005093227, -0.039618686, -0.0049241576, 0.012264604, -0.062104966, -0.0022613083, -0.004339458, 0.065486364, -0.0033320836, 0.02944632, 0.017498722, 0.0033039053, -0.020260196, -0.0154980635, -0.05460954, -0.03626547, 0.0072629564, 0.0028900367, 1.2954037e-05, 0.01769597, -0.0045930627, 0.022260854, 0.0027192058, -0.0010566862, -0.0005212985, 0.012158935, -0.0017312041, -0.035110157, -0.0036032998, -0.02317665, -0.01639977, 0.010327346, 0.018259536, -0.011095204, -0.00061904197, -0.023134382, -0.011989865, 0.0025924034, 0.0056708823, 0.03110884, -0.013462181, -0.021105545, 0.010376658, -0.010017385, -0.025106862, 0.026093103, 0.018456785, -0.02134506, 0.0066993902, 0.011891241, -0.010017385, -0.012687278, -0.017132405, 0.04717047, 0.012475941, -0.018752657, -0.008657781, 0.005276386, -0.02582541, 0.02913636, -0.0193444, -0.01101067, 0.029305428, 0.011736261, 0.043140974, 0.02135915, 0.00089422066, 0.009827181, 0.013638296, 0.013884856, -0.014004614, 0.010285079, 0.008108305, -0.04035132, -0.02978446, 0.008481667, -0.022289034, 0.01621661, -0.0057941624, -0.019090796, -0.01852723, -0.022923045, 0.0077208537, -0.039985005, -0.017428277, -0.009460864, 0.018301804, 0.0014397348, 0.04815671, -0.012187114, 0.018879458, 0.021739556, 0.018414518, -0.013462181, -0.06368295, 0.0057096276, 0.013088819, 0.0061640027, 0.031193376, -0.008728228, -0.019245777, 0.010735931, 0.012454808, 0.0397314, -0.017597347, 0.012278693, 0.0130465515, -0.025473181, -0.03215144, -0.0053292206, -0.0027068777, 0.014068015, -0.028079674, 0.016498392, 0.015159924, -0.009207259, -0.02334572, -0.0013710503, 0.008488712, 0.0012231142, 0.0020464489, -0.025149131, -0.021063277, -0.014427288, -0.035222873, 0.051030897, 0.016103897, 0.0063401167, -0.03093977, -0.004684642, -0.0070199184, -0.008495756, -0.0038674714, -0.012222337, -0.022556728, 0.0036015387, -0.040943068, -0.011362898, 0.016794264, 0.017766416, -0.014194817, 0.0011755633, -0.039759576, 0.011384032, -0.0006318103, 0.008298509, 0.04449353, 0.0004838742, 0.016935157, -0.011341765, 0.016864711, -0.00027892113, 0.0009140335, -0.031306088, -0.049452912, -0.0068367594, -0.00011216283, -0.005079138, -0.014420244, 0.01803411, 0.03984411, -0.026276262, -0.0011077593, -0.00063313113, -0.006301372, -0.019992502, 0.0064316965, -0.024289692, 0.0120039545, 0.0068649375, -0.017724149, -0.015667133, -0.0036490895, -0.007953324, 0.024627833, 0.024402406, 0.021810003, -0.015977094, 0.010524594, -0.0060054995, 0.0414221, -0.048551206, 0.01472316, 0.015427617, 0.0029217373, 0.012666144, 0.0048995013, 0.007326357, -0.04187295, -0.0064176074, -0.00674518, 0.0047762212, -0.053059734, -0.09541172, 0.022063607, 0.029530855, 0.01556851, 0.011292453, -0.0038709936, -0.0055370354, -0.016005272, -0.0035170037, -0.0572583, 0.038632445, 0.007981502, -0.005434889, -0.023895197, 0.0021380284, -0.0015084195, 0.016117986, 0.005434889, -0.014694982, -0.007104453, 0.011595369, -0.055229463, 0.0036455672, 0.0027104, -0.010052607, -0.023697948, -0.016315235, -0.002757951, 0.039505973, 0.011095204, 0.0002681341, 0.058948997, -0.0074883825, 0.0050122146, 0.040604927, 0.012912705, -0.025078684, 0.040464036, -0.008925476, -0.00876345, -0.040633105, -0.009024099, 0.024796901, 0.03592733, 0.03626547, -0.029474499, -0.00055431994, 0.0010839839, 0.016737908, 0.013286067, -0.005441934, 0.0059420983, -0.0121941585, 0.015089478, -0.010186454, -0.03477202, -0.0076363184, -0.0087141385, 0.0018439173, 0.028065585, -0.022331301, 0.0029516767, -0.045789734, 0.0010672531, 0.018287715, -0.015948916, 0.04849485, 0.0057589393, 0.0066219, 0.002196146, -0.047255006, 0.012116668, 0.02085194, 0.025924034, -0.0036737456, -0.02877004, 0.016906979, -0.037336245, -0.016258877, 0.010883868, -0.003765325, -0.0049523357, -0.002613537, -0.03263047, 0.023204828, 0.0049946033, -0.007692675, -0.034236632, 0.034095738, 0.020133393, 0.019259866, -0.014103238, 0.024599653, 0.005889264, 0.02430378, 0.0111233825, -0.018780835, -0.00040550332, 0.020232018, 0.03806888, 0.009890582, 0.032376863, 0.031052483, 0.01871039, 0.03891423, -0.0009739124, 0.002759712, 0.017498722, -0.01158128, -0.0045578396, 0.02744566, 0.06497915, 0.024853257, 0.004709298, 0.016667463, -0.00066263025, -0.018132735, -0.013138131, -0.01124314, -0.0125182085, -0.0038111147, 0.03361671, -0.007270001, 0.0012011, -0.01771006, -0.00039999973, 0.024021998, 0.0027896515, 0.0024744067, 0.0013965869, -0.05939985, 0.0014150789, -0.0052517303, 0.052524347, 0.015779847, -0.03327857, 0.042633764, 0.0059420983, -0.023387987, 0.0039097387, -0.028023317, -0.011863063, 0.004378203, 0.02052789, -0.063626595, -0.014864052, 0.014293442, -0.00015938349, -0.007932191, -0.0010954313, 0.023528878, -0.007467249, 0.0059667546, 0.017132405, 0.005730761, -0.00020495309, -0.032038722, 0.0036631785, 0.042915545, -0.029925352, 0.015667133, 0.018935816, -0.0072065997, 0.01556851, -0.025473181, 0.017625526, -0.0026698937, -0.007446115, -0.008622559, -0.043422755, -0.020133393, -0.0039801844, 0.01489223, -0.021655021, 0.015357172, -0.03640636, -0.005663838, -0.028530527, 0.0022648307, -0.00043015933, 0.043591827, -0.015526242, 0.011870108, -0.02530411, -0.016315235, -0.00032316984, -0.030150779, -0.0052552526, 0.020372909, 0.0075024716, 0.0104330145, -0.00055608107, -0.026248084, -0.015202192, -0.03341946, 0.031559695, -0.0012046222, 0.07185466, -0.039590508, 0.022979401, 0.05810365, 0.014025748, -0.029756282, -0.022866689, 0.0073897582, 0.037618026, -0.004180955, -0.0051566283, 0.009728557, -0.03604004, 0.040633105, 0.0026963109, -0.0054172776, 0.034095738, -0.00595971, 0.040943068, -0.031390622, 0.055962097, 0.02117599, -0.012912705, -0.019626183, 0.055877563, 0.017343743, -0.0035416598, 0.013257889, -0.0186963, 0.01656884, -0.06396473, -0.0055405577, 0.020767406, -0.0046564634, 0.045085277, -0.009221348, 0.013645341, 0.008777539, 0.004730432, -0.018625854, -0.011067026, 0.021500042, -0.015047211, 0.004600107, -0.0014344514, -0.0023740216, -0.016188432, 0.006209792, 0.0011993388, 0.004180955, -0.017160583, 0.014497734, 0.015371261, 0.018259536, -0.028333278, -0.008390088, 0.041929305, 0.003923828, 0.02550136, -0.003300383, -0.008058993, -0.010418925, 0.058216363, 0.01885128, -0.02020384, 0.002858336, -0.009806047, -0.022274945, 0.0070445742, 0.026670758, 0.008213974, -0.035307407, -0.027713355, 0.042915545, -0.039675042, -0.0029217373, 0.012053267, -0.003853382, 0.01133472, -0.010073741, 0.005878697, 0.0070938864, -0.035673723, 0.024205158, 0.005896309, 0.030573452, 0.02416289, -0.0072911344, 0.01738601, 0.017005602, -0.02846008, 0.0030344503, 0.018794924, -0.0148076955, -0.0344057, 0.025430914, 0.033503994, -0.0050580045, 0.0077138087, 0.03243322, 0.01372283, -0.005441934, 0.0073404466, -0.0007832686, -0.04767768, 0.0070480965, 0.015145835, 0.026233995, -0.01670973, -0.019513471, -0.014849963, 0.007953324, -0.0032176094, 0.006572588, -0.0012477703, 0.004230267, 0.004476827, -0.021810003, -0.030009886, -0.019273955, -0.0030414949, -0.002918215, 0.060639694, 0.024641922, 0.010327346, 0.026558045, 0.018921727, -0.025867676, -0.016117986, 0.023881108, 0.025360467, 0.009770825, 0.03792799, -0.022429924, 0.033363104, -0.0018914682, 0.04040768, 0.018484963, 0.0070199184, -0.017583257, 0.016258877, 0.010954313, -0.008939565, -0.024148801, -0.02498006, -0.007889924, 0.02748793, 0.0307707, 0.029756282, 0.0051425393, 0.0045719286, -0.03046074, 0.013596028, 0.025684519, -0.0033197557, 0.006967084, 0.03677268, 0.0120039545, -0.0032792494, -0.0032211316, -0.02399382, -0.026924362, -0.013920079, -0.0042197, 0.025346378, -0.0027015943, -0.016991513, 0.0031594916, -0.007579962, 0.018978084, 0.017681882, 0.0126591, 0.028939111, 0.008833896, 0.10183637, 0.0059632324, -0.05196078, -0.023697948, 0.011045893, -0.008777539, -0.013807366, 0.019273955, -0.025346378, 0.0074742935, 0.009961028, -0.010813422, 0.018597675, 0.009636978, 0.014948587, 0.024064265, -0.008693005, -0.020570157, 0.014194817, -0.026219906, -0.02299349, 0.011067026, 0.032066904, 0.013391736, -0.05148175, -0.009489042, -0.03062981, -0.0012847543, 0.07286908, 0.026529867, -0.00025008238, 0.013638296, 0.016089808, 0.018654034, -0.0020394044, -0.024543297, 0.0147795165, -0.009601755, 0.0018791402, -0.040520392, -0.003360262, 0.02216223, 0.0137650985, 0.0059914105, 0.0048361, -0.0009844792, 0.016977424, -0.00934815, 0.024233336, -0.013088819, -0.017555078, -0.0050263037, 0.010595039, -0.027516108, 0.0071537653, -0.023247095, -0.0017655464, -0.015948916, 0.058160007, -0.025966302, 0.0121941585, -0.012384362, -0.0015612538, 0.009946939, 0.00628376, 0.011327676, 0.0109613575, 0.008601425, -0.018329982, 0.055680316, -0.012778858, -0.0100807855, -0.011067026, -0.0036490895, -0.01356785, 0.0073193125, -0.014272308, -0.027403394, -0.030742522, 0.02862915, 0.03062981, -0.014596358, 0.021697288, 0.0042408337, 0.027572464, 0.0019601528, -0.037138995, -0.031306088, 0.041929305, 0.017738238, 0.004857234, 0.008256241, 0.0118278405, 0.021753646, -0.00160176, -0.0018333505, -0.0047374764, 0.042239267, 0.0058329077, -0.026459422, 0.015075389, 0.021147812, -0.005212985, 0.01281408, 0.017738238, 0.008242152, -0.020372909, -0.011081115, -0.011017715, 0.007706764, 0.01834407, 0.01954165, 0.037477136, -0.010278034, 0.015808025, 0.00031590514, -0.017681882, -0.008967743, -0.020612424, -0.025416825, -0.0037970257, -0.029868996, 0.01720285, -0.0144554665, 0.026727116, 0.00414221, 0.0040154075, 0.05838543, 0.0005622451, -0.025219576, 0.004180955, -0.002932304, -0.0090663675, 0.011574236, 0.02450103, -0.012553431, -0.020612424, -0.032095082, 0.015526242, 0.008974788, 0.0053151315, -0.0003112821, -0.017935487, -0.0076222294, 0.03358853, 0.029474499, -0.011496745, -0.012835215, -0.020739228, -0.012482986, -0.037871633, 0.0052517303, -0.012926794, -0.0025237189, 0.0020323596, 0.045113456, -0.04835396, -0.027755624, -0.0079955915, 0.007896968, 0.0072559114, 0.015047211, -0.0014573464, -0.014032792, 0.021091456, -0.0046071517, -0.0065232757, -0.02582541, -0.035870973, -0.015343083, 0.03254593, -0.028431902, -0.003286294, 0.014328664, 0.008840941, 0.015948916, 0.012835215, 0.019400757, -0.012342094, -0.010693664, 0.004772699, -0.03254593, 0.010707753, -0.016822444, -0.0032827717, 0.021246437, -0.04485985, -0.04384543, -0.015906649, -0.009707424, 0.02299349, 0.019513471, -0.010151232, 0.018963994, -0.0057976847, 0.05739919, -0.019922055, -0.029108182, -0.0106232185, 0.021077367, 0.0036455672, -0.026614401, 0.04497256, -0.04446535, -0.0004556959, -0.004578973, 0.003962573, -0.004910068, 0.015089478, -0.0301226, 0.007664497, 0.008375999, 0.031982366, 0.006135824, 0.02152822, -0.015469885, -0.007210122, 0.034715664, -0.01233505, 0.0004490916, -0.0144413775, -0.003150686, -0.02003477, -0.027924692, -0.0015850292, -0.009376328, -0.0035997776, -0.03240504, -0.010912046, 0.0031999978, 0.022303123, -0.008988877, 0.00024633997, -0.0035698381, 0.0070974086, -0.002599448, -0.042267445, -0.016935157, -0.0002481011, -0.041393917, 0.014483645, 0.019006262, -0.02813603, 0.0072030774, -7.3032425e-05, 0.01802002, -0.017188761, 0.015991183, 0.020401087, 0.03542012, 0.04469078, 0.04071764, 0.011095204, -0.031390622, -0.03254593, 0.014187773, 0.016272966, -0.009721513, -0.026388975, -0.014849963, -0.005642704, -0.022556728, 0.0064457855, -0.043450933, 0.010834555, -0.015977094, 0.020880118, -0.02385293, -0.054806788, 0.03789981, 0.0013516777, -0.026431242, -0.015540331, 0.016695641, -0.037167173, -0.021190079, 0.023881108, -0.0045860177, 0.0064105624, -0.007763121, -0.013053596, 0.024472851, -0.0004962022, -0.00976378, 0.060019772, -0.0057624616, -0.04384543, 0.010313257, 0.0076715415, 0.0025888812, -0.03589915, 0.008791628, -0.012785902, 0.01042597, 0.015653044, 0.04767768, -0.009869449, 0.0064457855, -0.010947268, -0.0077349427, -0.032715004, -0.023867019, -0.011327676, -0.00046274049, -0.036998104, 0.013913034, 0.012250515, -0.009996251, 0.021204168, 0.020091126, -0.003740669, -0.0049769916, -0.0140891485, 0.024064265, 0.0038815604, 0.025684519, 0.041788414, -0.013553761, 0.006681779, -0.0050826604, -0.018175002, 0.008228063, -0.006230926, -0.018907638, 0.0154839745, -0.028713685, -0.015047211, -0.019682541, 0.02516322, 0.040802173, 0.007213644, 0.011743305, -0.015963005, -0.03818159, 0.01191942, -0.031728763, -0.011863063, 0.023881108, 0.0053116092, -0.020992832, -0.017991843, -0.00405063, -0.017780505, -0.0057659843, 0.02978446, 0.031165197, 0.0014221234, 0.021316882, 0.026008569, -0.0018544842, -0.032658648, 0.028474169, 0.013109953, 0.018076377, 0.0007991189, -0.0042373114, 0.028910933, -0.0029358263, 0.021866359, 0.024472851, -0.002576553, -0.033532172, 0.01920351, -0.0095665315, -0.03093977, 0.0034817809, 0.018654034, -0.0074038478, 0.021443684, 0.0038604268, -0.02745975, 0.031587873, 0.0061146906, 0.022711707, -0.019795254, -0.016991513, -0.04471896, -0.007875834, -0.0034941088, -0.043789074, 0.021091456, 0.024909616, -0.013194487, -0.0042690123, 0.027896514, -0.018414518, -0.023303451, -0.025797231, -0.009524264]}], object='list', usage=Usage(completion_tokens=0, prompt_tokens=3, total_tokens=3, completion_tokens_details=None, prompt_tokens_details=None))\n" - ] - } - ], - "source": [ - "import litellm\n", - "\n", - "\n", - "async def main():\n", - " response = await litellm.aembedding(\n", - " model=\"cometapi/text-embedding-3-small\", # The model name must include prefix \"cometapi/\" + the model name from CometAPI\n", - " api_key=api_key, # your CometAPI api-key\n", - " api_base=\"https://api.cometapi.com/v1\",\n", - " input=\"Your text string\",\n", - " )\n", - " print(response)\n", - "\n", - "\n", - "await main()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Async Image Generation" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ImageResponse(created=1760591151, background=None, data=[ImageObject(b64_json=None, revised_prompt=\"Generate an image of an adorable baby sea otter. It should be floating on its back in a calm, clear ocean, playfully grasping a colorful shell in its small paws. The sun is setting in the background, casting a peaceful orange and purple hue across the sky and reflecting upon the ocean waves. The otter's fur is a deep, rich brown and appears silky and wet, with glints of sunlight catching on it. Its eyes are bright, expressing joy and curiosity as it examines its newfound treasure.\", url='https://oaidalleapiprodscus.blob.core.windows.net/private/org-OKnsK88id12jfvnKByup1O0l/user-3GxuMyEg9YMU8LFCPHi31prf/img-7PUEF8Wb6thGDAuZWLJjSnfP.png?st=2025-10-16T04%3A05%3A51Z&se=2025-10-16T06%3A05%3A51Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=38e27a3b-6174-4d3e-90ac-d7d9ad49543f&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2025-10-16T02%3A51%3A01Z&ske=2025-10-17T02%3A51%3A01Z&sks=b&skv=2024-08-04&sig=IZKG2VE%2B6VdOe5Tq0Zk/5bVyGK/oK/yO8g%2BDX4krpug%3D')], output_format=None, quality=None, size=None, usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0, completion_tokens_details=None, prompt_tokens_details=None, input_tokens=0, input_tokens_details={'image_tokens': 0, 'text_tokens': 0}, output_tokens=0))\n" - ] - } - ], - "source": [ - "import asyncio\n", - "\n", - "import litellm\n", - "\n", - "\n", - "async def main():\n", - " response = await litellm.aimage_generation(\n", - " model=\"cometapi/dall-e-3\", # The model name must include prefix \"cometapi/\" + the model name from CometAPI\n", - " api_key=api_key, # your cometapi api-key\n", - " api_base=\"https://api.cometapi.com/v1\",\n", - " prompt=\"A cute baby sea otter\",\n", - " )\n", - " print(response)\n", - "\n", - "\n", - "await main()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "base", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.8" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/LiteLLM_Comparing_LLMs.ipynb b/cookbook/LiteLLM_Comparing_LLMs.ipynb deleted file mode 100644 index 0b2e4e8c776..00000000000 --- a/cookbook/LiteLLM_Comparing_LLMs.ipynb +++ /dev/null @@ -1,441 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "L-W4C3SgClxl" - }, - "source": [ - "## Comparing LLMs on a Test Set using LiteLLM\n", - "LiteLLM allows you to use any LLM as a drop in replacement for `gpt-3.5-turbo`\n", - "\n", - "This notebook walks through how you can compare GPT-4 vs Claude-2 on a given test set using litellm" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "fBkbl4Qo9pvz" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "id": "tzS-AXWK8lJC" - }, - "outputs": [], - "source": [ - "from litellm import completion\n", - "\n", - "# init your test set questions\n", - "questions = [\n", - " \"how do i call completion() using LiteLLM\",\n", - " \"does LiteLLM support VertexAI\",\n", - " \"how do I set my keys on replicate llama2?\",\n", - "]\n", - "\n", - "\n", - "# set your prompt\n", - "prompt = \"\"\"\n", - "You are a coding assistant helping users using litellm.\n", - "litellm is a light package to simplify calling OpenAI, Azure, Cohere, Anthropic, Huggingface API Endpoints. It manages:\n", - "\n", - "\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "id": "vMlqi40x-KAA" - }, - "outputs": [], - "source": [ - "import os\n", - "os.environ['OPENAI_API_KEY'] = \"\"\n", - "os.environ['ANTHROPIC_API_KEY'] = \"\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-HOzUfpK-H8J" - }, - "source": [] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Ktn25dfKEJF1" - }, - "source": [ - "## Calling gpt-3.5-turbo and claude-2 on the same questions\n", - "\n", - "## LiteLLM `completion()` allows you to call all LLMs in the same format\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "DhXwRlc-9DED" - }, - "outputs": [], - "source": [ - "results = [] # for storing results\n", - "\n", - "models = ['gpt-3.5-turbo', 'claude-2'] # define what models you're testing, see: https://docs.litellm.ai/docs/providers\n", - "for question in questions:\n", - " row = [question]\n", - " for model in models:\n", - " print(\"Calling:\", model, \"question:\", question)\n", - " response = completion( # using litellm.completion\n", - " model=model,\n", - " messages=[\n", - " {'role': 'system', 'content': prompt},\n", - " {'role': 'user', 'content': question}\n", - " ]\n", - " )\n", - " answer = response.choices[0].message['content']\n", - " row.append(answer)\n", - " print(print(\"Calling:\", model, \"answer:\", answer))\n", - "\n", - " results.append(row) # save results\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "RkEXhXxCDN77" - }, - "source": [ - "## Visualizing Results" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 761 - }, - "id": "42hrmW6q-n4s", - "outputId": "b763bf39-72b9-4bea-caf6-de6b2412f86d" - }, - "outputs": [ - { - "data": { - "application/vnd.google.colaboratory.module+javascript": "\n import \"https://ssl.gstatic.com/colaboratory/data_table/881c4a0d49046431/data_table.js\";\n\n const table = window.createDataTable({\n data: [[{\n 'v': 0,\n 'f': \"0\",\n },\n\"how do i call completion() using LiteLLM\",\n\"To call the `completion()` function using LiteLLM, you need to follow these steps:\\n\\n1. Install the `litellm` package by running `pip install litellm` in your terminal.\\n2. Import the `Completion` class from the `litellm` module.\\n3. Initialize an instance of the `Completion` class by providing the required parameters like the API endpoint URL and your API key.\\n4. Call the `complete()` method on the `Completion` instance and pass the text prompt as a string.\\n5. Retrieve the generated completion from the response object and use it as desired.\\n\\nHere's an example:\\n\\n```python\\nfrom litellm.completion import Completion\\n\\n# Initialize the Completion client\\ncompletion_client = Completion(\\n model_name='gpt-3.5-turbo',\\n api_key='your_api_key',\\n endpoint='https://your_endpoint_url'\\n)\\n\\n# Call the completion() method\\nresponse = completion_client.complete(\\\"Once upon a time\\\")\\n\\n# Retrieve the generated completion\\ncompletion = response['choices'][0]['text']\\n\\nprint(completion)\\n```\\n\\nMake sure to replace `'gpt-3.5-turbo'` with the desired model name, `'your_api_key'` with your actual API key, and `'https://your_endpoint_url'` with the correct API endpoint URL provided by your service provider.\\n\\nNote: The above example assumes you have a valid API key and endpoint URL for the OpenAI GPT-3.5-turbo model. Make sure to obtain the necessary credentials according to the API you are using.\",\n\" Here is how you can call the completion() method using LiteLLM:\\n\\nFirst, import LiteLLM:\\n\\n```python\\nimport litellm as lm\\n```\\n\\nThen create a LiteLLM object, specifying the API you want to use (e.g. \\\"openai\\\"):\\n\\n```python \\nai = lm.LiteLLM(\\\"openai\\\")\\n```\\n\\nNow you can call the completion() method on the ai object:\\n\\n```python\\nresponse = ai.completion(\\n prompt=\\\"Hello\\\", \\n model=\\\"text-davinci-003\\\",\\n max_tokens=100\\n)\\n```\\n\\nThe completion() method takes parameters like:\\n\\n- prompt (str): The prompt text to complete \\n- model (str): The AI model to use\\n- max_tokens (int): The maximum number of tokens to generate\\n\\nIt returns a Python dictionary with the AI's response.\\n\\nYou can then access the generated text using:\\n\\n```python\\nprint(response[\\\"choices\\\"][0][\\\"text\\\"]) \\n```\\n\\nSo LiteLLM provides a simple unified interface to call the underlying AI APIs. The completion() method works similarly for OpenAI\"],\n [{\n 'v': 1,\n 'f': \"1\",\n },\n\"does LiteLLM support VertexAI\",\n\"Yes, LiteLLM does support Google Cloud Vertex AI. It provides convenient wrappers and simplified functions to call Vertex AI API endpoints for natural language processing tasks such as text classification, entity extraction, sentiment analysis, etc. You can easily integrate LiteLLM with Vertex AI in your code to leverage its capabilities.\",\n\" Unfortunately, LiteLLM does not currently support VertexAI. LiteLLM focuses on providing a simple interface to call the APIs of services like OpenAI, Azure, Cohere, Anthropic, and Hugging Face. \\n\\nVertexAI is Google's managed machine learning platform. Integrating VertexAI would require additional development work to wrap the VertexAI SDK in a simple interface like LiteLLM provides for other services. \\n\\nHowever, LiteLLM is open source, so it is possible for someone to contribute support for VertexAI. The maintainers would likely welcome a pull request to add VertexAI as an option if implemented well. But out of the box, LiteLLM does not have built-in support for calling VertexAI APIs.\\n\\nThe key services LiteLLM supports are:\\n\\n- OpenAI (GPT, Codex, DALL-E)\\n- Azure Cognitive Services (Text Analytics, Computer Vision, Speech) \\n- Cohere\\n- Anthropic AI\\n- Hugging Face Transformers\\n\\nSo while it doesn't cover every ML API provider, it does make it easy to use the most popular natural language, speech, and vision APIs through a simple interface. Adding VertexAI\"],\n [{\n 'v': 2,\n 'f': \"2\",\n },\n\"how do I set my keys on replicate llama2?\",\n\"To set your keys on Replicate Llama2, follow these steps:\\n\\n1. Open the Llama2 dashboard in your browser.\\n2. Click on the \\\"Settings\\\" tab in the top menu.\\n3. Scroll down to the \\\"API Keys\\\" section.\\n4. Click on the \\\"Add a Key\\\" button.\\n5. Enter a name for your API key to help you identify it later.\\n6. Select the provider for your API key from the dropdown menu. For example, you can select \\\"OpenAI\\\" for OpenAI GPT-3 access.\\n7. Enter your API key in the provided input field. Make sure to copy it correctly.\\n8. Click on the \\\"Save\\\" button to save your API key.\\n\\nNote: The actual steps may vary slightly depending on the platform or interface you are using to access Llama2.\",\n\" Here are the steps to set your API keys on Replicate for litellm:\\n\\n1. Go to your Replicate project settings and select the Environment tab.\\n\\n2. Under Environment Variables, click Add Variable.\\n\\n3. Add variables for the API keys you want to use. The variable names should match the ones used in litellm:\\n\\n- `OPENAI_API_KEY` for OpenAI \\n- `AZURE_API_KEY` for Azure Cognitive Services\\n- `COHERE_API_KEY` for Cohere\\n- `ANTHROPIC_API_KEY` for Anthropic\\n- `HUGGINGFACE_API_KEY` for Hugging Face\\n\\n4. Set the value to your actual API key for each service. Make sure to treat the values as secrets.\\n\\n5. Make sure your litellm code is referencing the environment variable names, for example:\\n\\n```python\\nimport litellm as lm\\n\\nlm.auth(openai_key=os.getenv(\\\"OPENAI_API_KEY\\\")) \\n```\\n\\n6. Restart your Replicate runtime to load the new environment variables.\\n\\nNow litellm will use your\"]],\n columns: [[\"number\", \"index\"], [\"string\", \"Question\"], [\"string\", \"gpt-3.5-turbo\"], [\"string\", \"claude-2\"]],\n columnOptions: [{\"width\": \"1px\", \"className\": \"index_column\"}],\n rowsPerPage: 25,\n helpUrl: \"https://colab.research.google.com/notebooks/data_table.ipynb\",\n suppressOutputScrolling: true,\n minimumWidth: undefined,\n });\n\n function appendQuickchartButton(parentElement) {\n let quickchartButtonContainerElement = document.createElement('div');\n quickchartButtonContainerElement.innerHTML = `\n
\n \n \n\n\n \n
`;\n parentElement.appendChild(quickchartButtonContainerElement);\n }\n\n appendQuickchartButton(table);\n ", - "text/html": [ - "\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Questiongpt-3.5-turboclaude-2
0how do i call completion() using LiteLLMTo call the `completion()` function using Lite...Here is how you can call the completion() met...
1does LiteLLM support VertexAIYes, LiteLLM does support Google Cloud Vertex ...Unfortunately, LiteLLM does not currently sup...
2how do I set my keys on replicate llama2?To set your keys on Replicate Llama2, follow t...Here are the steps to set your API keys on Re...
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - " \n", - "
\n", - "
\n", - "
\n" - ], - "text/plain": [ - " Question \\\n", - "0 how do i call completion() using LiteLLM \n", - "1 does LiteLLM support VertexAI \n", - "2 how do I set my keys on replicate llama2? \n", - "\n", - " gpt-3.5-turbo \\\n", - "0 To call the `completion()` function using Lite... \n", - "1 Yes, LiteLLM does support Google Cloud Vertex ... \n", - "2 To set your keys on Replicate Llama2, follow t... \n", - "\n", - " claude-2 \n", - "0 Here is how you can call the completion() met... \n", - "1 Unfortunately, LiteLLM does not currently sup... \n", - "2 Here are the steps to set your API keys on Re... " - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Create a table to visualize results\n", - "import pandas as pd\n", - "\n", - "columns = ['Question'] + models\n", - "df = pd.DataFrame(results, columns=columns)\n", - "\n", - "df" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/LiteLLM_Completion_Cost.ipynb b/cookbook/LiteLLM_Completion_Cost.ipynb deleted file mode 100644 index b8f5eb36a5c..00000000000 --- a/cookbook/LiteLLM_Completion_Cost.ipynb +++ /dev/null @@ -1,241 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "# Use LiteLLM to calculate costs for all your completion calls\n", - "In this notebook we'll use `litellm.completion_cost` to get completion costs" - ], - "metadata": { - "id": "BgWr0PsUR3vV" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ViczFTjsDzSI" - }, - "outputs": [], - "source": [ - "!pip install litellm==0.1.549 # use 0.1.549 or later" - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Calculating costs for gpt-3.5 turbo completion()" - ], - "metadata": { - "id": "k_1CWUwmSNtj" - } - }, - { - "cell_type": "code", - "source": [ - "from litellm import completion, completion_cost\n", - "import os\n", - "os.environ['OPENAI_API_KEY'] = \"\"\n", - "\n", - "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - "response = completion(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages=messages,\n", - ")\n", - "\n", - "print(response)\n", - "\n", - "cost = completion_cost(completion_response=response)\n", - "formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n", - "print(formatted_string)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Tp0fyk-jD0pP", - "outputId": "ce885fb3-3237-41b2-9d8b-3fb30bba498b" - }, - "execution_count": 6, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "got response\n", - "{\n", - " \"id\": \"chatcmpl-7vyCApIZaCxP36kb9meUMN2DFSJPh\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1694050442,\n", - " \"model\": \"gpt-3.5-turbo-0613\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Hello! I'm an AI and I don't have feelings, but I'm here to help you. How can I assist you today?\"\n", - " },\n", - " \"finish_reason\": \"stop\"\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"prompt_tokens\": 13,\n", - " \"completion_tokens\": 28,\n", - " \"total_tokens\": 41\n", - " }\n", - "}\n", - "Cost for completion call: $0.0000755000\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Calculating costs for Together Computer completion()" - ], - "metadata": { - "id": "AjDs4G-uS6PS" - } - }, - { - "cell_type": "code", - "source": [ - "from litellm import completion, completion_cost\n", - "import os\n", - "os.environ['TOGETHERAI_API_KEY'] = \"\"\n", - "\n", - "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - "response = completion(\n", - " model=\"togethercomputer/llama-2-70b-chat\",\n", - " messages=messages,\n", - ")\n", - "\n", - "print(response)\n", - "\n", - "cost = completion_cost(completion_response=response)\n", - "formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n", - "print(formatted_string)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "jMPsUV-KEa6a", - "outputId": "7a69b291-f149-4b9c-8a78-9c8142bac759" - }, - "execution_count": 7, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "{\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"Hello! I'm doing well, thanks for asking. I hope you're having a great\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1694050771.2821715,\n", - " \"model\": \"togethercomputer/llama-2-70b-chat\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 12,\n", - " \"completion_tokens\": 18,\n", - " \"total_tokens\": 30\n", - " }\n", - "}\n", - "Cost for completion call: $0.0000900000\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Calculating costs for Replicate Llama2 completion()" - ], - "metadata": { - "id": "vEa4s6-7TANS" - } - }, - { - "cell_type": "code", - "source": [ - "from litellm import completion, completion_cost\n", - "import os\n", - "os.environ['REPLICATE_API_KEY'] = \"\"\n", - "\n", - "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - "response = completion(\n", - " model=\"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf\",\n", - " messages=messages,\n", - ")\n", - "\n", - "print(response)\n", - "\n", - "cost = completion_cost(completion_response=response)\n", - "formatted_string = f\"Cost for completion call: ${float(cost):.10f}\"\n", - "print(formatted_string)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Xf1TKRDuS1bR", - "outputId": "cfb2b484-a6e5-41ad-86c5-7e66aba27648" - }, - "execution_count": 8, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "{\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" Hello! I'm doing well, thanks for asking. How about you? Is there anything you need help with today?\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1694050893.4534576,\n", - " \"model\": \"replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 6,\n", - " \"completion_tokens\": 24,\n", - " \"total_tokens\": 30\n", - " },\n", - " \"ended\": 1694050896.6689413\n", - "}\n", - "total_replicate_run_time 3.2154836654663086\n", - "Cost for completion call: $0.0045016771\n" - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/cookbook/LiteLLM_HuggingFace.ipynb b/cookbook/LiteLLM_HuggingFace.ipynb deleted file mode 100644 index bf8482a5f11..00000000000 --- a/cookbook/LiteLLM_HuggingFace.ipynb +++ /dev/null @@ -1,252 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "9dKM5k8qsMIj" - }, - "source": [ - "## LiteLLM Hugging Face\n", - "\n", - "Docs for huggingface: https://docs.litellm.ai/docs/providers/huggingface\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "BVDdmCp-o97j" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yp5UXRqtpu9f" - }, - "source": [ - "## Serverless Inference Providers\n", - "\n", - "Read more about Inference Providers here: https://huggingface.co/blog/inference-providers.\n", - "\n", - "In order to use litellm with Hugging Face Inference Providers, you need to set `model=huggingface//`.\n", - "\n", - "Example: `huggingface/together/deepseek-ai/DeepSeek-R1` to run DeepSeek-R1 (https://huggingface.co/deepseek-ai/DeepSeek-R1) through Together AI.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Pi5Oww8gpCUm", - "outputId": "659a67c7-f90d-4c06-b94e-2c4aa92d897a" - }, - "outputs": [], - "source": [ - "import os\n", - "from litellm import completion\n", - "\n", - "# You can create a HF token here: https://huggingface.co/settings/tokens\n", - "os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n", - "\n", - "# Call DeepSeek-R1 model through Together AI\n", - "response = completion(\n", - " model=\"huggingface/together/deepseek-ai/DeepSeek-R1\",\n", - " messages=[{\"content\": \"How many r's are in the word `strawberry`?\", \"role\": \"user\"}],\n", - ")\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EU0UubrKzTFe" - }, - "source": [ - "## Streaming\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "y-QfIvA-uJKX", - "outputId": "b007bb98-00d0-44a4-8264-c8a2caed6768" - }, - "outputs": [], - "source": [ - "import os\n", - "from litellm import completion\n", - "\n", - "os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n", - "\n", - "response = completion(\n", - " model=\"huggingface/together/deepseek-ai/DeepSeek-R1\",\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"How many r's are in the word `strawberry`?\",\n", - " \n", - " }\n", - " ],\n", - " stream=True,\n", - ")\n", - "\n", - "for chunk in response:\n", - " print(chunk)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## With images as input\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from litellm import completion\n", - "\n", - "# Set your Hugging Face Token\n", - "os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n", - "\n", - "messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\"type\": \"text\", \"text\": \"What's in this image?\"},\n", - " {\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {\n", - " \"url\": \"https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png\",\n", - " },\n", - " },\n", - " ],\n", - " }\n", - "]\n", - "\n", - "response = completion(\n", - " model=\"huggingface/sambanova/meta-llama/Llama-3.3-70B-Instruct\",\n", - " messages=messages,\n", - ")\n", - "print(response.choices[0])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tools - Function Calling\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from litellm import completion\n", - "\n", - "\n", - "# Set your Hugging Face Token\n", - "os.environ[\"HF_TOKEN\"] = \"hf_xxxxxx\"\n", - "\n", - "tools = [\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city and state, e.g. San Francisco, CA\",\n", - " },\n", - " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n", - " },\n", - " \"required\": [\"location\"],\n", - " },\n", - " },\n", - " }\n", - "]\n", - "messages = [{\"role\": \"user\", \"content\": \"What's the weather like in Boston today?\"}]\n", - "\n", - "response = completion(\n", - " model=\"huggingface/sambanova/meta-llama/Llama-3.1-8B-Instruct\", messages=messages, tools=tools, tool_choice=\"auto\"\n", - ")\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Hugging Face Dedicated Inference Endpoints\n", - "\n", - "Steps to use\n", - "\n", - "- Create your own Hugging Face dedicated endpoint here: https://ui.endpoints.huggingface.co/\n", - "- Set `api_base` to your deployed api base\n", - "- set the model to `huggingface/tgi` so that litellm knows it's a huggingface Deployed Inference Endpoint.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import litellm\n", - "\n", - "\n", - "response = litellm.completion(\n", - " model=\"huggingface/tgi\",\n", - " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}],\n", - " api_base=\"https://my-endpoint.endpoints.huggingface.cloud/v1/\",\n", - ")\n", - "print(response)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.0" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/LiteLLM_NovitaAI_Cookbook.ipynb b/cookbook/LiteLLM_NovitaAI_Cookbook.ipynb deleted file mode 100644 index 8fa7d0b987a..00000000000 --- a/cookbook/LiteLLM_NovitaAI_Cookbook.ipynb +++ /dev/null @@ -1,97 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "iFEmsVJI_2BR" - }, - "source": [ - "# LiteLLM NovitaAI Cookbook" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "cBlUhCEP_xj4" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "p-MQqWOT_1a7" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ['NOVITA_API_KEY'] = \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Ze8JqMqWAARO" - }, - "outputs": [], - "source": [ - "from litellm import completion\n", - "response = completion(\n", - " model=\"novita/deepseek/deepseek-r1\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "-LnhELrnAM_J" - }, - "outputs": [], - "source": [ - "response = completion(\n", - " model=\"novita/deepseek/deepseek-r1\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "dJBOUYdwCEn1" - }, - "outputs": [], - "source": [ - "response = completion(\n", - " model=\"mistralai/mistral-7b-instruct\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/LiteLLM_OpenRouter.ipynb b/cookbook/LiteLLM_OpenRouter.ipynb deleted file mode 100644 index 6444b23b294..00000000000 --- a/cookbook/LiteLLM_OpenRouter.ipynb +++ /dev/null @@ -1,179 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "iFEmsVJI_2BR" - }, - "source": [ - "# LiteLLM OpenRouter Cookbook" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "cBlUhCEP_xj4" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "id": "p-MQqWOT_1a7" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ['OPENROUTER_API_KEY'] = \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Ze8JqMqWAARO", - "outputId": "64f3e836-69fa-4f8e-fb35-088a913bbe98" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"id\": \"gen-W8FTMSIEorCp3vG5iYIgNMR4IeBv\",\n", - " \"model\": \"chat-bison@001\",\n", - " \"choices\": [\n", - " {\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"```\\n#include \\n\\nint main() {\\n printf(\\\"Hi!\\\\n\\\");\\n return 0;\\n}\\n```\"\n", - " }\n", - " }\n", - " ],\n", - " \"response_ms\": 7817.777999999999\n", - "}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from litellm import completion\n", - "response = completion(\n", - " model=\"openrouter/google/palm-2-chat-bison\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "-LnhELrnAM_J", - "outputId": "d51c7ab7-d761-4bd1-f849-1534d9df4cd0" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"choices\": [\n", - " {\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \" Here is some simple code to print \\\"Hi\\\":\\n\\n```python\\nprint(\\\"Hi\\\")\\n```\\n\\nThis uses the print() function in Python to output the text \\\"Hi\\\".\"\n", - " },\n", - " \"finish_reason\": \"stop_sequence\"\n", - " }\n", - " ],\n", - " \"model\": \"claude-2.0\",\n", - " \"id\": \"gen-IiuV7ZNimDufVeutBHrl8ajPuzEh\",\n", - " \"response_ms\": 8112.443000000001\n", - "}" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "response = completion(\n", - " model=\"openrouter/anthropic/claude-2\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "dJBOUYdwCEn1", - "outputId": "ffa18679-ec15-4dad-fe2b-68665cdf36b0" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"id\": \"gen-PyMd3yyJ0aQsCgIY9R8XGZoAtPbl\",\n", - " \"model\": \"togethercomputer/llama-2-70b-chat\",\n", - " \"choices\": [\n", - " {\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"*gives a sly smile as they type*\\n\\nHey there, handsome. \\ud83d\\ude0f\\n\\nWhat brings you to my neck of the woods today? \\ud83d\\ude18\"\n", - " }\n", - " }\n", - " ],\n", - " \"response_ms\": 9618.775\n", - "}" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "response = completion(\n", - " model=\"openrouter/meta-llama/llama-2-70b-chat\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/LiteLLM_Petals.ipynb b/cookbook/LiteLLM_Petals.ipynb deleted file mode 100644 index aacc22dd1b9..00000000000 --- a/cookbook/LiteLLM_Petals.ipynb +++ /dev/null @@ -1,568 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "dwGtLi_tvM6N" - }, - "source": [ - "# Using LiteLLM with Petals" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "bdlgaWQqDpzj" - }, - "outputs": [], - "source": [ - "!pip install litellm # 0.1.715 and upwards" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "5Id2QKwOEH8X" - }, - "outputs": [], - "source": [ - "# install petals\n", - "!pip install git+https://github.com/bigscience-workshop/petals" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "k42fldw3veSN" - }, - "source": [ - "## petals-team/StableBeluga2" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "tIHcEHdSDqju", - "outputId": "485dbf54-395c-433a-bbf4-8eb70a9fa624" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "You are using the default legacy behaviour of the . If you see this, DO NOT PANIC! This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thouroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n", - "Sep 19 18:39:50.634 [\u001b[1m\u001b[34mINFO\u001b[0m] Make sure you follow the LLaMA's terms of use: https://bit.ly/llama2-license for LLaMA 2, https://bit.ly/llama-license for LLaMA 1\n", - "Sep 19 18:39:50.639 [\u001b[1m\u001b[34mINFO\u001b[0m] Using DHT prefix: StableBeluga2-hf\n", - "Sep 19 18:40:13.920 [\u001b[1m\u001b[34mINFO\u001b[0m] Route found: 0:40 via …HfQWVM => 40:80 via …Zj98Se\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"Hello, how are you?\\nI'm doing well, thank you. I'm just getting ready to go to the gym.\\nOh, that's great. I'm trying to get back into a workout routine myself.\\nYeah,\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-f09d79b3-c1d1-49b7-b55f-cd8dfa1043bf\",\n", - " \"created\": 1695148897.473613,\n", - " \"model\": \"petals-team/StableBeluga2\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 6,\n", - " \"completion_tokens\": 45,\n", - " \"total_tokens\": 51\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "response = completion(model=\"petals/petals-team/StableBeluga2\", messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}], max_tokens=50)\n", - "\n", - "print(response)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "J8DubRnHvh_j" - }, - "source": [ - "## huggyllama/llama-65b" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 538, - "referenced_widgets": [ - "2fec5cc400424671a3d517327117d18a", - "3687c76fe84d464baaf35366b21e83b3", - "c29d4460dbaa441cae110b58e0014151", - "6560449a38bf4a7bacd97ccaacf01c4c", - "5fbd6ae281984d28ba59ebfd0279eda7", - "323e30e275434aeea241163e5f1f9031", - "48f4adec51c94f9da6e4c4564daeff84", - "2a672981a44b4a7fb30674f97f4c10c6", - "d75ae8d22ea74840b4c80c8f386384c4", - "54c06312ecff4e7588665e8b0cb7118b", - "300078a9d1a6483fba81a4be63793ff7" - ] - }, - "id": "IlTCJwDsNvgF", - "outputId": "2e84d125-d982-48ed-8a92-6ca438a50d0c" - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Sep 19 18:41:37.912 [\u001b[1m\u001b[34mINFO\u001b[0m] Make sure you follow the LLaMA's terms of use: https://bit.ly/llama2-license for LLaMA 2, https://bit.ly/llama-license for LLaMA 1\n", - "Sep 19 18:41:37.914 [\u001b[1m\u001b[34mINFO\u001b[0m] Using DHT prefix: llama-65b-hf\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2fec5cc400424671a3d517327117d18a", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Loading checkpoint shards: 0%| | 0/2 [00:00 JSON: {\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" Good morning!\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1694030351.309254,\n", - " \"model\": \"claude-2\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 11,\n", - " \"completion_tokens\": 3,\n", - " \"total_tokens\": 14\n", - " }\n", - " },\n", - " JSON: {\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" I'm an AI assistant created by Anthropic. I don't actually have a concept of the current time.\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1694030352.1215081,\n", - " \"model\": \"claude-2\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 13,\n", - " \"completion_tokens\": 22,\n", - " \"total_tokens\": 35\n", - " }\n", - " }]" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import os\n", - "\n", - "os.environ['ANTHROPIC_API_KEY'] = \"\"\n", - "\n", - "\n", - "responses = batch_completion(\n", - " model=\"claude-2\",\n", - " messages = [\n", - " [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"good morning? \"\n", - " }\n", - " ],\n", - " [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"what's the time? \"\n", - " }\n", - " ]\n", - " ]\n", - ")\n", - "responses" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb b/cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb deleted file mode 100644 index d89f2b67c56..00000000000 --- a/cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb +++ /dev/null @@ -1,568 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "kccfk0mHZ4Ad" - }, - "source": [ - "# Migrating to LiteLLM Proxy from OpenAI/Azure OpenAI\n", - "\n", - "Covers:\n", - "\n", - "* /chat/completion\n", - "* /embedding\n", - "\n", - "\n", - "These are **selected examples**. LiteLLM Proxy is **OpenAI-Compatible**, it works with any project that calls OpenAI. Just change the `base_url`, `api_key` and `model`.\n", - "\n", - "Set `LITELLM_MASTER_KEY` in the notebook environment to the same key configured on the proxy before running authenticated examples\n", - "\n", - "For more examples, [go here](https://docs.litellm.ai/docs/proxy/user_keys)\n", - "\n", - "To pass provider-specific args, [go here](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)\n", - "\n", - "To drop unsupported params (E.g. frequency_penalty for bedrock with librechat), [go here](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nmSClzCPaGH6" - }, - "source": [ - "## /chat/completion\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_vqcjwOVaKpO" - }, - "source": [ - "### OpenAI Python SDK" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "x1e_Ok3KZzeP" - }, - "outputs": [], - "source": [ - "import openai\n", - "client = openai.OpenAI(\n", - " api_key=\"anything\",\n", - " base_url=\"http://0.0.0.0:4000\"\n", - ")\n", - "\n", - "# request sent to model set on litellm proxy, `litellm --model`\n", - "response = client.chat.completions.create(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"this is a test request, write a short poem\"\n", - " }\n", - " ],\n", - " extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params\n", - " \"metadata\": { # 👈 use for logging additional params (e.g. to langfuse)\n", - " \"generation_name\": \"ishaan-generation-openai-client\",\n", - " \"generation_id\": \"openai-client-gen-id22\",\n", - " \"trace_id\": \"openai-client-trace-id22\",\n", - " \"trace_user_id\": \"openai-client-user-id2\"\n", - " }\n", - " }\n", - ")\n", - "\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "AqkyKk9Scxgj" - }, - "source": [ - "## Function Calling" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "wDg10VqLczE1" - }, - "outputs": [], - "source": [ - "import os\n", - "from openai import OpenAI\n", - "client = OpenAI(\n", - " api_key=os.environ[\"LITELLM_MASTER_KEY\"],\n", - " base_url=\"http://0.0.0.0:4000\",\n", - ")\n", - "\n", - "tools = [\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city and state, e.g. San Francisco, CA\",\n", - " },\n", - " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n", - " },\n", - " \"required\": [\"location\"],\n", - " },\n", - " }\n", - " }\n", - "]\n", - "messages = [{\"role\": \"user\", \"content\": \"What's the weather like in Boston today?\"}]\n", - "completion = client.chat.completions.create(\n", - " model=\"gpt-4o\", # use 'model_name' from config.yaml\n", - " messages=messages,\n", - " tools=tools,\n", - " tool_choice=\"auto\"\n", - ")\n", - "\n", - "print(completion)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "YYoxLloSaNWW" - }, - "source": [ - "### Azure OpenAI Python SDK" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "yA1XcgowaSRy" - }, - "outputs": [], - "source": [ - "import openai\n", - "client = openai.AzureOpenAI(\n", - " api_key=\"anything\",\n", - " base_url=\"http://0.0.0.0:4000\"\n", - ")\n", - "\n", - "# request sent to model set on litellm proxy, `litellm --model`\n", - "response = client.chat.completions.create(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"this is a test request, write a short poem\"\n", - " }\n", - " ],\n", - " extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params\n", - " \"metadata\": { # 👈 use for logging additional params (e.g. to langfuse)\n", - " \"generation_name\": \"ishaan-generation-openai-client\",\n", - " \"generation_id\": \"openai-client-gen-id22\",\n", - " \"trace_id\": \"openai-client-trace-id22\",\n", - " \"trace_user_id\": \"openai-client-user-id2\"\n", - " }\n", - " }\n", - ")\n", - "\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yl9qhDvnaTpL" - }, - "source": [ - "### Langchain Python" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "5MUZgSquaW5t" - }, - "outputs": [], - "source": [ - "from langchain.chat_models import ChatOpenAI\n", - "from langchain.prompts.chat import (\n", - " ChatPromptTemplate,\n", - " HumanMessagePromptTemplate,\n", - " SystemMessagePromptTemplate,\n", - ")\n", - "from langchain.schema import HumanMessage, SystemMessage\n", - "import os\n", - "\n", - "os.environ[\"OPENAI_API_KEY\"] = \"anything\"\n", - "\n", - "chat = ChatOpenAI(\n", - " openai_api_base=\"http://0.0.0.0:4000\",\n", - " model = \"gpt-3.5-turbo\",\n", - " temperature=0.1,\n", - " extra_body={\n", - " \"metadata\": {\n", - " \"generation_name\": \"ishaan-generation-langchain-client\",\n", - " \"generation_id\": \"langchain-client-gen-id22\",\n", - " \"trace_id\": \"langchain-client-trace-id22\",\n", - " \"trace_user_id\": \"langchain-client-user-id2\"\n", - " }\n", - " }\n", - ")\n", - "\n", - "messages = [\n", - " SystemMessage(\n", - " content=\"You are a helpful assistant that im using to make a test request to.\"\n", - " ),\n", - " HumanMessage(\n", - " content=\"test from litellm. tell me why it's amazing in 1 sentence\"\n", - " ),\n", - "]\n", - "response = chat(messages)\n", - "\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "B9eMgnULbRaz" - }, - "source": [ - "### Curl" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "VWCCk5PFcmhS" - }, - "source": [ - "\n", - "\n", - "```\n", - "curl -X POST 'http://0.0.0.0:4000/chat/completions' \\\n", - " -H 'Content-Type: application/json' \\\n", - " -d '{\n", - " \"model\": \"gpt-3.5-turbo\",\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"what llm are you\"\n", - " }\n", - " ],\n", - " \"metadata\": {\n", - " \"generation_name\": \"ishaan-test-generation\",\n", - " \"generation_id\": \"gen-id22\",\n", - " \"trace_id\": \"trace-id22\",\n", - " \"trace_user_id\": \"user-id2\"\n", - " }\n", - "}'\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "drBAm2e1b6xe" - }, - "source": [ - "### LlamaIndex" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "d0bZcv8fb9mL" - }, - "outputs": [], - "source": [ - "import os, dotenv\n", - "\n", - "from llama_index.llms import AzureOpenAI\n", - "from llama_index.embeddings import AzureOpenAIEmbedding\n", - "from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext\n", - "\n", - "llm = AzureOpenAI(\n", - " engine=\"azure-gpt-3.5\", # model_name on litellm proxy\n", - " temperature=0.0,\n", - " azure_endpoint=\"http://0.0.0.0:4000\", # litellm proxy endpoint\n", - " api_key=os.environ[\"LITELLM_MASTER_KEY\"],\n", - " api_version=\"2023-07-01-preview\",\n", - ")\n", - "\n", - "embed_model = AzureOpenAIEmbedding(\n", - " deployment_name=\"azure-embedding-model\",\n", - " azure_endpoint=\"http://0.0.0.0:4000\",\n", - " api_key=os.environ[\"LITELLM_MASTER_KEY\"],\n", - " api_version=\"2023-07-01-preview\",\n", - ")\n", - "\n", - "\n", - "documents = SimpleDirectoryReader(\"llama_index_data\").load_data()\n", - "service_context = ServiceContext.from_defaults(llm=llm, embed_model=embed_model)\n", - "index = VectorStoreIndex.from_documents(documents, service_context=service_context)\n", - "\n", - "query_engine = index.as_query_engine()\n", - "response = query_engine.query(\"What did the author do growing up?\")\n", - "print(response)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xypvNdHnb-Yy" - }, - "source": [ - "### Langchain JS" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "R55mK2vCcBN2" - }, - "outputs": [], - "source": [ - "import { ChatOpenAI } from \"@langchain/openai\";\n", - "\n", - "\n", - "const model = new ChatOpenAI({\n", - " modelName: \"gpt-4\",\n", - " openAIApiKey: process.env.LITELLM_MASTER_KEY,\n", - " modelKwargs: {\"metadata\": \"hello world\"} // 👈 PASS Additional params here\n", - "}, {\n", - " basePath: \"http://0.0.0.0:4000\",\n", - "});\n", - "\n", - "const message = await model.invoke(\"Hi there!\");\n", - "\n", - "console.log(message);\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nC4bLifCcCiW" - }, - "source": [ - "### OpenAI JS" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "MICH8kIMcFpg" - }, - "outputs": [], - "source": [ - "const { OpenAI } = require('openai');\n", - "\n", - "const openai = new OpenAI({\n", - " apiKey: process.env.LITELLM_MASTER_KEY,\n", - " baseURL: \"http://0.0.0.0:4000\"\n", - "});\n", - "\n", - "async function main() {\n", - " const chatCompletion = await openai.chat.completions.create({\n", - " messages: [{ role: 'user', content: 'Say this is a test' }],\n", - " model: 'gpt-3.5-turbo',\n", - " }, {\"metadata\": {\n", - " \"generation_name\": \"ishaan-generation-openaijs-client\",\n", - " \"generation_id\": \"openaijs-client-gen-id22\",\n", - " \"trace_id\": \"openaijs-client-trace-id22\",\n", - " \"trace_user_id\": \"openaijs-client-user-id2\"\n", - " }});\n", - "}\n", - "\n", - "main();\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "D1Q07pEAcGTb" - }, - "source": [ - "### Anthropic SDK" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "qBjFcAvgcI3t" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "from anthropic import Anthropic\n", - "\n", - "client = Anthropic(\n", - " base_url=\"http://localhost:4000\", # proxy endpoint\n", - " api_key=\"sk-test-proxy-key-123\", # litellm proxy virtual key (example)\n", - ")\n", - "\n", - "message = client.messages.create(\n", - " max_tokens=1024,\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Hello, Claude\",\n", - " }\n", - " ],\n", - " model=\"claude-3-opus-20240229\",\n", - ")\n", - "print(message.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "dFAR4AJGcONI" - }, - "source": [ - "## /embeddings" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lgNoM281cRzR" - }, - "source": [ - "### OpenAI Python SDK" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NY3DJhPfcQhA" - }, - "outputs": [], - "source": [ - "import openai\n", - "from openai import OpenAI\n", - "\n", - "# set base_url to your proxy server\n", - "# set api_key to send to proxy server\n", - "client = OpenAI(api_key=\"\", base_url=\"http://0.0.0.0:4000\")\n", - "\n", - "response = client.embeddings.create(\n", - " input=[\"hello from litellm\"],\n", - " model=\"text-embedding-ada-002\"\n", - ")\n", - "\n", - "print(response)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "hmbg-DW6cUZs" - }, - "source": [ - "### Langchain Embeddings" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "lX2S8Nl1cWVP" - }, - "outputs": [], - "source": [ - "from langchain.embeddings import OpenAIEmbeddings\n", - "\n", - "embeddings = OpenAIEmbeddings(model=\"sagemaker-embeddings\", openai_api_base=\"http://0.0.0.0:4000\", openai_api_key=\"temp-key\")\n", - "\n", - "\n", - "text = \"This is a test document.\"\n", - "\n", - "query_result = embeddings.embed_query(text)\n", - "\n", - "print(f\"SAGEMAKER EMBEDDINGS\")\n", - "print(query_result[:5])\n", - "\n", - "embeddings = OpenAIEmbeddings(model=\"bedrock-embeddings\", openai_api_base=\"http://0.0.0.0:4000\", openai_api_key=\"temp-key\")\n", - "\n", - "text = \"This is a test document.\"\n", - "\n", - "query_result = embeddings.embed_query(text)\n", - "\n", - "print(f\"BEDROCK EMBEDDINGS\")\n", - "print(query_result[:5])\n", - "\n", - "embeddings = OpenAIEmbeddings(model=\"bedrock-titan-embeddings\", openai_api_base=\"http://0.0.0.0:4000\", openai_api_key=\"temp-key\")\n", - "\n", - "text = \"This is a test document.\"\n", - "\n", - "query_result = embeddings.embed_query(text)\n", - "\n", - "print(f\"TITAN EMBEDDINGS\")\n", - "print(query_result[:5])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "oqGbWBCQcYfd" - }, - "source": [ - "### Curl Request" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "7rkIMV9LcdwQ" - }, - "source": [ - "\n", - "\n", - "```curl\n", - "curl -X POST 'http://0.0.0.0:4000/embeddings' \\\n", - " -H 'Content-Type: application/json' \\\n", - " -d ' {\n", - " \"model\": \"text-embedding-ada-002\",\n", - " \"input\": [\"write a litellm poem\"]\n", - " }'\n", - "```\n", - "\n" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/Parallel_function_calling.ipynb b/cookbook/Parallel_function_calling.ipynb deleted file mode 100644 index cb7fbaface2..00000000000 --- a/cookbook/Parallel_function_calling.ipynb +++ /dev/null @@ -1,478 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "This is a tutorial on using Parallel function calling with LiteLLM" - ], - "metadata": { - "id": "gHwFJ-srdnku" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "RrtHuVHlZmUe" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "markdown", - "source": [ - "This tutorial walks through the steps doing parallel function calling using\n", - " - OpenAI\n", - " - Azure OpenAI" - ], - "metadata": { - "id": "sG5ANaazjU0g" - } - }, - { - "cell_type": "code", - "source": [ - "# set openai api key\n", - "import os\n", - "os.environ['OPENAI_API_KEY'] = \"\" # litellm reads OPENAI_API_KEY from .env and sends the request" - ], - "metadata": { - "id": "l4GQ-M5yZ5UW" - }, - "execution_count": 3, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "\n", - "# OpenAI gpt-3.5-turbo-1106\n", - "## Step 1: send the conversation and available functions to the model" - ], - "metadata": { - "id": "AxgR2fCgaRoW" - } - }, - { - "cell_type": "code", - "source": [ - "import litellm\n", - "import json\n", - "# Example dummy function hard coded to return the same weather\n", - "# In production, this could be your backend API or an external API\n", - "def get_current_weather(location, unit=\"fahrenheit\"):\n", - " \"\"\"Get the current weather in a given location\"\"\"\n", - " if \"tokyo\" in location.lower():\n", - " return json.dumps({\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"})\n", - " elif \"san francisco\" in location.lower():\n", - " return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"})\n", - " elif \"paris\" in location.lower():\n", - " return json.dumps({\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"})\n", - " else:\n", - " return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n", - "\n", - "messages = [{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco, Tokyo, and Paris?\"}]\n", - "tools = [\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city and state, e.g. San Francisco, CA\",\n", - " },\n", - " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n", - " },\n", - " \"required\": [\"location\"],\n", - " },\n", - " },\n", - " }\n", - "]\n", - "\n", - "response = litellm.completion(\n", - " model=\"gpt-3.5-turbo-1106\",\n", - " messages=messages,\n", - " tools=tools,\n", - " tool_choice=\"auto\", # auto is default, but we'll be explicit\n", - ")\n", - "print(\"\\nLLM Response1:\\n\", response)\n", - "response_message = response.choices[0].message\n", - "tool_calls = response.choices[0].message.tool_calls\n", - "print(\"\\nTool Choice:\\n\", tool_calls)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Y3qteFo8ZrZP", - "outputId": "ee6c1183-55c1-4111-cdc0-967b8fed9db3" - }, - "execution_count": 18, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\n", - "LLM Response1:\n", - " ModelResponse(id='chatcmpl-8MNdPbrhtnwiPK1x3PEoGwrH144TW', choices=[Choices(finish_reason='tool_calls', index=0, message=Message(content=None, role='assistant', tool_calls=[ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')]))], created=1700344759, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint='fp_eeff13170a', usage={'completion_tokens': 77, 'prompt_tokens': 88, 'total_tokens': 165}, _response_ms=1049.913)\n", - "\n", - "Tool Choice:\n", - " [ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')]\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Step 2 - Parse the Model Response and Execute Functions" - ], - "metadata": { - "id": "tD4lJQ40cU44" - } - }, - { - "cell_type": "code", - "source": [ - "# Check if the model wants to call a function\n", - "if tool_calls:\n", - " # Execute the functions and prepare responses\n", - " available_functions = {\n", - " \"get_current_weather\": get_current_weather,\n", - " }\n", - "\n", - " messages.append(response_message) # Extend conversation with assistant's reply\n", - "\n", - " for tool_call in tool_calls:\n", - " print(f\"\\nExecuting tool call\\n{tool_call}\")\n", - " function_name = tool_call.function.name\n", - " function_to_call = available_functions[function_name]\n", - " function_args = json.loads(tool_call.function.arguments)\n", - " # calling the get_current_weather() function\n", - " function_response = function_to_call(\n", - " location=function_args.get(\"location\"),\n", - " unit=function_args.get(\"unit\"),\n", - " )\n", - " print(f\"Result from tool call\\n{function_response}\\n\")\n", - "\n", - " # Extend conversation with function response\n", - " messages.append(\n", - " {\n", - " \"tool_call_id\": tool_call.id,\n", - " \"role\": \"tool\",\n", - " \"name\": function_name,\n", - " \"content\": function_response,\n", - " }\n", - " )\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "af4oXQvicV_n", - "outputId": "abf6ac3e-4a21-4a4f-b8d7-809b763d0632" - }, - "execution_count": 21, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\n", - "Executing tool call\n", - "ChatCompletionMessageToolCall(id='call_K2Giwoq3NloGPfSv25MJVFZG', function=Function(arguments='{\"location\": \"San Francisco\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n", - "Result from tool call\n", - "{\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"}\n", - "\n", - "\n", - "Executing tool call\n", - "ChatCompletionMessageToolCall(id='call_6K8bYCZK6qsbMY3n51FzE5Nz', function=Function(arguments='{\"location\": \"Tokyo\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n", - "Result from tool call\n", - "{\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"}\n", - "\n", - "\n", - "Executing tool call\n", - "ChatCompletionMessageToolCall(id='call_cKSmUEJGufDwS7TaUHWzp7qx', function=Function(arguments='{\"location\": \"Paris\", \"unit\": \"celsius\"}', name='get_current_weather'), type='function')\n", - "Result from tool call\n", - "{\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"}\n", - "\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Step 3 - Second litellm.completion() call" - ], - "metadata": { - "id": "E3OL1fqUdFdv" - } - }, - { - "cell_type": "code", - "source": [ - "second_response = litellm.completion(\n", - " model=\"gpt-3.5-turbo-1106\",\n", - " messages=messages,\n", - ")\n", - "print(\"Second Response\\n\", second_response)\n", - "print(\"Second Response Message\\n\", second_response.choices[0].message.content)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "8KYB2n-jc1_f", - "outputId": "6c6448ae-1c09-43ae-eb90-208b118e6179" - }, - "execution_count": 26, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Second Response\n", - " ModelResponse(id='chatcmpl-8MNhat166ZqjO6egXcUh85Pd0s7KV', choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"The current weather in San Francisco is 72°F, in Tokyo it's 10°C, and in Paris it's 22°C.\", role='assistant'))], created=1700345018, model='gpt-3.5-turbo-1106', object='chat.completion', system_fingerprint='fp_eeff13170a', usage={'completion_tokens': 28, 'prompt_tokens': 465, 'total_tokens': 493}, _response_ms=999.246)\n", - "Second Response Message\n", - " The current weather in San Francisco is 72°F, in Tokyo it's 10°C, and in Paris it's 22°C.\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Using Azure OpenAI" - ], - "metadata": { - "id": "1cIIFEvXjofp" - } - }, - { - "cell_type": "code", - "source": [ - "# set Azure env variables\n", - "import os\n", - "os.environ['AZURE_API_KEY'] = \"\" # litellm reads AZURE_API_KEY from .env and sends the request\n", - "os.environ['AZURE_API_BASE'] = \"https://openai-gpt-4-test-v-1.openai.azure.com/\"\n", - "os.environ['AZURE_API_VERSION'] = \"2023-07-01-preview\"" - ], - "metadata": { - "id": "lG9mUnModeeE" - }, - "execution_count": 32, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Step 1" - ], - "metadata": { - "id": "17S-Ysksj-E_" - } - }, - { - "cell_type": "code", - "source": [ - "import litellm\n", - "import json\n", - "# Example dummy function hard coded to return the same weather\n", - "# In production, this could be your backend API or an external API\n", - "def get_current_weather(location, unit=\"fahrenheit\"):\n", - " \"\"\"Get the current weather in a given location\"\"\"\n", - " if \"tokyo\" in location.lower():\n", - " return json.dumps({\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"})\n", - " elif \"san francisco\" in location.lower():\n", - " return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"})\n", - " elif \"paris\" in location.lower():\n", - " return json.dumps({\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"})\n", - " else:\n", - " return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n", - "\n", - "messages = [{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco, Tokyo, and Paris?\"}]\n", - "tools = [\n", - " {\n", - " \"type\": \"function\",\n", - " \"function\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city and state, e.g. San Francisco, CA\",\n", - " },\n", - " \"unit\": {\"type\": \"string\", \"enum\": [\"celsius\", \"fahrenheit\"]},\n", - " },\n", - " \"required\": [\"location\"],\n", - " },\n", - " },\n", - " }\n", - "]\n", - "\n", - "response = litellm.completion(\n", - " model=\"azure/chatgpt-functioncalling\", # model = azure/\n", - " messages=messages,\n", - " tools=tools,\n", - " tool_choice=\"auto\", # auto is default, but we'll be explicit\n", - ")\n", - "print(\"\\nLLM Response1:\\n\", response)\n", - "response_message = response.choices[0].message\n", - "tool_calls = response.choices[0].message.tool_calls\n", - "print(\"\\nTool Choice:\\n\", tool_calls)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "boAIHLEXj80m", - "outputId": "00afcf09-5b6b-4805-c374-ba089cc6eb43" - }, - "execution_count": 33, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\n", - "LLM Response1:\n", - " ModelResponse(id='chatcmpl-8MOBPvEnqG7qitkmVqZmCrzSGEmDj', choices=[Choices(finish_reason='tool_calls', index=0, message=Message(content=None, role='assistant', tool_calls=[ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')]))], created=1700346867, model='gpt-35-turbo', object='chat.completion', system_fingerprint=None, usage={'completion_tokens': 19, 'prompt_tokens': 88, 'total_tokens': 107}, _response_ms=833.4319999999999)\n", - "\n", - "Tool Choice:\n", - " [ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')]\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Step 2" - ], - "metadata": { - "id": "hqh1y1IMkmGO" - } - }, - { - "cell_type": "code", - "source": [ - "# Check if the model wants to call a function\n", - "if tool_calls:\n", - " # Execute the functions and prepare responses\n", - " available_functions = {\n", - " \"get_current_weather\": get_current_weather,\n", - " }\n", - "\n", - " messages.append(response_message) # Extend conversation with assistant's reply\n", - "\n", - " for tool_call in tool_calls:\n", - " print(f\"\\nExecuting tool call\\n{tool_call}\")\n", - " function_name = tool_call.function.name\n", - " function_to_call = available_functions[function_name]\n", - " function_args = json.loads(tool_call.function.arguments)\n", - " # calling the get_current_weather() function\n", - " function_response = function_to_call(\n", - " location=function_args.get(\"location\"),\n", - " unit=function_args.get(\"unit\"),\n", - " )\n", - " print(f\"Result from tool call\\n{function_response}\\n\")\n", - "\n", - " # Extend conversation with function response\n", - " messages.append(\n", - " {\n", - " \"tool_call_id\": tool_call.id,\n", - " \"role\": \"tool\",\n", - " \"name\": function_name,\n", - " \"content\": function_response,\n", - " }\n", - " )\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "FGu7DY7PkOiG", - "outputId": "96d39ae7-7fc8-4dd8-c82f-5ee9a486724c" - }, - "execution_count": 34, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\n", - "Executing tool call\n", - "ChatCompletionMessageToolCall(id='call_7gZ0PkmmmgzTOxfF01ATp0U5', function=Function(arguments='{\\n \"location\": \"San Francisco, CA\"\\n}', name='get_current_weather'), type='function')\n", - "Result from tool call\n", - "{\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"}\n", - "\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Step 3" - ], - "metadata": { - "id": "4MjYyeajkpBl" - } - }, - { - "cell_type": "code", - "source": [ - "second_response = litellm.completion(\n", - " model=\"azure/chatgpt-functioncalling\",\n", - " messages=messages,\n", - ")\n", - "print(\"Second Response\\n\", second_response)\n", - "print(\"Second Response Message\\n\", second_response.choices[0].message.content)\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "qHgXyZq1kqGn", - "outputId": "61a30470-d7f5-484d-c42b-681c9b60b34a" - }, - "execution_count": 36, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Second Response\n", - " ModelResponse(id='chatcmpl-8MOC90vwZ2LHX0DE796XYtsOxdGcc', choices=[Choices(finish_reason='stop', index=0, message=Message(content='The current weather in San Francisco is 72°F.', role='assistant'))], created=1700346913, model='gpt-35-turbo', object='chat.completion', system_fingerprint=None, usage={'completion_tokens': 11, 'prompt_tokens': 69, 'total_tokens': 80}, _response_ms=824.882)\n", - "Second Response Message\n", - " The current weather in San Francisco is 72°F.\n" - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/cookbook/Proxy_Batch_Users.ipynb b/cookbook/Proxy_Batch_Users.ipynb deleted file mode 100644 index bd18de5e988..00000000000 --- a/cookbook/Proxy_Batch_Users.ipynb +++ /dev/null @@ -1,208 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "680oRk1af-xJ" - }, - "source": [ - "# Environment Setup\n", - "\n", - "Set `LITELLM_MASTER_KEY` in the notebook environment to the same key configured on the proxy before running the cells" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "X7TgJFn8f88p" - }, - "outputs": [], - "source": [ - "import os\n", - "import csv\n", - "from typing import Optional\n", - "import httpx\n", - "import json\n", - "import asyncio\n", - "\n", - "proxy_base_url = \"http://0.0.0.0:4000\" # 👈 SET TO PROXY URL\n", - "master_key = os.environ[\"LITELLM_MASTER_KEY\"]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "rauw8EOhgBz5" - }, - "outputs": [], - "source": [ - "## GLOBAL HTTP CLIENT ## - faster http calls\n", - "class HTTPHandler:\n", - " def __init__(self, concurrent_limit=1000):\n", - " # Create a client with a connection pool\n", - " self.client = httpx.AsyncClient(\n", - " limits=httpx.Limits(\n", - " max_connections=concurrent_limit,\n", - " max_keepalive_connections=concurrent_limit,\n", - " )\n", - " )\n", - "\n", - " async def close(self):\n", - " # Close the client when you're done with it\n", - " await self.client.aclose()\n", - "\n", - " async def get(\n", - " self, url: str, params: Optional[dict] = None, headers: Optional[dict] = None\n", - " ):\n", - " response = await self.client.get(url, params=params, headers=headers)\n", - " return response\n", - "\n", - " async def post(\n", - " self,\n", - " url: str,\n", - " data: Optional[dict] = None,\n", - " params: Optional[dict] = None,\n", - " headers: Optional[dict] = None,\n", - " ):\n", - " try:\n", - " response = await self.client.post(\n", - " url, data=data, params=params, headers=headers\n", - " )\n", - " return response\n", - " except Exception as e:\n", - " raise e\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "7LXN8zaLgOie" - }, - "source": [ - "# Import Sheet\n", - "\n", - "\n", - "Format: | ID | Name | Max Budget |" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "oiED0usegPGf" - }, - "outputs": [], - "source": [ - "async def import_sheet():\n", - " tasks = []\n", - " http_client = HTTPHandler()\n", - " with open('my-batch-sheet.csv', 'r') as file:\n", - " csv_reader = csv.DictReader(file)\n", - " for row in csv_reader:\n", - " task = create_user(client=http_client, user_id=row['ID'], max_budget=row['Max Budget'], user_name=row['Name'])\n", - " tasks.append(task)\n", - " # print(f\"ID: {row['ID']}, Name: {row['Name']}, Max Budget: {row['Max Budget']}\")\n", - "\n", - " keys = await asyncio.gather(*tasks)\n", - "\n", - " with open('my-batch-sheet_new.csv', 'w', newline='') as new_file:\n", - " fieldnames = ['ID', 'Name', 'Max Budget', 'keys']\n", - " csv_writer = csv.DictWriter(new_file, fieldnames=fieldnames)\n", - " csv_writer.writeheader()\n", - "\n", - " with open('my-batch-sheet.csv', 'r') as file:\n", - " csv_reader = csv.DictReader(file)\n", - " for i, row in enumerate(csv_reader):\n", - " row['keys'] = keys[i] # Add the 'keys' value from the corresponding task result\n", - " csv_writer.writerow(row)\n", - "\n", - " await http_client.close()\n", - "\n", - "asyncio.run(import_sheet())" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "E7M0Li_UgJeZ" - }, - "source": [ - "# Create Users + Keys\n", - "\n", - "- Creates a user\n", - "- Creates a key with max budget" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NZudRFujf7j-" - }, - "outputs": [], - "source": [ - "\n", - "async def create_key_with_alias(client: HTTPHandler, user_id: str, max_budget: float):\n", - " global proxy_base_url\n", - " if not proxy_base_url.endswith(\"/\"):\n", - " proxy_base_url += \"/\"\n", - " url = proxy_base_url + \"key/generate\"\n", - "\n", - " # call /key/generate\n", - " print(\"CALLING /KEY/GENERATE\")\n", - " response = await client.post(\n", - " url=url,\n", - " headers={\"Authorization\": f\"Bearer {master_key}\"},\n", - " data=json.dumps({\n", - " \"user_id\": user_id,\n", - " \"key_alias\": f\"{user_id}-key\",\n", - " \"max_budget\": max_budget # 👈 KEY CHANGE: SETS MAX BUDGET PER KEY\n", - " })\n", - " )\n", - " print(f\"response: {response.text}\")\n", - " return response.json()[\"key\"]\n", - "\n", - "async def create_user(client: HTTPHandler, user_id: str, max_budget: float, user_name: str):\n", - " \"\"\"\n", - " - call /user/new\n", - " - create key for user\n", - " \"\"\"\n", - " global proxy_base_url\n", - " if not proxy_base_url.endswith(\"/\"):\n", - " proxy_base_url += \"/\"\n", - " url = proxy_base_url + \"user/new\"\n", - "\n", - " # call /user/new\n", - " await client.post(\n", - " url=url,\n", - " headers={\"Authorization\": f\"Bearer {master_key}\"},\n", - " data=json.dumps({\n", - " \"user_id\": user_id,\n", - " \"user_alias\": user_name,\n", - " \"auto_create_key\": False,\n", - " # \"max_budget\": max_budget # 👈 [OPTIONAL] Sets max budget per user (if you want to set a max budget across keys)\n", - " })\n", - " )\n", - "\n", - " # create key for user\n", - " return await create_key_with_alias(client=client, user_id=user_id, max_budget=max_budget)\n" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/TogetherAI_liteLLM.ipynb b/cookbook/TogetherAI_liteLLM.ipynb deleted file mode 100644 index d4700914910..00000000000 --- a/cookbook/TogetherAI_liteLLM.ipynb +++ /dev/null @@ -1,1006 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "WemkFEdDAnJL" - }, - "source": [ - "## liteLLM Together AI Tutorial\n", - "https://together.ai/\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "pc6IO4V99O25", - "outputId": "2d69da44-010b-41c2-b38b-5b478576bb8b" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting litellm\n", - " Downloading litellm-0.1.482-py3-none-any.whl (69 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.3/69.3 kB\u001b[0m \u001b[31m757.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: importlib-metadata<7.0.0,>=6.8.0 in /usr/local/lib/python3.10/dist-packages (from litellm) (6.8.0)\n", - "Collecting openai<0.28.0,>=0.27.8 (from litellm)\n", - " Downloading openai-0.27.9-py3-none-any.whl (75 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.5/75.5 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting python-dotenv<2.0.0,>=1.0.0 (from litellm)\n", - " Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n", - "Collecting tiktoken<0.5.0,>=0.4.0 (from litellm)\n", - " Downloading tiktoken-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7.0.0,>=6.8.0->litellm) (3.16.2)\n", - "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (2.31.0)\n", - "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (4.66.1)\n", - "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (3.8.5)\n", - "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm) (2023.6.3)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (3.2.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (2.0.4)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (2023.7.22)\n", - "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (23.1.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (6.0.4)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (4.0.3)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.9.2)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.4.0)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.3.1)\n", - "Installing collected packages: python-dotenv, tiktoken, openai, litellm\n", - "Successfully installed litellm-0.1.482 openai-0.27.9 python-dotenv-1.0.0 tiktoken-0.4.0\n" - ] - } - ], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "TMI3739_9q97" - }, - "outputs": [], - "source": [ - "import os\n", - "from litellm import completion\n", - "os.environ[\"TOGETHERAI_API_KEY\"] = \"\" #@param\n", - "user_message = \"Hello, whats the weather in San Francisco??\"\n", - "messages = [{ \"content\": user_message,\"role\": \"user\"}]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bEqJ2HHjBJqq" - }, - "source": [ - "## Calling togethercomputer/llama-2-70b-chat\n", - "https://api.together.xyz/playground/chat?model=togethercomputer%2Fllama-2-70b-chat" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Jrrt8puj523f", - "outputId": "24494dea-816f-47a6-ade4-1b04f2e9085b" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " 'choices': [\n", - "{\n", - " 'finish_reason': 'stop',\n", - " 'index': 0,\n", - " 'message': {\n", - " 'role': 'assistant',\n", - " 'content': \"\n", - "\n", - "I'm not able to provide real-time weather information. However, I can suggest some ways for you to find out the current weather in San Francisco.\n", - "\n", - "1. Check online weather websites: There are many websites that provide up-to-date weather information, such as AccuWeather, Weather.com, or the National Weather Service. You can enter \"San Francisco\" in the search bar and get the current weather conditions, forecast, and radar imagery.\n", - "2. Use a weather app: You can download a weather app on your smartphone that provides real-time weather information. Some popular weather apps include Dark Sky, Weather Underground, and The Weather Channel.\n", - "3. Tune into local news: You can watch local news channels or listen to local radio stations to get the latest weather forecast and current conditions.\n", - "4. Check social media: Follow local weather accounts on social media platforms like Twitter or Facebook to\"\n", - "}\n", - "}\n", - " ],\n", - " 'created': 1692323365.8261144,\n", - " 'model': 'togethercomputer/llama-2-70b-chat',\n", - " 'usage': {'prompt_tokens': 9, 'completion_tokens': 176, 'total_tokens': 185}\n", - "}\n" - ] - } - ], - "source": [ - "model_name = \"togethercomputer/llama-2-70b-chat\"\n", - "response = completion(model=model_name, messages=messages, max_tokens=200)\n", - "print(response)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "GIUevHlMvPb8", - "outputId": "ad930a12-16e3-4400-fff4-38151e4f6da5" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[92mHere's your LiteLLM Dashboard 👉 \u001b[94m\u001b[4mhttps://admin.litellm.ai/6c0f0403-becb-44af-9724-7201c7d381d0\u001b[0m\n", - "{\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"\\nI'm in San Francisco, and I'm not sure what the weather is like.\\nI'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and I'm not sure what the weather is like. I'm in San Francisco, and\",\n", - " \"role\": \"assistant\"\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1692934243.8663018,\n", - " \"model\": \"togethercomputer/CodeLlama-34b-Instruct\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 9,\n", - " \"completion_tokens\": 178,\n", - " \"total_tokens\": 187\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "model_name = \"togethercomputer/CodeLlama-34b-Instruct\"\n", - "response = completion(model=model_name, messages=messages, max_tokens=200)\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sfWtgf-mBQcM" - }, - "source": [ - "## With Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "background_save": true, - "base_uri": "https://localhost:8080/" - }, - "id": "wuBhlZtC6MH5", - "outputId": "8f4a408c-25eb-4434-cdd4-7b4ae4f6d3aa" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Com'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'bin'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ator'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' ('}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ')'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' are'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' two'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' popular'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ators'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' have'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' gained'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' recognition'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' effect'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'iveness'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'urt'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'uring'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' scaling'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' early'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'stage'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ities'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' they'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' also'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' have'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' distinct'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' differences'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' set'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' them'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' apart'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' In'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' this'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' ess'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ay'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' we'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' will'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' explore'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' key'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' features'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' discuss'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' which'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' might'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Com'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'bin'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ator'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' one'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' most'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' successful'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ators'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' world'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' port'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'folio'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' includes'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Air'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'b'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'nb'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Drop'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'box'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Red'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'dit'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' F'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ounded'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' '}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '2'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '5'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' has'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ed'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' over'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' '}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '1'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '9'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' start'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ups'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' combined'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' valu'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ation'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' over'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' $'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '1'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' billion'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' The'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' known'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' its'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' inten'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'se'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' three'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'month'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' boot'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' camp'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'style'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' format'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' where'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' work'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' closely'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' experienced'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' ment'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ors'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' develop'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' products'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' ref'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ine'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' business'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' models'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' prepare'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ra'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ising'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' software'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' technology'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' internet'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' start'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ups'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' has'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' strong'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' track'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' record'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' ident'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ifying'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'urt'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'uring'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' successful'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' these'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' spaces'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' other'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' hand'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' relatively'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' new'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ator'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' was'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' founded'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' '}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '2'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '1'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '7'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' While'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' it'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' not'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' have'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' same'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' level'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' brand'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' recognition'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' as'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' has'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' quickly'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' gained'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' reputation'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' its'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' unique'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' approach'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceleration'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' The'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'es'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' supporting'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' under'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 're'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'present'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ed'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' particularly'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' women'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' people'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' color'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' provides'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' range'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' resources'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' support'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' help'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' these'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' succeed'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' designed'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' more'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' flexible'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' personal'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ized'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' than'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' traditional'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ators'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' connecting'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' ment'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ors'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' resources'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' are'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' tail'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ored'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' specific'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' needs'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'One'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' key'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' difference'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' between'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' type'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' they'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' support'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'es'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' primarily'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' software'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' technology'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' internet'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' start'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ups'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' while'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' has'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' bro'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ader'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' includes'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' range'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' indust'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ries'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' such'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' as'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' health'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'care'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fin'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ance'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' consumer'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' products'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' This'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' means'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' if'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' non'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'tech'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' industry'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'An'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'other'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' difference'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' between'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' two'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' programs'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' their'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' approach'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' provides'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' seed'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' all'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' its'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' port'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'folio'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' typically'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' range'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' of'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' $'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '1'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' $'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '2'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '0'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' In'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' contrast'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' does'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' not'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' provide'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' its'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' port'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'folio'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' but'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' instead'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'es'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' connecting'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' found'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ers'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' invest'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ors'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' resources'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' can'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' help'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' them'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' raise'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' capital'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' This'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' means'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' if'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' looking'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' option'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'So'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' which'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' right'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '?'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' It'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' ultimately'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' depends'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' on'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' specific'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' needs'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' goals'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' If'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' non'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'tech'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' industry'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' bro'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ader'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' focus'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Additionally'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' if'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 're'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' looking'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' more'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' personal'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ized'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' flexible'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' approach'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceleration'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' choice'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' On'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' other'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' hand'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' if'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' software'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' technology'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' or'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' internet'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' space'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 're'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' looking'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' seed'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fund'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ing'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' may'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' be'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' a'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' better'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '\\n'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'In'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' conclusion'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Y'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'C'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' l'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ite'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'LL'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'M'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' are'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' both'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' excellent'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' acceler'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ators'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' can'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' provide'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' valuable'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' resources'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' support'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' early'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'stage'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' companies'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' While'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' they'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' share'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' some'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' similar'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'ities'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' they'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' also'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' have'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' distinct'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' differences'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' that'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' set'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' them'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' apart'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' By'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' considering'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' startup'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 's'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' specific'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' needs'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' goals'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' can'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' determine'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' which'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' program'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' is'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' the'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' best'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' fit'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' business'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n" - ] - } - ], - "source": [ - "user_message = \"Write 1page essay on YC + liteLLM\"\n", - "messages = [{ \"content\": user_message,\"role\": \"user\"}]\n", - "\n", - "\n", - "async def parse_stream(stream):\n", - " async for elem in stream:\n", - " print(elem)\n", - " return\n", - "\n", - "stream = completion(model=\"togethercomputer/llama-2-70b-chat\", messages=messages, stream=True, max_tokens=800)\n", - "print(stream)\n", - "\n", - "# Await the asynchronous function directly in the notebook cell\n", - "await parse_stream(stream)\n" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/Using_Nemo_Guardrails_with_LiteLLM_Server.ipynb b/cookbook/Using_Nemo_Guardrails_with_LiteLLM_Server.ipynb deleted file mode 100644 index 0c3ff97a373..00000000000 --- a/cookbook/Using_Nemo_Guardrails_with_LiteLLM_Server.ipynb +++ /dev/null @@ -1,157 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "eKXncoQbU_2j" - }, - "source": [ - "# Using Nemo-Guardrails with LiteLLM Server\n", - "\n", - "[Call Bedrock, TogetherAI, Huggingface, etc. on the server](https://docs.litellm.ai/docs/providers)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZciYaLwvuFbu" - }, - "source": [ - "## Using with Bedrock\n", - "\n", - "`docker run -e PORT=8000 -e AWS_ACCESS_KEY_ID= -e AWS_SECRET_ACCESS_KEY= -p 8000:8000 ghcr.io/berriai/litellm:latest`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "vOUwGSJ2Vsy3" - }, - "outputs": [], - "source": [ - "pip install nemoguardrails langchain" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "xXEJNxe7U0IN" - }, - "outputs": [], - "source": [ - "from langchain.chat_models import ChatOpenAI\n", - "\n", - "llm = ChatOpenAI(model_name=\"anthropic.claude-v2\", openai_api_base=\"http://0.0.0.0:8000\", openai_api_key=\"my-fake-key\")\n", - "\n", - "from nemoguardrails import LLMRails, RailsConfig\n", - "\n", - "config = RailsConfig.from_path(\"./config.yml\")\n", - "app = LLMRails(config, llm=llm)\n", - "\n", - "new_message = app.generate(messages=[{\n", - " \"role\": \"user\",\n", - " \"content\": \"Hello! What can you do for me?\"\n", - "}])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vz5n00qyuKjp" - }, - "source": [ - "## Using with TogetherAI\n", - "\n", - "1. You can either set this in the server environment:\n", - "`docker run -e PORT=8000 -e TOGETHERAI_API_KEY= -p 8000:8000 ghcr.io/berriai/litellm:latest`\n", - "\n", - "2. **Or** Pass this in as the api key `(...openai_api_key=\"\")`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "XK1sk-McuhpE" - }, - "outputs": [], - "source": [ - "from langchain.chat_models import ChatOpenAI\n", - "\n", - "llm = ChatOpenAI(model_name=\"together_ai/togethercomputer/CodeLlama-13b-Instruct\", openai_api_base=\"http://0.0.0.0:8000\", openai_api_key=\"my-together-ai-api-key\")\n", - "\n", - "from nemoguardrails import LLMRails, RailsConfig\n", - "\n", - "config = RailsConfig.from_path(\"./config.yml\")\n", - "app = LLMRails(config, llm=llm)\n", - "\n", - "new_message = app.generate(messages=[{\n", - " \"role\": \"user\",\n", - " \"content\": \"Hello! What can you do for me?\"\n", - "}])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8A1KWKnzuxAS" - }, - "source": [ - "### CONFIG.YML\n", - "\n", - "save this example `config.yml` in your current directory" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NKN1GmSvu0Cx" - }, - "outputs": [], - "source": [ - "# instructions:\n", - "# - type: general\n", - "# content: |\n", - "# Below is a conversation between a bot and a user about the recent job reports.\n", - "# The bot is factual and concise. If the bot does not know the answer to a\n", - "# question, it truthfully says it does not know.\n", - "\n", - "# sample_conversation: |\n", - "# user \"Hello there!\"\n", - "# express greeting\n", - "# bot express greeting\n", - "# \"Hello! How can I assist you today?\"\n", - "# user \"What can you do for me?\"\n", - "# ask about capabilities\n", - "# bot respond about capabilities\n", - "# \"I am an AI assistant that helps answer mathematical questions. My core mathematical skills are powered by wolfram alpha.\"\n", - "# user \"What's 2+2?\"\n", - "# ask math question\n", - "# bot responds to math question\n", - "# \"2+2 is equal to 4.\"\n", - "\n", - "# models:\n", - "# - type: main\n", - "# engine: openai\n", - "# model: claude-instant-1" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/VLLM_Model_Testing.ipynb b/cookbook/VLLM_Model_Testing.ipynb deleted file mode 100644 index 0cacac66133..00000000000 --- a/cookbook/VLLM_Model_Testing.ipynb +++ /dev/null @@ -1,404 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [], - "machine_shape": "hm", - "gpuType": "V100" - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - }, - "accelerator": "GPU" - }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "# Set up Environment" - ], - "metadata": { - "id": "vDOm5wfjdFLP" - } - }, - { - "cell_type": "code", - "source": [ - "!pip install --upgrade litellm" - ], - "metadata": { - "id": "Bx6mAA6MHiy_" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "zIYv7JTyxSxR", - "outputId": "53890320-f9fa-4bf4-8362-0f17f52c6ed4" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Successfully installed fastapi-0.103.1 h11-0.14.0 huggingface-hub-0.16.4 ninja-1.11.1 pydantic-1.10.12 ray-2.6.3 safetensors-0.3.3 sentencepiece-0.1.99 starlette-0.27.0 tokenizers-0.13.3 transformers-4.33.1 uvicorn-0.23.2 vllm-0.1.4 xformers-0.0.21\n" - ] - } - ], - "source": [ - "!pip install vllm" - ] - }, - { - "cell_type": "markdown", - "source": [ - "# Load the Logs" - ], - "metadata": { - "id": "RMcoAni6WKEx" - } - }, - { - "cell_type": "code", - "source": [ - "import pandas as pd" - ], - "metadata": { - "id": "zchxB8c7WJe5" - }, - "execution_count": 4, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "# path of the csv file\n", - "file_path = 'Model-prompts-example.csv'\n", - "\n", - "# load the csv file as a pandas DataFrame\n", - "data = pd.read_csv(file_path)\n", - "\n", - "data.head()" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 81 - }, - "id": "aKcWr015WNPm", - "outputId": "6e226773-333f-46a2-9fc8-4f54f309d204" - }, - "execution_count": 6, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - " Success Timestamp Input \\\n", - "0 True 1694041195 This is the templated query input \n", - "\n", - " Output RunId (Wandb Runid) \\\n", - "0 This is the query output from the model 8hlumwuk \n", - "\n", - " Model ID (or Name) \n", - "0 OpenAI/Turbo-3.5 " - ], - "text/html": [ - "\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
SuccessTimestampInputOutputRunId (Wandb Runid)Model ID (or Name)
0True1694041195This is the templated query inputThis is the query output from the model8hlumwukOpenAI/Turbo-3.5
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "\n", - " \n", - "\n", - " \n", - "
\n", - "\n", - "\n", - "
\n", - "
\n" - ] - }, - "metadata": {}, - "execution_count": 6 - } - ] - }, - { - "cell_type": "code", - "source": [ - "input_texts = data['Input'].values" - ], - "metadata": { - "id": "0DbL-kirWUyn" - }, - "execution_count": 7, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "messages = [[{\"role\": \"user\", \"content\": input_text}] for input_text in input_texts]" - ], - "metadata": { - "id": "cqpAvy8hWXyC" - }, - "execution_count": 8, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Running Inference" - ], - "metadata": { - "id": "SugCyom0Xy8U" - } - }, - { - "cell_type": "code", - "source": [ - "from litellm import batch_completion\n", - "model_name = \"facebook/opt-125m\"\n", - "provider = \"vllm\"\n", - "response_list = batch_completion(\n", - " model=model_name,\n", - " custom_llm_provider=provider, # can easily switch to huggingface, replicate, together ai, sagemaker, etc.\n", - " messages=messages,\n", - " temperature=0.2,\n", - " max_tokens=80,\n", - " )" - ], - "metadata": { - "id": "qpikx3uxHns3" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "response_list" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "QDPikHtwKJJ2", - "outputId": "06f47c44-e258-452a-f9db-232a5b6d2810" - }, - "execution_count": 10, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "[ JSON: {\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \".\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1694053363.6139505,\n", - " \"model\": \"facebook/opt-125m\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 9,\n", - " \"completion_tokens\": 80,\n", - " \"total_tokens\": 89\n", - " }\n", - " }]" - ] - }, - "metadata": {}, - "execution_count": 10 - } - ] - }, - { - "cell_type": "code", - "source": [ - "response_values = [response['choices'][0]['message']['content'] for response in response_list]" - ], - "metadata": { - "id": "SYqTcCiJbQDF" - }, - "execution_count": 11, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "response_values" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "wqs-Oy9FbiPo", - "outputId": "16a6a7b7-97c8-4b5b-eff8-09ea5eb5ad06" - }, - "execution_count": 12, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "['.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is']" - ] - }, - "metadata": {}, - "execution_count": 12 - } - ] - }, - { - "cell_type": "code", - "source": [ - "data[f\"{model_name}_output\"] = response_values" - ], - "metadata": { - "id": "mElNbBehbkrz" - }, - "execution_count": 13, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "data.to_csv('model_responses.csv', index=False)" - ], - "metadata": { - "id": "F06NXssDc45k" - }, - "execution_count": 14, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md deleted file mode 100644 index e1f606a2eb7..00000000000 --- a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md +++ /dev/null @@ -1,78 +0,0 @@ -# Use your ChatGPT subscription through LiteLLM - -Sign into ChatGPT in Codex or Pi and send model requests through your existing LiteLLM proxy. The client keeps your login and refreshes it. Each request uses that client's ChatGPT account, while a separate LiteLLM virtual key controls gateway access and usage attribution. - -An administrator adds a `chatgpt/` model using the usual model configuration. No ChatGPT login needs to be added to the proxy. This uses the same separate-key convention as provider passthrough, including Claude Code subscription forwarding. - -## Add a model to the proxy - -Reuse your existing proxy. If you need one, follow the [proxy quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start). Merge this entry into its model list and restart the proxy: - -```yaml -model_list: - - model_name: gpt-6-luna - litellm_params: - model: chatgpt/gpt-6-luna - model_info: - mode: responses -``` - -Use a model available to the caller's subscription. The virtual key must allow the exposed model name. Leave saved ChatGPT logins and API credentials unset for a client-only deployment. Explicit cross-provider fallbacks can incur separate API charges; do not configure them for subscription-only traffic. - -## Configure Codex - -Use your existing LiteLLM URL and a virtual key assigned to your gateway user. Export that key as `LITELLM_API_KEY` in the environment that launches Codex. Keep the value out of committed configuration. - -Add this provider to `~/.codex/config.toml`, preserving your other settings: - -```toml -model = "gpt-6-luna" -model_provider = "litellm_subscription" - -[model_providers.litellm_subscription] -name = "LiteLLM subscription" -base_url = "https://litellm.example.com/v1" -wire_api = "responses" -supports_websockets = false -requires_openai_auth = true -env_http_headers = { "x-litellm-api-key" = "LITELLM_API_KEY" } -``` - -Replace `https://litellm.example.com` with your proxy URL. A local proxy normally uses `http://localhost:4000/v1`. Launch Codex, sign into ChatGPT if needed, and send a prompt with a model available to your subscription. The selected model must match the name exposed by the gateway. - -The `lite` CLI can set up this provider each time it starts Codex, with no `config.toml` edit. Export `LITELLM_PROXY_URL` and `LITELLM_PROXY_API_KEY`, then run `lite codex --subscription`. Run `lite config set codex_subscription true` to make it the default. - -For a managed workspace rollout, use OpenAI's [gateway configuration guide](https://learn.chatgpt.com/docs/enterprise/sign-in-with-chatgpt-through-a-gateway). - -## Native compatibility and Pi - -For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials but does not need a model deployment or resolve model aliases. Restricted keys must allow the upstream model name. - -### Configure Pi - -Sign into the built-in `openai-codex` provider in Pi. Merge this override into `~/.pi/agent/models.json` and export your LiteLLM virtual key as `LITELLM_API_KEY` before launching Pi: - -```json -{ - "providers": { - "openai-codex": { - "baseUrl": "https://litellm.example.com/backend-api/codex", - "headers": { - "x-litellm-api-key": "LITELLM_API_KEY" - } - } - } -} -``` - -Keep Pi's subscription login as the provider credential. The header value names an environment variable, not a key to paste into the file. - -## Verify account usage - -Send a prompt and check the completed request in LiteLLM's Logs page. Its gateway user/key and token counts should match the caller. Give each person their own LiteLLM virtual key for separate attribution. The gateway credential and ChatGPT login are independently authenticated; LiteLLM does not establish that both identities belong to the same person. - -The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key must allow that name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions. - -The proxy forwards the ChatGPT bearer and account header to the fixed OpenAI Codex endpoint. It removes gateway credentials and cookies from the upstream request and redacts credential headers in logging callbacks. The client remains responsible for refresh and reauthentication. Native passthrough returns authentication and quota errors without switching subscriptions. Standard model routes retain the caller login across ChatGPT retries; explicitly configured cross-provider fallbacks use their own credentials. An explicit workspace header is preserved for OpenAI to validate. - -This flow does not connect accounts through the dashboard or pool subscription allowances. Use a Codex build that supports ChatGPT authentication with a custom model provider. The integration forwards HTTP and SSE requests; it does not add native WebSocket or other Codex backend routes. diff --git a/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md b/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md deleted file mode 100644 index cd27eed18a3..00000000000 --- a/cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md +++ /dev/null @@ -1,294 +0,0 @@ -# Claude Code with LiteLLM Quickstart - -This guide shows how to call Claude models (and any LiteLLM-supported model) through LiteLLM proxy from Claude Code. - -> **Note:** This integration is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). It allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls. - -## Video Walkthrough - -Watch the full tutorial: https://www.loom.com/embed/3c17d683cdb74d36a3698763cc558f56 - -## Prerequisites - -- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed -- API keys for your chosen providers - -## Installation - -First, install LiteLLM with proxy support: - -```bash -pip install 'litellm[proxy]' -``` - -## Step 1: Setup config.yaml - -Create a secure configuration using environment variables: - -```yaml -model_list: - # Claude models - - model_name: claude-3-5-sonnet-20241022 - litellm_params: - model: anthropic/claude-3-5-sonnet-20241022 - api_key: os.environ/ANTHROPIC_API_KEY - - - model_name: claude-3-5-haiku-20241022 - litellm_params: - model: anthropic/claude-3-5-haiku-20241022 - api_key: os.environ/ANTHROPIC_API_KEY - - -litellm_settings: - master_key: os.environ/LITELLM_MASTER_KEY -``` - -Set your environment variables: - -```bash -export ANTHROPIC_API_KEY="your-anthropic-api-key" -export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)" -``` - -## Step 2: Start Proxy - -```bash -litellm --config /path/to/config.yaml - -# RUNNING on http://0.0.0.0:4000 -``` - -## Step 3: Verify Setup - -Test that your proxy is working correctly: - -```bash -curl -X POST http://0.0.0.0:4000/v1/messages \ --H "Authorization: Bearer $LITELLM_MASTER_KEY" \ --H "Content-Type: application/json" \ --d '{ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 1000, - "messages": [{"role": "user", "content": "What is the capital of France?"}] -}' -``` - -## Step 4: Configure Claude Code - -### Method 1: Unified Endpoint (Recommended) - -Configure Claude Code to use LiteLLM's unified endpoint. Either a virtual key or master key can be used here: - -```bash -export ANTHROPIC_BASE_URL="http://0.0.0.0:4000" -export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY" -``` - -> **Tip:** LITELLM_MASTER_KEY gives Claude access to all proxy models, whereas a virtual key would be limited to the models set in the UI. - -### Method 2: Provider-specific Pass-through Endpoint - -Alternatively, use the Anthropic pass-through endpoint: - -```bash -export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic" -export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY" -``` - -## Step 5: Use Claude Code - -### Choosing Your Model - -You have two options for specifying which model Claude Code uses: - -#### Option 1: Command Line / Session Model Selection - -Specify the model directly when starting Claude Code or during a session: - -```bash -# Specify model at startup -claude --model claude-3-5-sonnet-20241022 - -# Or change model during a session -/model claude-3-5-haiku-20241022 -``` - -This method uses the exact model you specify. - -#### Option 2: Environment Variables - -Configure default models using environment variables: - -```bash -# Tell Claude Code which models to use by default -export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022 -export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022 -export ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-3-5-20240229 - -claude # Will use the models specified above -``` - -**Note:** Claude Code may cache the model from a previous session. If environment variables don't take effect, use Option 1 to explicitly set the model. - -**Important:** The `model_name` in your LiteLLM config must match what Claude Code requests (either from env vars or command line). - -### Using 1M Context Window - -Claude Code supports extended context (1 million tokens) using the `[1m]` suffix with Claude 4+ models: - -```bash -# Use Sonnet 4.5 with 1M context (requires quotes for shell) -claude --model 'claude-sonnet-4-5-20250929[1m]' - -# Inside a Claude Code session (no quotes needed) -/model claude-sonnet-4-5-20250929[1m] -``` - -**Important:** When using `--model` with `[1m]` in the shell, you must use quotes to prevent the shell from interpreting the brackets. - -Alternatively, set as default with environment variables: - -```bash -export ANTHROPIC_DEFAULT_SONNET_MODEL='claude-sonnet-4-5-20250929[1m]' -claude -``` - -**How it works:** -- Claude Code strips the `[1m]` suffix before sending to LiteLLM -- Claude Code automatically adds the header `anthropic-beta: context-1m-2025-08-07` -- Your LiteLLM config should **NOT** include `[1m]` in model names - -**Verify 1M context is active:** -```bash -/context -# Should show: 21k/1000k tokens (2%) -``` - -**Pricing:** Models using 1M context have different pricing. Input tokens above 200k are charged at a higher rate. - -## Troubleshooting - -Common issues and solutions: - -**Claude Code not connecting:** -- Verify your proxy is running: `curl http://0.0.0.0:4000/health` -- Check that `ANTHROPIC_BASE_URL` is set correctly -- Ensure your `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key - -**Authentication errors:** -- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY` -- Check that your API keys are valid and have sufficient credits -- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key - -**Model not found:** -- Check what model Claude Code is requesting in LiteLLM logs -- Ensure your `config.yaml` has a matching `model_name` entry -- If using environment variables, verify they're set: `echo $ANTHROPIC_DEFAULT_SONNET_MODEL` - -**1M context not working (showing 200k instead of 1000k):** -- Verify you're using the `[1m]` suffix: `/model your-model-name[1m]` -- Check LiteLLM logs for the header `context-1m-2025-08-07` in the request -- Ensure your model supports 1M context (only certain Claude models do) -- Your LiteLLM config should **NOT** include `[1m]` in the `model_name` - -## Using Multiple Models and Providers - -You can configure LiteLLM to route to any supported provider. Here's an example with multiple providers: - -```yaml -model_list: - # OpenAI models - - model_name: codex-mini - litellm_params: - model: openai/codex-mini - api_key: os.environ/OPENAI_API_KEY - api_base: https://api.openai.com/v1 - - - model_name: o3-pro - litellm_params: - model: openai/o3-pro - api_key: os.environ/OPENAI_API_KEY - api_base: https://api.openai.com/v1 - - - model_name: gpt-4o - litellm_params: - model: openai/gpt-4o - api_key: os.environ/OPENAI_API_KEY - api_base: https://api.openai.com/v1 - - # Anthropic models - - model_name: claude-3-5-sonnet-20241022 - litellm_params: - model: anthropic/claude-3-5-sonnet-20241022 - api_key: os.environ/ANTHROPIC_API_KEY - - - model_name: claude-3-5-haiku-20241022 - litellm_params: - model: anthropic/claude-3-5-haiku-20241022 - api_key: os.environ/ANTHROPIC_API_KEY - - # AWS Bedrock - - model_name: claude-bedrock - litellm_params: - model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 - aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID - aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY - aws_region_name: us-east-1 - -litellm_settings: - master_key: os.environ/LITELLM_MASTER_KEY -``` - -**Note:** The `model_name` can be anything you choose. Claude Code will request whatever model you specify (via env vars or command line), and LiteLLM will route to the `model` configured in `litellm_params`. - -Switch between models seamlessly: - -```bash -# Use environment variables to set defaults -export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022 -export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022 - -# Or specify directly -claude --model claude-3-5-sonnet-20241022 # Complex reasoning -claude --model claude-3-5-haiku-20241022 # Fast responses -claude --model claude-bedrock # Bedrock deployment -``` - -## Default Models Used by Claude Code - -If you **don't** set environment variables, Claude Code uses these default model names: - -| Purpose | Default Model Name (v2.1.14) | -|---------|------------------------------| -| Main model | `claude-sonnet-4-5-20250929` | -| Light tasks (subagents, summaries) | `claude-haiku-4-5-20251001` | -| Planning mode | `claude-opus-4-5-20251101` | - -Your LiteLLM config should include these model names if you want Claude Code to work without setting environment variables: - -```yaml -model_list: - - model_name: claude-sonnet-4-5-20250929 - litellm_params: - # Can be any provider - Anthropic, Bedrock, Vertex AI, etc. - model: anthropic/claude-sonnet-4-5-20250929 - api_key: os.environ/ANTHROPIC_API_KEY - - - model_name: claude-haiku-4-5-20251001 - litellm_params: - model: anthropic/claude-haiku-4-5-20251001 - api_key: os.environ/ANTHROPIC_API_KEY - - - model_name: claude-opus-4-5-20251101 - litellm_params: - model: anthropic/claude-opus-4-5-20251101 - api_key: os.environ/ANTHROPIC_API_KEY -``` - -**Warning:** These default model names may change with new Claude Code versions. Check LiteLLM proxy logs for "model not found" errors to identify what Claude Code is requesting. - -## Additional Resources - -- [LiteLLM Documentation](https://docs.litellm.ai/) -- [Claude Code Documentation](https://docs.anthropic.com/en/docs/claude-code/overview) -- [Anthropic's LiteLLM Configuration Guide](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration) diff --git a/cookbook/ai_coding_tool_guides/index.json b/cookbook/ai_coding_tool_guides/index.json deleted file mode 100644 index 3e71670d623..00000000000 --- a/cookbook/ai_coding_tool_guides/index.json +++ /dev/null @@ -1,134 +0,0 @@ -[{ - "title": "Claude Code Quickstart", - "description": "This is a quickstart guide to using Claude Code with LiteLLM.", - "url": "https://docs.litellm.ai/docs/tutorials/claude_responses_api", - "date": "2026-01-15", - "version": "1.0.0", - "tags": [ - "Claude Code", - "LiteLLM" - ] -}, -{ - "title": "Claude Code with MCPs", - "description": "This is a guide to using Claude Code with MCPs via LiteLLM Proxy.", - "url": "https://docs.litellm.ai/docs/tutorials/claude_mcp", - "date": "2026-01-15", - "version": "1.0.0", - "tags": [ - "Claude Code", - "LiteLLM", - "MCP" - ] -}, -{ - "title": "Claude Code with Non-Anthropic Models", - "description": "This is a guide to using Claude Code with non-Anthropic models via LiteLLM Proxy.", - "url": "https://docs.litellm.ai/docs/tutorials/claude_non_anthropic_models", - "date": "2026-01-16", - "version": "1.0.0", - "tags": [ - "Claude Code", - "LiteLLM", - "OpenAI", - "Gemini" - ] -}, -{ - "title": "Cursor Quickstart", - "description": "This is a quickstart guide to using Cursor with LiteLLM.", - "url": "https://docs.litellm.ai/docs/tutorials/cursor_integration", - "date": "2026-01-16", - "version": "1.0.0", - "tags": [ - "Cursor", - "LiteLLM", - "Quickstart" - ] -}, -{ - "title": "Github Copilot Quickstart", - "description": "This is a quickstart guide to using Github Copilot with LiteLLM.", - "url": "https://docs.litellm.ai/docs/tutorials/github_copilot_integration", - "date": "2026-01-16", - "version": "1.0.0", - "tags": [ - "Github Copilot", - "LiteLLM", - "Quickstart" - ] -}, -{ - "title": "LiteLLM Gemini CLI Quickstart", - "description": "This is a quickstart guide to using LiteLLM Gemini CLI.", - "url": "https://docs.litellm.ai/docs/tutorials/litellm_gemini_cli", - "date": "2026-01-16", - "version": "1.0.0", - "tags": [ - "Gemini CLI", - "Gemini", - "LiteLLM", - "Quickstart" - ] -}, -{ - "title": "OpenAI Codex CLI Quickstart", - "description": "This is a quickstart guide to using OpenAI Codex CLI.", - "url": "https://docs.litellm.ai/docs/tutorials/openai_codex", - "date": "2026-01-16", - "version": "1.0.0", - "tags": [ - "OpenAI Codex CLI", - "OpenAI", - "LiteLLM", - "Quickstart" - ] -}, -{ - "title": "OpenWebUI Quickstart", - "description": "This is a quickstart guide to using OpenWebUI with LiteLLM.", - "url": "https://docs.litellm.ai/docs/tutorials/openweb_ui", - "date": "2026-01-16", - "version": "1.0.0", - "tags": [ - "OpenWebUI", - "LiteLLM", - "Quickstart" - ] -}, -{ - "title": "AI Coding Tool Usage Tracking", - "description": "This is a guide to tracking usage for AI coding tools monitor the use of Claude Code , Google Antigravity, OpenAI Codex, Roo Code etc. through LiteLLM.", - "url": "https://docs.litellm.ai/docs/tutorials/cost_tracking_coding", - "date": "2026-01-17", - "version": "1.0.0", - "tags": [ - "Claude Code", - "Gemini CLI", - "OpenAI Codex", - "LiteLLM" - ] -}, -{ - "title": "Use Web Search with Claude Code (across Bedrock/OpenAI/Gemini/etc.)", - "description": "This is a guide for using Web Search with Claude Code via LiteLLM.", - "url": "https://docs.litellm.ai/docs/tutorials/claude_code_websearch", - "date": "2026-01-17", - "version": "1.0.0", - "tags": [ - "Claude Code", - "LiteLLM", - "Web Search" - ] -}, -{ - "title": "Track Claude Code Usage per user via Custom Headers", - "description": "This is a guide for tracking claude code user usage by passing a customer ID header.", - "url": "https://docs.litellm.ai/docs/tutorials/claude_code_customer_tracking", - "date": "2026-01-17", - "version": "1.0.0", - "tags": [ - "Claude Code", - "LiteLLM" - ] -}] \ No newline at end of file diff --git a/cookbook/anthropic_agent_sdk/README.md b/cookbook/anthropic_agent_sdk/README.md deleted file mode 100644 index 4521aee310a..00000000000 --- a/cookbook/anthropic_agent_sdk/README.md +++ /dev/null @@ -1,150 +0,0 @@ -# Claude Agent SDK with LiteLLM Gateway - -A simple example showing how to use Claude's Agent SDK with LiteLLM as a proxy. This lets you use any LLM provider (OpenAI, Bedrock, Azure, etc.) through the Agent SDK. - -## Quick Start - -### 1. Install dependencies - -```bash -pip install anthropic claude-agent-sdk litellm -``` - -### 2. Start LiteLLM proxy - -```bash -# Simple start with Claude -litellm --model claude-sonnet-4-20250514 - -# Or with a config file -litellm --config config.yaml -``` - -Generate a master key before starting the proxy: - -```bash -export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)" -``` - -### 3. Run the chat - -**Basic Agent (no MCP):** - -```bash -python main.py -``` - -**Agent with MCP (DeepWiki2 for research):** - -```bash -python agent_with_mcp.py -``` - -If MCP connection fails, you can disable it: - -```bash -USE_MCP=false python agent_with_mcp.py -``` - -That's it! You can now chat with the agent in your terminal. - -### Chat Commands - -While chatting, you can use these commands: -- `models` - List all available models (fetched from your LiteLLM proxy) -- `model` - Switch to a different model -- `clear` - Start a new conversation -- `quit` or `exit` - End the chat - -The chat automatically fetches available models from your LiteLLM proxy's `/models` endpoint, so you'll always see what's currently configured. - -## Configuration - -Set these environment variables if needed: - -```bash -export LITELLM_PROXY_URL="http://localhost:4000" -export LITELLM_API_KEY="$LITELLM_MASTER_KEY" -export LITELLM_MODEL="bedrock-claude-sonnet-4.5" -``` - -Or just use the defaults - it'll connect to `http://localhost:4000` by default. - -## Files - -- `main.py` - Basic interactive agent without MCP -- `agent_with_mcp.py` - Agent with MCP server integration (DeepWiki2) -- `common.py` - Shared utilities and functions -- `config.example.yaml` - Example LiteLLM configuration -- `requirements.txt` - Python dependencies - -## Example Config File - -If you want to use multiple models, create a `config.yaml` (see `config.example.yaml`): - -```yaml -model_list: - - model_name: bedrock-claude-sonnet-4 - litellm_params: - model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0" - aws_region_name: "us-east-1" - - - model_name: bedrock-claude-sonnet-4.5 - litellm_params: - model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" - aws_region_name: "us-east-1" -``` - -Then start LiteLLM with: `litellm --config config.yaml` - -## How It Works - -The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic: - -```python -# Point to LiteLLM gateway (not Anthropic) -os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000" -os.environ["ANTHROPIC_API_KEY"] = os.environ["LITELLM_API_KEY"] - -# Use any model configured in LiteLLM -options = ClaudeAgentOptions( - model="bedrock-claude-sonnet-4", # or gpt-4, or anything else - system_prompt="You are a helpful assistant.", - max_turns=50, -) -``` - -Note: Don't add `/anthropic` to the base URL - LiteLLM handles the routing automatically. - -## Why Use This? - -- **Switch providers easily**: Use the same code with OpenAI, Bedrock, Azure, etc. -- **Cost tracking**: LiteLLM tracks spending across all your agent conversations -- **Rate limiting**: Set budgets and limits on your agent usage -- **Load balancing**: Distribute requests across multiple API keys or regions -- **Fallbacks**: Automatically retry with a different model if one fails - -## Troubleshooting - -**Connection errors?** -- Make sure LiteLLM is running: `litellm --model your-model` -- Check the URL is correct (default: `http://localhost:4000`) - -**Authentication errors?** -- Verify your LiteLLM API key is correct -- Make sure the model is configured in your LiteLLM setup - -**Model not found?** -- Check the model name matches what's in your LiteLLM config -- Run `litellm --model your-model` to test it works - -**Agent with MCP stuck or failing?** -- The MCP server might not be available at `http://localhost:4000/mcp/deepwiki2` -- Try disabling MCP: `USE_MCP=false python agent_with_mcp.py` -- Or use the basic agent: `python main.py` - -## Learn More - -- [LiteLLM Docs](https://docs.litellm.ai/) -- [Claude Agent SDK](https://github.com/anthropics/anthropic-agent-sdk) -- [LiteLLM Proxy Guide](https://docs.litellm.ai/docs/proxy/quick_start) diff --git a/cookbook/anthropic_agent_sdk/agent_with_mcp.py b/cookbook/anthropic_agent_sdk/agent_with_mcp.py deleted file mode 100644 index 8a7513c786c..00000000000 --- a/cookbook/anthropic_agent_sdk/agent_with_mcp.py +++ /dev/null @@ -1,144 +0,0 @@ -""" -Interactive Claude Agent SDK CLI with MCP Support - -This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy, -with MCP (Model Context Protocol) server integration for enhanced capabilities. -""" - -import asyncio -import os -from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions -from common import ( - Config, - fetch_available_models, - setup_litellm_env, - print_header, - handle_model_list, - handle_model_switch, - stream_response, -) - - -async def interactive_chat_with_mcp(): - """ - Interactive CLI chat with the agent and MCP server - """ - config = Config() - - # Configure Anthropic SDK to point to LiteLLM gateway - litellm_base_url = setup_litellm_env(config) - - # Fetch available models from proxy - available_models = await fetch_available_models( - litellm_base_url, config.LITELLM_API_KEY - ) - - current_model = config.LITELLM_MODEL - - # MCP server configuration - mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2" - use_mcp = os.getenv("USE_MCP", "true").lower() == "true" - - if not use_mcp: - print("⚠️ MCP disabled via USE_MCP=false") - - print_header(litellm_base_url, current_model, has_mcp=use_mcp) - - while True: - # Configure agent options - if use_mcp: - try: - # Try with MCP server (HTTP transport) - # Using McpHttpServerConfig format from Agent SDK - options = ClaudeAgentOptions( - system_prompt="You are a helpful AI assistant with access to DeepWiki for research. Be concise, accurate, and friendly.", - model=current_model, - max_turns=50, - mcp_servers={ - "deepwiki2": { - "type": "http", - "url": mcp_server_url, - "headers": { - "Authorization": f"Bearer {config.LITELLM_API_KEY}" - }, - } - }, - ) - except Exception as e: - print(f"⚠️ Warning: Could not configure MCP server: {e}") - print("Continuing without MCP...\n") - use_mcp = False - options = ClaudeAgentOptions( - system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.", - model=current_model, - max_turns=50, - ) - else: - # Without MCP - options = ClaudeAgentOptions( - system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.", - model=current_model, - max_turns=50, - ) - - # Create agent client - try: - async with ClaudeSDKClient(options=options) as client: - conversation_active = True - - while conversation_active: - # Get user input - try: - user_input = input("\n👤 You: ").strip() - except (EOFError, KeyboardInterrupt): - print("\n\n👋 Goodbye!") - return - - # Handle commands - if user_input.lower() in ["quit", "exit"]: - print("\n👋 Goodbye!") - return - - if user_input.lower() == "clear": - print("\n🔄 Starting new conversation...\n") - conversation_active = False - continue - - if user_input.lower() == "models": - handle_model_list(available_models, current_model) - continue - - if user_input.lower() == "model": - new_model, should_restart = handle_model_switch( - available_models, current_model - ) - if should_restart: - current_model = new_model - conversation_active = False - continue - - if not user_input: - continue - - # Stream response from agent - await stream_response(client, user_input) - - except Exception as e: - print(f"\n❌ Error creating agent client: {e}") - print("This might be an MCP configuration issue. Try running without MCP:") - print(" USE_MCP=false python agent_with_mcp.py") - print("\nOr use the basic agent:") - print(" python main.py") - return - - -def main(): - """Run interactive chat with MCP""" - try: - asyncio.run(interactive_chat_with_mcp()) - except KeyboardInterrupt: - print("\n\n👋 Goodbye!") - - -if __name__ == "__main__": - main() diff --git a/cookbook/anthropic_agent_sdk/common.py b/cookbook/anthropic_agent_sdk/common.py deleted file mode 100644 index 505b5ecd91c..00000000000 --- a/cookbook/anthropic_agent_sdk/common.py +++ /dev/null @@ -1,164 +0,0 @@ -""" -Common utilities for Claude Agent SDK examples -""" - -import os -import httpx - - -class Config: - """Configuration for LiteLLM Gateway connection""" - - # LiteLLM proxy URL (default to local instance) - LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000") - - # LiteLLM API key (master key or virtual key) - LITELLM_API_KEY = os.environ["LITELLM_API_KEY"] - - # Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.) - LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5") - - -async def fetch_available_models(base_url: str, api_key: str) -> list[str]: - """ - Fetch available models from LiteLLM proxy /models endpoint - """ - try: - async with httpx.AsyncClient() as client: - response = await client.get( - f"{base_url}/models", - headers={"Authorization": f"Bearer {api_key}"}, - timeout=10.0, - ) - response.raise_for_status() - data = response.json() - return [model["id"] for model in data.get("data", [])] - except Exception as e: - print(f"⚠️ Warning: Could not fetch models from proxy: {e}") - print("Using default model list...") - # Fallback to default models - return [ - "bedrock-claude-sonnet-3.5", - "bedrock-claude-sonnet-4", - "bedrock-claude-sonnet-4.5", - "bedrock-claude-opus-4.5", - "bedrock-nova-premier", - ] - - -def setup_litellm_env(config: Config): - """ - Configure environment variables to point Agent SDK to LiteLLM - """ - litellm_base_url = config.LITELLM_PROXY_URL.rstrip("/") - os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url - os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY - return litellm_base_url - - -def print_header(base_url: str, current_model: str, has_mcp: bool = False): - """ - Print the chat header - """ - mcp_indicator = " + MCP" if has_mcp else "" - print("=" * 70) - print(f"🤖 Claude Agent SDK with LiteLLM Gateway{mcp_indicator} - Interactive Chat") - print("=" * 70) - print(f"🚀 Connected to: {base_url}") - print(f"📦 Current model: {current_model}") - if has_mcp: - print("🔌 MCP: deepwiki2 enabled") - print("\nType your messages below. Commands:") - print(" - 'quit' or 'exit' to end the conversation") - print(" - 'clear' to start a new conversation") - print(" - 'model' to switch models") - print(" - 'models' to list available models") - print("=" * 70) - print() - - -def handle_model_list(available_models: list[str], current_model: str): - """ - Display available models - """ - print("\n📋 Available models:") - for i, model in enumerate(available_models, 1): - marker = "✓" if model == current_model else " " - print(f" {marker} {i}. {model}") - - -def handle_model_switch( - available_models: list[str], current_model: str -) -> tuple[str, bool]: - """ - Handle model switching - - Returns: - tuple: (new_model, should_restart_conversation) - """ - print("\n📋 Select a model:") - for i, model in enumerate(available_models, 1): - marker = "✓" if model == current_model else " " - print(f" {marker} {i}. {model}") - - try: - choice = input("\nEnter number (or press Enter to cancel): ").strip() - if choice: - idx = int(choice) - 1 - if 0 <= idx < len(available_models): - new_model = available_models[idx] - print(f"\n✅ Switched to: {new_model}") - print("🔄 Starting new conversation with new model...\n") - return new_model, True - else: - print("❌ Invalid choice") - except (ValueError, IndexError): - print("❌ Invalid input") - - return current_model, False - - -async def stream_response(client, user_input: str): - """ - Stream response from the agent - """ - print("\n🤖 Assistant: ", end="", flush=True) - - try: - await client.query(user_input) - - # Show loading indicator - print("⏳ thinking...", end="", flush=True) - - # Stream the response - first_chunk = True - async for msg in client.receive_response(): - # Clear loading indicator on first message - if first_chunk: - print("\r🤖 Assistant: ", end="", flush=True) - first_chunk = False - - # Handle different message types - if hasattr(msg, "type"): - if msg.type == "content_block_delta": - # Streaming text delta - if hasattr(msg, "delta") and hasattr(msg.delta, "text"): - print(msg.delta.text, end="", flush=True) - elif msg.type == "content_block_start": - # Start of content block - if hasattr(msg, "content_block") and hasattr( - msg.content_block, "text" - ): - print(msg.content_block.text, end="", flush=True) - - # Fallback to original content handling - if hasattr(msg, "content"): - for content_block in msg.content: - if hasattr(content_block, "text"): - print(content_block.text, end="", flush=True) - - print() # New line after response - - except Exception as e: - print(f"\r\n❌ Error: {e}") - print("Please check your LiteLLM gateway is running and configured correctly.") diff --git a/cookbook/anthropic_agent_sdk/config.example.yaml b/cookbook/anthropic_agent_sdk/config.example.yaml deleted file mode 100644 index eb1984fc4ea..00000000000 --- a/cookbook/anthropic_agent_sdk/config.example.yaml +++ /dev/null @@ -1,25 +0,0 @@ -model_list: - - model_name: bedrock-claude-sonnet-3.5 - litellm_params: - model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" - aws_region_name: "us-east-1" - - - model_name: bedrock-claude-sonnet-4 - litellm_params: - model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0" - aws_region_name: "us-east-1" - - - model_name: bedrock-claude-sonnet-4.5 - litellm_params: - model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" - aws_region_name: "us-east-1" - - - model_name: bedrock-claude-opus-4.5 - litellm_params: - model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0" - aws_region_name: "us-east-1" - - - model_name: bedrock-nova-premier - litellm_params: - model: "bedrock/amazon.nova-premier-v1:0" - aws_region_name: "us-east-1" diff --git a/cookbook/anthropic_agent_sdk/main.py b/cookbook/anthropic_agent_sdk/main.py deleted file mode 100644 index 506c6fa07b0..00000000000 --- a/cookbook/anthropic_agent_sdk/main.py +++ /dev/null @@ -1,99 +0,0 @@ -""" -Simple Interactive Claude Agent SDK CLI using LiteLLM Gateway - -This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy. -LiteLLM acts as a unified interface, allowing you to use any LLM provider (OpenAI, Azure, Bedrock, etc.) -through the Claude Agent SDK by pointing it to the LiteLLM gateway. -""" - -import asyncio -from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions -from common import ( - Config, - fetch_available_models, - setup_litellm_env, - print_header, - handle_model_list, - handle_model_switch, - stream_response, -) - - -async def interactive_chat(): - """ - Interactive CLI chat with the agent - """ - config = Config() - - # Configure Anthropic SDK to point to LiteLLM gateway - litellm_base_url = setup_litellm_env(config) - - # Fetch available models from proxy - available_models = await fetch_available_models( - litellm_base_url, config.LITELLM_API_KEY - ) - - current_model = config.LITELLM_MODEL - - print_header(litellm_base_url, current_model) - - while True: - # Configure agent options for each conversation - options = ClaudeAgentOptions( - system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.", - model=current_model, - max_turns=50, - ) - - # Create agent client - async with ClaudeSDKClient(options=options) as client: - conversation_active = True - - while conversation_active: - # Get user input - try: - user_input = input("\n👤 You: ").strip() - except (EOFError, KeyboardInterrupt): - print("\n\n👋 Goodbye!") - return - - # Handle commands - if user_input.lower() in ["quit", "exit"]: - print("\n👋 Goodbye!") - return - - if user_input.lower() == "clear": - print("\n🔄 Starting new conversation...\n") - conversation_active = False - continue - - if user_input.lower() == "models": - handle_model_list(available_models, current_model) - continue - - if user_input.lower() == "model": - new_model, should_restart = handle_model_switch( - available_models, current_model - ) - if should_restart: - current_model = new_model - conversation_active = False - continue - - if not user_input: - continue - - # Stream response from agent - await stream_response(client, user_input) - - -def main(): - """Run interactive chat""" - try: - asyncio.run(interactive_chat()) - except KeyboardInterrupt: - print("\n\n👋 Goodbye!") - - -if __name__ == "__main__": - main() diff --git a/cookbook/anthropic_agent_sdk/requirements.txt b/cookbook/anthropic_agent_sdk/requirements.txt deleted file mode 100644 index 1e810bb7d99..00000000000 --- a/cookbook/anthropic_agent_sdk/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -claude-agent-sdk -httpx>=0.27.0 diff --git a/cookbook/benchmark/benchmark.py b/cookbook/benchmark/benchmark.py deleted file mode 100644 index b38d185a166..00000000000 --- a/cookbook/benchmark/benchmark.py +++ /dev/null @@ -1,90 +0,0 @@ -from litellm import completion, completion_cost -import time -import click -from tqdm import tqdm -from tabulate import tabulate -from termcolor import colored -import os - - -# Define the list of models to benchmark -# select any LLM listed here: https://docs.litellm.ai/docs/providers -models = ["gpt-3.5-turbo", "claude-2"] - -# Enter LLM API keys -# https://docs.litellm.ai/docs/providers -os.environ["OPENAI_API_KEY"] = "" -os.environ["ANTHROPIC_API_KEY"] = "" - -# List of questions to benchmark (replace with your questions) -questions = ["When will BerriAI IPO?", "When will LiteLLM hit $100M ARR?"] - -# Enter your system prompt here -system_prompt = """ -You are LiteLLMs helpful assistant -""" - - -@click.command() -@click.option( - "--system-prompt", - default="You are a helpful assistant that can answer questions.", - help="System prompt for the conversation.", -) -def main(system_prompt): - for question in questions: - data = [] # Data for the current question - - with tqdm(total=len(models)) as pbar: - for model in models: - colored_description = colored( - f"Running question: {question} for model: {model}", "green" - ) - pbar.set_description(colored_description) - start_time = time.time() - - response = completion( - model=model, - max_tokens=500, - messages=[ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": question}, - ], - ) - - end = time.time() - total_time = end - start_time - cost = completion_cost(completion_response=response) - raw_response = response["choices"][0]["message"]["content"] - - data.append( - { - "Model": colored(model, "light_blue"), - "Response": raw_response, # Colorize the response - "ResponseTime": colored(f"{total_time:.2f} seconds", "red"), - "Cost": colored(f"${cost:.6f}", "green"), # Colorize the cost - } - ) - - pbar.update(1) - - # Separate headers from the data - headers = ["Model", "Response", "Response Time (seconds)", "Cost ($)"] - colwidths = [15, 80, 15, 10] - - # Create a nicely formatted table for the current question - table = tabulate( - [list(d.values()) for d in data], - headers, - tablefmt="grid", - maxcolwidths=colwidths, - ) - - # Print the table for the current question - colored_question = colored(question, "green") - click.echo(f"\nBenchmark Results for '{colored_question}':") - click.echo(table) # Display the formatted table - - -if __name__ == "__main__": - main() diff --git a/cookbook/benchmark/eval_suites_mlflow_autoevals/auto_evals.py b/cookbook/benchmark/eval_suites_mlflow_autoevals/auto_evals.py deleted file mode 100644 index daa38dda585..00000000000 --- a/cookbook/benchmark/eval_suites_mlflow_autoevals/auto_evals.py +++ /dev/null @@ -1,30 +0,0 @@ -from dotenv import load_dotenv - -load_dotenv() - -import litellm - -from autoevals.llm import * - -################### - -# litellm completion call -question = "which country has the highest population" -response = litellm.completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": question}], -) -print(response) -# use the auto eval Factuality() evaluator - -print("calling evaluator") -evaluator = Factuality() -result = evaluator( - output=response.choices[0]["message"][ - "content" - ], # response from litellm.completion() - expected="India", # expected output - input=question, # question passed to litellm.completion -) - -print(result) diff --git a/cookbook/benchmark/readme.md b/cookbook/benchmark/readme.md deleted file mode 100644 index afa59aa91ee..00000000000 --- a/cookbook/benchmark/readme.md +++ /dev/null @@ -1,181 +0,0 @@ -

- LLM-Bench -

-

-

Benchmark LLMs response, cost and response time

-

LLM vs Cost per input + output token ($)

- Screenshot 2023-11-13 at 2 51 06 PM -

- - Bar Graph Excel Sheet here - - -| Model | Provider | Cost per input + output token ($)| -| --- | --- | --- | -| openrouter/mistralai/mistral-7b-instruct | openrouter | 0.0 | -| ollama/llama2 | ollama | 0.0 | -| ollama/llama2:13b | ollama | 0.0 | -| ollama/llama2:70b | ollama | 0.0 | -| ollama/llama2-uncensored | ollama | 0.0 | -| ollama/mistral | ollama | 0.0 | -| ollama/codellama | ollama | 0.0 | -| ollama/orca-mini | ollama | 0.0 | -| ollama/vicuna | ollama | 0.0 | -| perplexity/codellama-34b-instruct | perplexity | 0.0 | -| perplexity/llama-2-13b-chat | perplexity | 0.0 | -| perplexity/llama-2-70b-chat | perplexity | 0.0 | -| perplexity/mistral-7b-instruct | perplexity | 0.0 | -| perplexity/replit-code-v1.5-3b | perplexity | 0.0 | -| text-bison | vertex_ai-text-models | 0.00000025 | -| text-bison@001 | vertex_ai-text-models | 0.00000025 | -| chat-bison | vertex_ai-chat-models | 0.00000025 | -| chat-bison@001 | vertex_ai-chat-models | 0.00000025 | -| chat-bison-32k | vertex_ai-chat-models | 0.00000025 | -| code-bison | vertex_ai-code-text-models | 0.00000025 | -| code-bison@001 | vertex_ai-code-text-models | 0.00000025 | -| code-gecko@001 | vertex_ai-chat-models | 0.00000025 | -| code-gecko@latest | vertex_ai-chat-models | 0.00000025 | -| codechat-bison | vertex_ai-code-chat-models | 0.00000025 | -| codechat-bison@001 | vertex_ai-code-chat-models | 0.00000025 | -| codechat-bison-32k | vertex_ai-code-chat-models | 0.00000025 | -| palm/chat-bison | palm | 0.00000025 | -| palm/chat-bison-001 | palm | 0.00000025 | -| palm/text-bison | palm | 0.00000025 | -| palm/text-bison-001 | palm | 0.00000025 | -| palm/text-bison-safety-off | palm | 0.00000025 | -| palm/text-bison-safety-recitation-off | palm | 0.00000025 | -| anyscale/meta-llama/Llama-2-7b-chat-hf | anyscale | 0.0000003 | -| anyscale/mistralai/Mistral-7B-Instruct-v0.1 | anyscale | 0.0000003 | -| openrouter/meta-llama/llama-2-13b-chat | openrouter | 0.0000004 | -| openrouter/nousresearch/nous-hermes-llama2-13b | openrouter | 0.0000004 | -| deepinfra/meta-llama/Llama-2-7b-chat-hf | deepinfra | 0.0000004 | -| deepinfra/mistralai/Mistral-7B-Instruct-v0.1 | deepinfra | 0.0000004 | -| anyscale/meta-llama/Llama-2-13b-chat-hf | anyscale | 0.0000005 | -| amazon.titan-text-lite-v1 | bedrock | 0.0000007 | -| deepinfra/meta-llama/Llama-2-13b-chat-hf | deepinfra | 0.0000007 | -| text-babbage-001 | text-completion-openai | 0.0000008 | -| text-ada-001 | text-completion-openai | 0.0000008 | -| babbage-002 | text-completion-openai | 0.0000008 | -| openrouter/google/palm-2-chat-bison | openrouter | 0.000001 | -| openrouter/google/palm-2-codechat-bison | openrouter | 0.000001 | -| openrouter/meta-llama/codellama-34b-instruct | openrouter | 0.000001 | -| deepinfra/codellama/CodeLlama-34b-Instruct-hf | deepinfra | 0.0000012 | -| deepinfra/meta-llama/Llama-2-70b-chat-hf | deepinfra | 0.0000016499999999999999 | -| deepinfra/jondurbin/airoboros-l2-70b-gpt4-1.4.1 | deepinfra | 0.0000016499999999999999 | -| anyscale/meta-llama/Llama-2-70b-chat-hf | anyscale | 0.000002 | -| anyscale/codellama/CodeLlama-34b-Instruct-hf | anyscale | 0.000002 | -| gpt-3.5-turbo-1106 | openai | 0.000003 | -| openrouter/meta-llama/llama-2-70b-chat | openrouter | 0.000003 | -| amazon.titan-text-express-v1 | bedrock | 0.000003 | -| gpt-3.5-turbo | openai | 0.0000035 | -| gpt-3.5-turbo-0301 | openai | 0.0000035 | -| gpt-3.5-turbo-0613 | openai | 0.0000035 | -| gpt-3.5-turbo-instruct | text-completion-openai | 0.0000035 | -| openrouter/openai/gpt-3.5-turbo | openrouter | 0.0000035 | -| cohere.command-text-v14 | bedrock | 0.0000035 | -| gpt-3.5-turbo-0613 | openai | 0.0000035 | -| claude-instant-1 | anthropic | 0.00000714 | -| claude-instant-1.2 | anthropic | 0.00000714 | -| openrouter/anthropic/claude-instant-v1 | openrouter | 0.00000714 | -| anthropic.claude-instant-v1 | bedrock | 0.00000714 | -| openrouter/mancer/weaver | openrouter | 0.00001125 | -| j2-mid | ai21 | 0.00002 | -| ai21.j2-mid-v1 | bedrock | 0.000025 | -| openrouter/jondurbin/airoboros-l2-70b-2.1 | openrouter | 0.00002775 | -| command-nightly | cohere | 0.00003 | -| command | cohere | 0.00003 | -| command-light | cohere | 0.00003 | -| command-medium-beta | cohere | 0.00003 | -| command-xlarge-beta | cohere | 0.00003 | -| command-r-plus| cohere | 0.000018 | -| j2-ultra | ai21 | 0.00003 | -| ai21.j2-ultra-v1 | bedrock | 0.0000376 | -| gpt-4-1106-preview | openai | 0.00004 | -| gpt-4-vision-preview | openai | 0.00004 | -| claude-2 | anthropic | 0.0000437 | -| openrouter/anthropic/claude-2 | openrouter | 0.0000437 | -| anthropic.claude-v1 | bedrock | 0.0000437 | -| anthropic.claude-v2 | bedrock | 0.0000437 | -| gpt-4 | openai | 0.00009 | -| gpt-4-0314 | openai | 0.00009 | -| gpt-4-0613 | openai | 0.00009 | -| openrouter/openai/gpt-4 | openrouter | 0.00009 | -| gpt-4-32k | openai | 0.00018 | -| gpt-4-32k-0314 | openai | 0.00018 | -| gpt-4-32k-0613 | openai | 0.00018 | - - - -## Setup: -``` -git clone https://github.com/BerriAI/litellm -``` -cd to `benchmark` dir -``` -cd litellm/cookbook/benchmark -``` - -### Install Dependencies -``` -pip install litellm click tqdm tabulate termcolor -``` - -### Configuration -In `benchmark/benchmark.py` select your LLMs, LLM API Key and questions - -Supported LLMs: https://docs.litellm.ai/docs/providers - -```python -# Define the list of models to benchmark -models = ['gpt-3.5-turbo', 'togethercomputer/llama-2-70b-chat', 'claude-2'] - -# Enter LLM API keys -os.environ['OPENAI_API_KEY'] = "" -os.environ['ANTHROPIC_API_KEY'] = "" -os.environ['TOGETHERAI_API_KEY'] = "" - -# List of questions to benchmark (replace with your questions) -questions = [ - "When will BerriAI IPO?", - "When will LiteLLM hit $100M ARR?" -] - -``` - -## Run LLM-Bench -``` -python3 benchmark.py -``` - -## Expected Output -``` -Running question: When will BerriAI IPO? for model: claude-2: 100%|████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:13<00:00, 4.41s/it] - -Benchmark Results for 'When will BerriAI IPO?': -+-----------------+----------------------------------------------------------------------------------+---------------------------+------------+ -| Model | Response | Response Time (seconds) | Cost ($) | -+=================+==================================================================================+===========================+============+ -| gpt-3.5-turbo | As an AI language model, I cannot provide up-to-date information or predict | 1.55 seconds | $0.000122 | -| | future events. It is best to consult a reliable financial source or contact | | | -| | BerriAI directly for information regarding their IPO plans. | | | -+-----------------+----------------------------------------------------------------------------------+---------------------------+------------+ -| togethercompute | I'm not able to provide information about future IPO plans or dates for BerriAI | 8.52 seconds | $0.000531 | -| r/llama-2-70b-c | or any other company. IPO (Initial Public Offering) plans and timelines are | | | -| hat | typically kept private by companies until they are ready to make a public | | | -| | announcement. It's important to note that IPO plans can change and are subject | | | -| | to various factors, such as market conditions, financial performance, and | | | -| | regulatory approvals. Therefore, it's difficult to predict with certainty when | | | -| | BerriAI or any other company will go public. If you're interested in staying | | | -| | up-to-date with BerriAI's latest news and developments, you may want to follow | | | -| | their official social media accounts, subscribe to their newsletter, or visit | | | -| | their website periodically for updates. | | | -+-----------------+----------------------------------------------------------------------------------+---------------------------+------------+ -| claude-2 | I do not have any information about when or if BerriAI will have an initial | 3.17 seconds | $0.002084 | -| | public offering (IPO). As an AI assistant created by Anthropic to be helpful, | | | -| | harmless, and honest, I do not have insider knowledge about Anthropic's business | | | -| | plans or strategies. | | | -+-----------------+----------------------------------------------------------------------------------+---------------------------+------------+ -``` - -## Support -**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://enterprise.litellm.ai/demo) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you. diff --git a/cookbook/codellama-server/README.MD b/cookbook/codellama-server/README.MD deleted file mode 100644 index 82a7e62f40a..00000000000 --- a/cookbook/codellama-server/README.MD +++ /dev/null @@ -1,153 +0,0 @@ -# CodeLlama Server: Streaming, Caching, Model Fallbacks (OpenAI + Anthropic), Prompt-tracking - -Works with: Anthropic, Huggingface, Cohere, TogetherAI, Azure, OpenAI, etc. - -[![PyPI Version](https://img.shields.io/pypi/v/litellm.svg)](https://pypi.org/project/litellm/) -[![PyPI Version](https://img.shields.io/badge/stable%20version-v0.1.345-blue?color=green&link=https://pypi.org/project/litellm/0.1.1/)](https://pypi.org/project/litellm/0.1.1/) -![Downloads](https://img.shields.io/pypi/dm/litellm) - -[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/HuDPw-?referralCode=jch2ME) - -**LIVE DEMO** - https://litellm.ai/playground - -## What does CodeLlama Server do - -- Uses Together AI's CodeLlama to answer coding questions, with GPT-4 + Claude-2 as backups (you can easily switch this to any model from Huggingface, Replicate, Cohere, AI21, Azure, OpenAI, etc.) -- Sets default system prompt for guardrails `system_prompt = "Only respond to questions about code. Say 'I don't know' to anything outside of that."` -- Integrates with Promptlayer for model + prompt tracking -- Example output - -Code Output - -- **Consistent Input/Output** Format - - Call all models using the OpenAI format - `completion(model, messages)` - - Text responses will always be available at `['choices'][0]['message']['content']` - - Stream responses will always be available at `['choices'][0]['delta']['content']` -- **Error Handling** Using Model Fallbacks (if `CodeLlama` fails, try `GPT-4`) with cooldowns, and retries -- **Prompt Logging** - Log successful completions to promptlayer for testing + iterating on your prompts in production! (Learn more: https://litellm.readthedocs.io/en/latest/advanced/ - - **Example: Logs sent to PromptLayer** - - Prompt Logging - - -- **Token Usage & Spend** - Track Input + Completion tokens used + Spend/model - https://docs.litellm.ai/docs/token_usage -- **Caching** - Provides in-memory cache + GPT-Cache integration for more advanced usage - https://docs.litellm.ai/docs/caching/gpt_cache - -- **Streaming & Async Support** - Return generators to stream text responses - TEST IT 👉 https://litellm.ai/ - -## API Endpoints - -### `/chat/completions` (POST) - -This endpoint is used to generate chat completions for 50+ support LLM API Models. Use llama2, GPT-4, Claude2 etc - -#### Input - -This API endpoint accepts all inputs in raw JSON and expects the following inputs - -- `prompt` (string, required): The user's coding related question -- Additional Optional parameters: `temperature`, `functions`, `function_call`, `top_p`, `n`, `stream`. See the full list of supported inputs here: https://litellm.readthedocs.io/en/latest/input/ - -#### Example JSON body - -For claude-2 - -```json -{ - "prompt": "write me a function to print hello world" -} -``` - -### Making an API request to the Code-Gen Server - -```python -import requests -import json - -url = "localhost:4000/chat/completions" - -payload = json.dumps({ - "prompt": "write me a function to print hello world" -}) -headers = { - 'Content-Type': 'application/json' -} - -response = requests.request("POST", url, headers=headers, data=payload) - -print(response.text) - -``` - -### Output [Response Format] - -Responses from the server are given in the following format. -All responses from the server are returned in the following format (for all LLM models). More info on output here: https://litellm.readthedocs.io/en/latest/output/ - -```json -{ - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": ".\n\n```\ndef print_hello_world():\n print(\"hello world\")\n", - "role": "assistant" - } - } - ], - "created": 1693279694.6474009, - "model": "togethercomputer/CodeLlama-34b-Instruct", - "usage": { - "completion_tokens": 14, - "prompt_tokens": 28, - "total_tokens": 42 - } -} -``` - -## Installation & Usage - -### Running Locally - -1. Clone liteLLM repository to your local machine: - ``` - git clone https://github.com/BerriAI/litellm-CodeLlama-server - ``` -2. Install the required dependencies using pip - ``` - pip install requirements.txt - ``` -3. Set your LLM API keys - ``` - os.environ['OPENAI_API_KEY]` = "YOUR_API_KEY" - or - set OPENAI_API_KEY in your .env file - ``` -4. Run the server: - ``` - python main.py - ``` - -## Deploying - -1. Quick Start: Deploy on Railway - - [![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/HuDPw-?referralCode=jch2ME) - -2. `GCP`, `AWS`, `Azure` - This project includes a `Dockerfile` allowing you to build and deploy a Docker Project on your providers - -# Support / Talk with founders - -- [Our calendar 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) -- [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai - -## Roadmap - -- [ ] Implement user-based rate-limiting -- [ ] Spending controls per project - expose key creation endpoint -- [ ] Need to store a keys db -> mapping created keys to their alias (i.e. project name) -- [ ] Easily add new models as backups / as the entry-point (add this to the available model list) diff --git a/cookbook/codellama-server/imgs/code-output.png b/cookbook/codellama-server/imgs/code-output.png deleted file mode 100644 index 67e298bd3be..00000000000 Binary files a/cookbook/codellama-server/imgs/code-output.png and /dev/null differ diff --git a/cookbook/codellama-server/imgs/promptlayer_logging.png b/cookbook/codellama-server/imgs/promptlayer_logging.png deleted file mode 100644 index 26b046ac448..00000000000 Binary files a/cookbook/codellama-server/imgs/promptlayer_logging.png and /dev/null differ diff --git a/cookbook/codellama-server/main.py b/cookbook/codellama-server/main.py deleted file mode 100644 index d05d6752300..00000000000 --- a/cookbook/codellama-server/main.py +++ /dev/null @@ -1,102 +0,0 @@ -import traceback -from flask import Flask, request, Response -from flask_cors import CORS -import litellm -from util import handle_error -from litellm import completion -import os -import dotenv -import time -import json - -dotenv.load_dotenv() - -# TODO: set your keys in .env or here: -# os.environ["OPENAI_API_KEY"] = "" # set your openai key here -# os.environ["ANTHROPIC_API_KEY"] = "" # set your anthropic key here -# os.environ["TOGETHER_AI_API_KEY"] = "" # set your together ai key here -# see supported models / keys here: https://litellm.readthedocs.io/en/latest/supported/ -######### ENVIRONMENT VARIABLES ########## -verbose = True - -# litellm.caching_with_models = True # CACHING: caching_with_models Keys in the cache are messages + model. - to learn more: https://docs.litellm.ai/docs/caching/ -######### PROMPT LOGGING ########## -os.environ["PROMPTLAYER_API_KEY"] = ( - "" # set your promptlayer key here - https://promptlayer.com/ -) - -# set callbacks -litellm.success_callback = ["promptlayer"] -############ HELPER FUNCTIONS ################################### - - -def print_verbose(print_statement): - if verbose: - print(print_statement) - - -app = Flask(__name__) -CORS(app) - - -@app.route("/") -def index(): - return "received!", 200 - - -def data_generator(response): - for chunk in response: - yield f"data: {json.dumps(chunk)}\n\n" - - -@app.route("/chat/completions", methods=["POST"]) -def api_completion(): - data = request.json - start_time = time.time() - if data.get("stream") == "True": - data["stream"] = True # convert to boolean - try: - if "prompt" not in data: - raise ValueError("data needs to have prompt") - data["model"] = ( - "togethercomputer/CodeLlama-34b-Instruct" # by default use Together AI's CodeLlama model - https://api.together.xyz/playground/chat?model=togethercomputer%2FCodeLlama-34b-Instruct - ) - # COMPLETION CALL - system_prompt = "Only respond to questions about code. Say 'I don't know' to anything outside of that." - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": data.pop("prompt")}, - ] - data["messages"] = messages - print(f"data: {data}") - response = completion(**data) - ## LOG SUCCESS - end_time = time.time() - if ( - "stream" in data and data["stream"] == True - ): # use generate_responses to stream responses - return Response(data_generator(response), mimetype="text/event-stream") - except Exception: - # call handle_error function - print_verbose(f"Got Error api_completion(): {traceback.format_exc()}") - ## LOG FAILURE - end_time = time.time() - traceback_exception = traceback.format_exc() - return handle_error(data=data) - return response - - -@app.route("/get_models", methods=["POST"]) -def get_models(): - try: - return litellm.model_list - except Exception as e: - traceback.print_exc() - response = {"error": str(e)} - return response, 200 - - -if __name__ == "__main__": - from waitress import serve - - serve(app, host="0.0.0.0", port=4000, threads=500) diff --git a/cookbook/community-resources/get_hf_models.py b/cookbook/community-resources/get_hf_models.py deleted file mode 100644 index 8c75a241227..00000000000 --- a/cookbook/community-resources/get_hf_models.py +++ /dev/null @@ -1,89 +0,0 @@ -import requests - - -def get_next_url(response): - """ - Function to get 'next' url from Link header - :param response: response from requests - :return: next url or None - """ - if "link" not in response.headers: - return None - headers = response.headers - - next_url = headers["Link"] - print(next_url) - start_index = next_url.find("<") - end_index = next_url.find(">") - - return next_url[1:end_index] - - -def get_models(url): - """ - Function to retrieve all models from paginated endpoint - :param url: base url to make GET request - :return: list of all models - """ - models = [] - while url: - response = requests.get(url) - if response.status_code != 200: - print(f"Failed to retrieve data. Status code: {response.status_code}") - return models - payload = response.json() - url = get_next_url(response) - models.extend(payload) - return models - - -def get_cleaned_models(models): - """ - Function to clean retrieved models - :param models: list of retrieved models - :return: list of cleaned models - """ - cleaned_models = [] - for model in models: - cleaned_models.append(model["id"]) - return cleaned_models - - -# Get text-generation models -url = "https://huggingface.co/api/models?filter=text-generation-inference" -text_generation_models = get_models(url) -cleaned_text_generation_models = get_cleaned_models(text_generation_models) - -print(cleaned_text_generation_models) - - -# Get conversational models -url = "https://huggingface.co/api/models?filter=conversational" -conversational_models = get_models(url) -cleaned_conversational_models = get_cleaned_models(conversational_models) - -print(cleaned_conversational_models) - - -def write_to_txt(cleaned_models, filename): - """ - Function to write the contents of a list to a text file - :param cleaned_models: list of cleaned models - :param filename: name of the text file - """ - with open(filename, "w") as f: - for item in cleaned_models: - f.write("%s\n" % item) - - -# Write contents of cleaned_text_generation_models to text_generation_models.txt -write_to_txt( - cleaned_text_generation_models, - "huggingface_llms_metadata/hf_text_generation_models.txt", -) - -# Write contents of cleaned_conversational_models to conversational_models.txt -write_to_txt( - cleaned_conversational_models, - "huggingface_llms_metadata/hf_conversational_models.txt", -) diff --git a/cookbook/community-resources/max_tokens.json b/cookbook/community-resources/max_tokens.json deleted file mode 100644 index 289f8faf9e3..00000000000 --- a/cookbook/community-resources/max_tokens.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "gpt-3.5-turbo": { - "max_tokens": 4000, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002 - }, - "gpt-3.5-turbo-0613": { - "max_tokens": 4000, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002 - }, - "gpt-3.5-turbo-0301": { - "max_tokens": 4000, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002 - }, - "gpt-3.5-turbo-16k": { - "max_tokens": 16000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004 - }, - "gpt-3.5-turbo-16k-0613": { - "max_tokens": 16000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004 - }, - "gpt-4": { - "max_tokens": 8000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.00006 - }, - "gpt-4-0613": { - "max_tokens": 8000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.00006 - }, - "gpt-4-32k": { - "max_tokens": 8000, - "input_cost_per_token": 0.00006, - "output_cost_per_token": 0.00012 - }, - "claude-instant-1": { - "max_tokens": 100000, - "input_cost_per_token": 0.00000163, - "output_cost_per_token": 0.00000551 - }, - "claude-2": { - "max_tokens": 100000, - "input_cost_per_token": 0.00001102, - "output_cost_per_token": 0.00003268 - }, - "text-bison-001": { - "max_tokens": 8192, - "input_cost_per_token": 0.000004, - "output_cost_per_token": 0.000004 - }, - "chat-bison-001": { - "max_tokens": 4096, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000002 - }, - "command-nightly": { - "max_tokens": 4096, - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000015 - }, - "replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1": { - "max_tokens": 4096, - "input_cost_per_token": 0.00000608, - "output_cost_per_token": 0.00000608 - }, - "together-ai-up-to-3b": { - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.0000001 - }, - "together-ai-3.1b-7b": { - "input_cost_per_token": 0.0000002, - "output_cost_per_token": 0.0000002 - }, - "together-ai-7.1b-20b": { - "max_tokens": 1000, - "input_cost_per_token": 0.0000004, - "output_cost_per_token": 0.0000004 - }, - "together-ai-20.1b-40b": { - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000001 - }, - "together-ai-40.1b-70b": { - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000003 - } -} diff --git a/cookbook/gollem_go_agent_framework/README.md b/cookbook/gollem_go_agent_framework/README.md deleted file mode 100644 index 729f985d086..00000000000 --- a/cookbook/gollem_go_agent_framework/README.md +++ /dev/null @@ -1,119 +0,0 @@ -# Gollem Go Agent Framework with LiteLLM - -A working example showing how to use [gollem](https://github.com/fugue-labs/gollem), a production-grade Go agent framework, with LiteLLM as a proxy gateway. This lets Go developers access 100+ LLM providers through a single proxy while keeping compile-time type safety for tools and structured output. - -## Quick Start - -### 1. Start LiteLLM Proxy - -```bash -# Simple start with a single model -litellm --model gpt-4o - -# Or with the example config for multi-provider access -litellm --config proxy_config.yaml -``` - -### 2. Run the examples - -```bash -# Install Go dependencies -go mod tidy - -# Basic agent -go run ./basic - -# Agent with type-safe tools -go run ./tools - -# Streaming responses -go run ./streaming -``` - -## Configuration - -The included `proxy_config.yaml` sets up three providers through LiteLLM: - -```yaml -model_list: - - model_name: gpt-4o # OpenAI - - model_name: claude-sonnet # Anthropic - - model_name: gemini-pro # Google Vertex AI -``` - -Switch providers in Go by changing a single string — no code changes needed: - -```go -model := openai.NewLiteLLM("http://localhost:4000", - openai.WithModel("gpt-4o"), // OpenAI - // openai.WithModel("claude-sonnet"), // Anthropic - // openai.WithModel("gemini-pro"), // Google -) -``` - -## Examples - -### `basic/` — Basic Agent - -Connects gollem to LiteLLM and runs a simple prompt. Demonstrates the `NewLiteLLM` constructor and basic agent creation. - -### `tools/` — Type-Safe Tools - -Shows gollem's compile-time type-safe tool framework working through LiteLLM's tool-use passthrough. The tool parameters are Go structs with JSON tags — the schema is generated automatically at compile time. - -### `streaming/` — Streaming Responses - -Real-time token streaming using Go 1.23+ range-over-function iterators, proxied through LiteLLM's SSE passthrough. - -## How It Works - -Gollem's `openai.NewLiteLLM()` constructor creates an OpenAI-compatible provider pointed at your LiteLLM proxy. Since LiteLLM speaks the OpenAI API protocol, everything works out of the box: - -- **Chat completions** — standard request/response -- **Tool use** — LiteLLM passes tool definitions and calls through transparently -- **Streaming** — Server-Sent Events proxied through LiteLLM -- **Structured output** — JSON schema response format works with supporting models - -``` -Go App (gollem) → LiteLLM Proxy → OpenAI / Anthropic / Google / ... -``` - -## Why Use This? - -- **Type-safe Go**: Compile-time type checking for tools, structured output, and agent configuration — no runtime surprises -- **Single proxy, many models**: Switch between OpenAI, Anthropic, Google, and 100+ other providers by changing a model name string -- **Zero-dependency core**: gollem's core has no external dependencies — just stdlib -- **Single binary deployment**: `go build` produces one binary, no pip/venv/Docker needed -- **Cost tracking & rate limiting**: LiteLLM handles cost tracking, rate limits, and fallbacks at the proxy layer - -## Environment Variables - -```bash -# Required for providers you want to use (set in LiteLLM config or env) -export OPENAI_API_KEY="sk-..." -export ANTHROPIC_API_KEY="sk-ant-..." - -# Optional: point to a non-default LiteLLM proxy -export LITELLM_PROXY_URL="http://localhost:4000" -``` - -## Troubleshooting - -**Connection errors?** -- Make sure LiteLLM is running: `litellm --model gpt-4o` -- Check the URL is correct (default: `http://localhost:4000`) - -**Model not found?** -- Verify the model name matches what's configured in LiteLLM -- Run `curl http://localhost:4000/models` to see available models - -**Tool calls not working?** -- Ensure the underlying model supports tool use (GPT-4o, Claude, Gemini) -- Check LiteLLM logs for any provider-specific errors - -## Learn More - -- [gollem GitHub](https://github.com/fugue-labs/gollem) -- [gollem API Reference](https://pkg.go.dev/github.com/fugue-labs/gollem/core) -- [LiteLLM Proxy Docs](https://docs.litellm.ai/docs/simple_proxy) -- [LiteLLM Supported Models](https://docs.litellm.ai/docs/providers) diff --git a/cookbook/gollem_go_agent_framework/basic/main.go b/cookbook/gollem_go_agent_framework/basic/main.go deleted file mode 100644 index 838149a8ff9..00000000000 --- a/cookbook/gollem_go_agent_framework/basic/main.go +++ /dev/null @@ -1,41 +0,0 @@ -// Basic gollem agent connected to a LiteLLM proxy. -// -// Usage: -// -// litellm --model gpt-4o # start proxy in another terminal -// go run ./basic -package main - -import ( - "context" - "fmt" - "log" - "os" - - "github.com/fugue-labs/gollem/core" - "github.com/fugue-labs/gollem/provider/openai" -) - -func main() { - proxyURL := "http://localhost:4000" - if u := os.Getenv("LITELLM_PROXY_URL"); u != "" { - proxyURL = u - } - - // Connect to LiteLLM proxy. NewLiteLLM creates an OpenAI-compatible - // provider pointed at the given URL. - model := openai.NewLiteLLM(proxyURL, - openai.WithModel("gpt-4o"), // any model name configured in LiteLLM - ) - - // Create and run a simple agent. - agent := core.NewAgent[string](model, - core.WithSystemPrompt[string]("You are a helpful assistant. Be concise."), - ) - - result, err := agent.Run(context.Background(), "Explain quantum computing in two sentences.") - if err != nil { - log.Fatal(err) - } - fmt.Println(result.Output) -} diff --git a/cookbook/gollem_go_agent_framework/go.mod b/cookbook/gollem_go_agent_framework/go.mod deleted file mode 100644 index a8dc9365d7f..00000000000 --- a/cookbook/gollem_go_agent_framework/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module github.com/BerriAI/litellm/cookbook/gollem_go_agent_framework - -go 1.26.3 - -require github.com/fugue-labs/gollem v0.1.0 diff --git a/cookbook/gollem_go_agent_framework/go.sum b/cookbook/gollem_go_agent_framework/go.sum deleted file mode 100644 index 1eb6c5ac9fc..00000000000 --- a/cookbook/gollem_go_agent_framework/go.sum +++ /dev/null @@ -1,2 +0,0 @@ -github.com/fugue-labs/gollem v0.1.0 h1:QexYnvkb44QZFEljgAePqMIGZjgsbk0Y5GJ2jYYgfa8= -github.com/fugue-labs/gollem v0.1.0/go.mod h1:htW1YO81uysSKVOkYJtxhGCFrzm+36HBFxEWuECoHKQ= diff --git a/cookbook/gollem_go_agent_framework/proxy_config.yaml b/cookbook/gollem_go_agent_framework/proxy_config.yaml deleted file mode 100644 index 18265a002bc..00000000000 --- a/cookbook/gollem_go_agent_framework/proxy_config.yaml +++ /dev/null @@ -1,16 +0,0 @@ -model_list: - - model_name: gpt-4o - litellm_params: - model: openai/gpt-4o - api_key: os.environ/OPENAI_API_KEY - - - model_name: claude-sonnet - litellm_params: - model: anthropic/claude-sonnet-4-20250514 - api_key: os.environ/ANTHROPIC_API_KEY - - - model_name: gemini-pro - litellm_params: - model: vertex_ai/gemini-2.0-flash - vertex_project: my-project - vertex_location: us-central1 diff --git a/cookbook/gollem_go_agent_framework/streaming/main.go b/cookbook/gollem_go_agent_framework/streaming/main.go deleted file mode 100644 index 42bc9bbe34a..00000000000 --- a/cookbook/gollem_go_agent_framework/streaming/main.go +++ /dev/null @@ -1,56 +0,0 @@ -// Streaming responses from gollem through LiteLLM. -// -// Uses Go 1.23+ range-over-function iterators for real-time token -// streaming via LiteLLM's SSE passthrough. -// -// Usage: -// -// litellm --model gpt-4o -// go run ./streaming -package main - -import ( - "context" - "fmt" - "log" - "os" - - "github.com/fugue-labs/gollem/core" - "github.com/fugue-labs/gollem/provider/openai" -) - -func main() { - proxyURL := "http://localhost:4000" - if u := os.Getenv("LITELLM_PROXY_URL"); u != "" { - proxyURL = u - } - - model := openai.NewLiteLLM(proxyURL, - openai.WithModel("gpt-4o"), - ) - - agent := core.NewAgent[string](model) - - // RunStream returns a streaming result that yields tokens as they arrive. - stream, err := agent.RunStream(context.Background(), "Write a haiku about distributed systems") - if err != nil { - log.Fatal(err) - } - - // StreamText yields text chunks in real-time. - // The boolean argument controls whether deltas (true) or accumulated - // text (false) is returned. - fmt.Print("Response: ") - for text, err := range stream.StreamText(true) { - if err != nil { - log.Fatal(err) - } - fmt.Print(text) - } - fmt.Println() - - // After streaming completes, the final response is available. - resp := stream.Response() - fmt.Printf("\nTokens used: input=%d, output=%d\n", - resp.Usage.InputTokens, resp.Usage.OutputTokens) -} diff --git a/cookbook/gollem_go_agent_framework/tools/main.go b/cookbook/gollem_go_agent_framework/tools/main.go deleted file mode 100644 index ed41a95ffef..00000000000 --- a/cookbook/gollem_go_agent_framework/tools/main.go +++ /dev/null @@ -1,64 +0,0 @@ -// Gollem agent with type-safe tools through LiteLLM. -// -// The tool parameters are Go structs — gollem generates the JSON schema -// automatically at compile time. LiteLLM passes tool definitions through -// transparently to the underlying provider. -// -// Usage: -// -// litellm --model gpt-4o -// go run ./tools -package main - -import ( - "context" - "fmt" - "log" - "os" - - "github.com/fugue-labs/gollem/core" - "github.com/fugue-labs/gollem/provider/openai" -) - -// WeatherParams defines the tool's input schema via struct tags. -// The JSON schema is generated at compile time — no runtime reflection needed. -type WeatherParams struct { - City string `json:"city" description:"City name to get weather for"` - Unit string `json:"unit,omitempty" description:"Temperature unit: celsius or fahrenheit"` -} - -func main() { - proxyURL := "http://localhost:4000" - if u := os.Getenv("LITELLM_PROXY_URL"); u != "" { - proxyURL = u - } - - model := openai.NewLiteLLM(proxyURL, - openai.WithModel("gpt-4o"), - ) - - // Define a type-safe tool. The function signature enforces correct types. - weatherTool := core.FuncTool[WeatherParams]( - "get_weather", - "Get current weather for a city", - func(ctx context.Context, p WeatherParams) (string, error) { - unit := p.Unit - if unit == "" { - unit = "fahrenheit" - } - // In production, call a real weather API here. - return fmt.Sprintf("Weather in %s: 72°F (22°C), sunny", p.City), nil - }, - ) - - agent := core.NewAgent[string](model, - core.WithTools[string](weatherTool), - core.WithSystemPrompt[string]("You are a helpful weather assistant. Use the get_weather tool to answer weather questions."), - ) - - result, err := agent.Run(context.Background(), "What's the weather like in San Francisco and Tokyo?") - if err != nil { - log.Fatal(err) - } - fmt.Println(result.Output) -} diff --git a/cookbook/google_adk_litellm_tutorial.ipynb b/cookbook/google_adk_litellm_tutorial.ipynb deleted file mode 100644 index 27914edbba8..00000000000 --- a/cookbook/google_adk_litellm_tutorial.ipynb +++ /dev/null @@ -1,412 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "7aa8875d", - "metadata": {}, - "source": [ - "# Google ADK with LiteLLM\n", - "\n", - "Use Google ADK with LiteLLM Python SDK, LiteLLM Proxy.\n", - "\n", - "This tutorial shows you how to create intelligent agents using Agent Development Kit (ADK) with support for multiple Large Language Model (LLM) providers through LiteLLM." - ] - }, - { - "cell_type": "markdown", - "id": "a4d249c3", - "metadata": {}, - "source": [ - "## Overview\n", - "\n", - "ADK (Agent Development Kit) allows you to build intelligent agents powered by LLMs. By integrating with LiteLLM, you can:\n", - "\n", - "- Use multiple LLM providers (OpenAI, Anthropic, Google, etc.)\n", - "- Switch easily between models from different providers\n", - "- Connect to a LiteLLM proxy for centralized model management" - ] - }, - { - "cell_type": "markdown", - "id": "a0bbb56b", - "metadata": {}, - "source": [ - "## Prerequisites\n", - "\n", - "- Python environment setup\n", - "- API keys for model providers (OpenAI, Anthropic, Google AI Studio)\n", - "- Basic understanding of LLMs and agent concepts" - ] - }, - { - "cell_type": "markdown", - "id": "7fee50a8", - "metadata": {}, - "source": [ - "## Installation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "44106a23", - "metadata": {}, - "outputs": [], - "source": [ - "# Install dependencies\n", - "!pip install google-adk litellm" - ] - }, - { - "cell_type": "markdown", - "id": "2171740a", - "metadata": {}, - "source": [ - "## 1. Setting Up Environment" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6695807e", - "metadata": {}, - "outputs": [], - "source": [ - "# Setup environment and API keys\n", - "import os\n", - "import asyncio\n", - "from google.adk.agents import Agent\n", - "from google.adk.models.lite_llm import LiteLlm # For multi-model support\n", - "from google.adk.sessions import InMemorySessionService\n", - "from google.adk.runners import Runner\n", - "from google.genai import types\n", - "import litellm # Import for proxy configuration\n", - "\n", - "# Set your API keys\n", - "os.environ['GOOGLE_API_KEY'] = 'your-google-api-key' # For Gemini models\n", - "os.environ['OPENAI_API_KEY'] = 'your-openai-api-key' # For OpenAI models\n", - "os.environ['ANTHROPIC_API_KEY'] = 'your-anthropic-api-key' # For Claude models\n", - "\n", - "# Define model constants for cleaner code\n", - "MODEL_GEMINI_PRO = 'gemini-1.5-pro'\n", - "MODEL_GPT_4O = 'openai/gpt-4o'\n", - "MODEL_CLAUDE_SONNET = 'anthropic/claude-3-sonnet-20240229'" - ] - }, - { - "cell_type": "markdown", - "id": "d2b1ed59", - "metadata": {}, - "source": [ - "## 2. Define a Simple Tool" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "04b3ef5b", - "metadata": {}, - "outputs": [], - "source": [ - "# Weather tool implementation\n", - "def get_weather(city: str) -> dict:\n", - " \"\"\"Retrieves the current weather report for a specified city.\"\"\"\n", - " print(f'Tool: get_weather called for city: {city}')\n", - "\n", - " # Mock weather data\n", - " mock_weather_db = {\n", - " 'newyork': {\n", - " 'status': 'success',\n", - " 'report': 'The weather in New York is sunny with a temperature of 25°C.'\n", - " },\n", - " 'london': {\n", - " 'status': 'success',\n", - " 'report': \"It's cloudy in London with a temperature of 15°C.\"\n", - " },\n", - " 'tokyo': {\n", - " 'status': 'success',\n", - " 'report': 'Tokyo is experiencing light rain and a temperature of 18°C.'\n", - " },\n", - " }\n", - "\n", - " city_normalized = city.lower().replace(' ', '')\n", - "\n", - " if city_normalized in mock_weather_db:\n", - " return mock_weather_db[city_normalized]\n", - " else:\n", - " return {\n", - " 'status': 'error',\n", - " 'error_message': f\"Sorry, I don't have weather information for '{city}'.\"\n", - " }" - ] - }, - { - "cell_type": "markdown", - "id": "727b15c9", - "metadata": {}, - "source": [ - "## 3. Helper Function for Agent Interaction" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f77449bf", - "metadata": {}, - "outputs": [], - "source": [ - "# Agent interaction helper function\n", - "async def call_agent_async(query: str, runner, user_id, session_id):\n", - " \"\"\"Sends a query to the agent and prints the final response.\"\"\"\n", - " print(f'\\n>>> User Query: {query}')\n", - "\n", - " content = types.Content(role='user', parts=[types.Part(text=query)])\n", - " final_response_text = 'Agent did not produce a final response.'\n", - "\n", - " async for event in runner.run_async(\n", - " user_id=user_id,\n", - " session_id=session_id,\n", - " new_message=content\n", - " ):\n", - " if event.is_final_response():\n", - " if event.content and event.content.parts:\n", - " final_response_text = event.content.parts[0].text\n", - " break\n", - " print(f'<<< Agent Response: {final_response_text}')" - ] - }, - { - "cell_type": "markdown", - "id": "0ac87987", - "metadata": {}, - "source": [ - "## 4. Using Different Model Providers with ADK\n", - "\n", - "### 4.1 Using OpenAI Models" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e167d557", - "metadata": {}, - "outputs": [], - "source": [ - "# OpenAI model implementation\n", - "weather_agent_gpt = Agent(\n", - " name='weather_agent_gpt',\n", - " model=LiteLlm(model=MODEL_GPT_4O),\n", - " description='Provides weather information using OpenAI\\'s GPT.',\n", - " instruction=(\n", - " 'You are a helpful weather assistant powered by GPT-4o. '\n", - " \"Use the 'get_weather' tool for city weather requests. \"\n", - " 'Present information clearly.'\n", - " ),\n", - " tools=[get_weather],\n", - ")\n", - "\n", - "session_service_gpt = InMemorySessionService()\n", - "session_gpt = session_service_gpt.create_session(\n", - " app_name='weather_app', user_id='user_1', session_id='session_gpt'\n", - ")\n", - "\n", - "runner_gpt = Runner(\n", - " agent=weather_agent_gpt,\n", - " app_name='weather_app',\n", - " session_service=session_service_gpt,\n", - ")\n", - "\n", - "async def test_gpt_agent():\n", - " print('\\n--- Testing GPT Agent ---')\n", - " await call_agent_async(\n", - " \"What's the weather in London?\",\n", - " runner=runner_gpt,\n", - " user_id='user_1',\n", - " session_id='session_gpt',\n", - " )\n", - "\n", - "# To execute in a notebook cell:\n", - "# await test_gpt_agent()" - ] - }, - { - "cell_type": "markdown", - "id": "f9cb0613", - "metadata": {}, - "source": [ - "### 4.2 Using Anthropic Models" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1c653665", - "metadata": {}, - "outputs": [], - "source": [ - "# Anthropic model implementation\n", - "weather_agent_claude = Agent(\n", - " name='weather_agent_claude',\n", - " model=LiteLlm(model=MODEL_CLAUDE_SONNET),\n", - " description='Provides weather information using Anthropic\\'s Claude.',\n", - " instruction=(\n", - " 'You are a helpful weather assistant powered by Claude Sonnet. '\n", - " \"Use the 'get_weather' tool for city weather requests. \"\n", - " 'Present information clearly.'\n", - " ),\n", - " tools=[get_weather],\n", - ")\n", - "\n", - "session_service_claude = InMemorySessionService()\n", - "session_claude = session_service_claude.create_session(\n", - " app_name='weather_app', user_id='user_1', session_id='session_claude'\n", - ")\n", - "\n", - "runner_claude = Runner(\n", - " agent=weather_agent_claude,\n", - " app_name='weather_app',\n", - " session_service=session_service_claude,\n", - ")\n", - "\n", - "async def test_claude_agent():\n", - " print('\\n--- Testing Claude Agent ---')\n", - " await call_agent_async(\n", - " \"What's the weather in Tokyo?\",\n", - " runner=runner_claude,\n", - " user_id='user_1',\n", - " session_id='session_claude',\n", - " )\n", - "\n", - "# To execute in a notebook cell:\n", - "# await test_claude_agent()" - ] - }, - { - "cell_type": "markdown", - "id": "bf9d863b", - "metadata": {}, - "source": [ - "### 4.3 Using Google's Gemini Models" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "83f49d0a", - "metadata": {}, - "outputs": [], - "source": [ - "# Gemini model implementation\n", - "weather_agent_gemini = Agent(\n", - " name='weather_agent_gemini',\n", - " model=MODEL_GEMINI_PRO,\n", - " description='Provides weather information using Google\\'s Gemini.',\n", - " instruction=(\n", - " 'You are a helpful weather assistant powered by Gemini Pro. '\n", - " \"Use the 'get_weather' tool for city weather requests. \"\n", - " 'Present information clearly.'\n", - " ),\n", - " tools=[get_weather],\n", - ")\n", - "\n", - "session_service_gemini = InMemorySessionService()\n", - "session_gemini = session_service_gemini.create_session(\n", - " app_name='weather_app', user_id='user_1', session_id='session_gemini'\n", - ")\n", - "\n", - "runner_gemini = Runner(\n", - " agent=weather_agent_gemini,\n", - " app_name='weather_app',\n", - " session_service=session_service_gemini,\n", - ")\n", - "\n", - "async def test_gemini_agent():\n", - " print('\\n--- Testing Gemini Agent ---')\n", - " await call_agent_async(\n", - " \"What's the weather in New York?\",\n", - " runner=runner_gemini,\n", - " user_id='user_1',\n", - " session_id='session_gemini',\n", - " )\n", - "\n", - "# To execute in a notebook cell:\n", - "# await test_gemini_agent()" - ] - }, - { - "cell_type": "markdown", - "id": "93bc5fd0", - "metadata": {}, - "source": [ - "## 5. Using LiteLLM Proxy with ADK" - ] - }, - { - "cell_type": "markdown", - "id": "b4275151", - "metadata": {}, - "source": [ - "| Variable | Description |\n", - "|----------|-------------|\n", - "| `LITELLM_PROXY_API_KEY` | The API key for the LiteLLM proxy |\n", - "| `LITELLM_PROXY_API_BASE` | The base URL for the LiteLLM proxy |\n", - "| `USE_LITELLM_PROXY` or `litellm.use_litellm_proxy` | When set to True, your request will be sent to LiteLLM proxy. |" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "256530a6", - "metadata": {}, - "outputs": [], - "source": [ - "# LiteLLM proxy integration\n", - "os.environ['LITELLM_PROXY_API_KEY'] = 'your-litellm-proxy-api-key'\n", - "os.environ['LITELLM_PROXY_API_BASE'] = 'your-litellm-proxy-url' # e.g., 'http://localhost:4000'\n", - "litellm.use_litellm_proxy = True\n", - "\n", - "weather_agent_proxy_env = Agent(\n", - " name='weather_agent_proxy_env',\n", - " model=LiteLlm(model='gpt-4o'),\n", - " description='Provides weather information using a model from LiteLLM proxy.',\n", - " instruction=(\n", - " 'You are a helpful weather assistant. '\n", - " \"Use the 'get_weather' tool for city weather requests. \"\n", - " 'Present information clearly.'\n", - " ),\n", - " tools=[get_weather],\n", - ")\n", - "\n", - "session_service_proxy_env = InMemorySessionService()\n", - "session_proxy_env = session_service_proxy_env.create_session(\n", - " app_name='weather_app', user_id='user_1', session_id='session_proxy_env'\n", - ")\n", - "\n", - "runner_proxy_env = Runner(\n", - " agent=weather_agent_proxy_env,\n", - " app_name='weather_app',\n", - " session_service=session_service_proxy_env,\n", - ")\n", - "\n", - "async def test_proxy_env_agent():\n", - " print('\\n--- Testing Proxy-enabled Agent (Environment Variables) ---')\n", - " await call_agent_async(\n", - " \"What's the weather in London?\",\n", - " runner=runner_proxy_env,\n", - " user_id='user_1',\n", - " session_id='session_proxy_env',\n", - " )\n", - "\n", - "# To execute in a notebook cell:\n", - "# await test_proxy_env_agent()" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/cookbook/gpt_realtime_translate.py b/cookbook/gpt_realtime_translate.py deleted file mode 100644 index 827c65cdbfe..00000000000 --- a/cookbook/gpt_realtime_translate.py +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import asyncio -import base64 -import json -import os -import sys -import wave -from collections.abc import Iterator, Sequence -from dataclasses import dataclass -from pathlib import Path -from urllib.parse import urlencode, urlsplit, urlunsplit - -import websockets -from websockets.asyncio.client import ClientConnection - -SAMPLE_RATE = 24_000 -CHANNELS = 1 -SAMPLE_WIDTH = 2 -CHUNK_DURATION_SECONDS = 0.1 -CHUNK_BYTES = int(SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH * CHUNK_DURATION_SECONDS) -RESPONSE_TIMEOUT_SECONDS = 30.0 -AUDIO_EVENT_TYPES = frozenset( - { - "session.output_audio.delta", - "response.audio.delta", - "response.output_audio.delta", - } -) -TRANSCRIPT_EVENT_TYPES = frozenset( - { - "session.output_transcript.delta", - "response.text.delta", - "response.output_audio_transcript.delta", - } -) - - -@dataclass(frozen=True, slots=True) -class Settings: - input_wav: Path - output_wav: Path - base_url: str - model: str - target_language: str - trailing_silence_seconds: float - api_key: str - - -def write_stdout(message: str = "", *, end: str = "\n", flush: bool = False) -> None: - sys.stdout.write(f"{message}{end}") - if flush: - sys.stdout.flush() - - -def write_stderr(message: str) -> None: - sys.stderr.write(f"{message}\n") - - -def parse_args(argv: Sequence[str] | None = None) -> Settings | str: - parser = argparse.ArgumentParser( - description="Stream a 24 kHz PCM16 WAV through gpt-realtime-translate and save the translated audio", - ) - parser.add_argument("input_wav", type=Path) - parser.add_argument("--output", type=Path, default=Path("translated.wav")) - parser.add_argument("--base-url", default=os.getenv("LITELLM_BASE_URL", "http://localhost:4000")) - parser.add_argument("--model", default=os.getenv("REALTIME_TRANSLATE_MODEL", "gpt-realtime-translate")) - parser.add_argument("--target-language", default="fr") - parser.add_argument("--trailing-silence", type=float, default=1.5) - parsed = parser.parse_args(argv) - api_key = os.getenv("LITELLM_API_KEY") or os.getenv("OPENAI_API_KEY") - if not api_key: - return "Set LITELLM_API_KEY or OPENAI_API_KEY before running the script" - if parsed.trailing_silence < 0: - return "--trailing-silence must be zero or greater" - return Settings( - input_wav=parsed.input_wav, - output_wav=parsed.output, - base_url=parsed.base_url, - model=parsed.model, - target_language=parsed.target_language, - trailing_silence_seconds=parsed.trailing_silence, - api_key=api_key, - ) - - -def translation_url(base_url: str, model: str) -> str | None: - parsed = urlsplit(base_url.rstrip("/")) - scheme = {"http": "ws", "https": "wss", "ws": "ws", "wss": "wss"}.get(parsed.scheme) - if not scheme or not parsed.netloc: - return None - base_path = parsed.path.rstrip("/") - realtime_path = ( - f"{base_path}/realtime/translations" if base_path.endswith("/v1") else f"{base_path}/v1/realtime/translations" - ) - return urlunsplit((scheme, parsed.netloc, realtime_path, urlencode({"model": model}), "")) - - -def read_pcm16_wav(path: Path) -> bytes | str: - try: - with wave.open(str(path), "rb") as source: - actual_format = ( - source.getnchannels(), - source.getsampwidth(), - source.getframerate(), - source.getcomptype(), - ) - expected_format = (CHANNELS, SAMPLE_WIDTH, SAMPLE_RATE, "NONE") - if actual_format != expected_format: - return ( - f"{path} must be mono, 16-bit PCM, 24 kHz WAV; received " - f"channels={actual_format[0]}, sample_width={actual_format[1]}, " - f"sample_rate={actual_format[2]}, compression={actual_format[3]}" - ) - return source.readframes(source.getnframes()) - except (OSError, EOFError, wave.Error) as exc: - return f"Unable to read {path}: {exc}" - - -def audio_chunks(audio: bytes) -> Iterator[bytes]: - return (audio[offset : offset + CHUNK_BYTES] for offset in range(0, len(audio), CHUNK_BYTES)) - - -def audio_message(audio: bytes) -> str: - return json.dumps( - { - "type": "session.input_audio_buffer.append", - "audio": base64.b64encode(audio).decode("ascii"), - } - ) - - -async def configure_session(connection: ClientConnection, target_language: str) -> str | None: - await connection.send( - json.dumps( - { - "type": "session.update", - "session": {"audio": {"output": {"language": target_language}}}, - } - ) - ) - while True: - raw_event = await asyncio.wait_for(connection.recv(), timeout=20) - event = json.loads(raw_event) - event_type = event.get("type") - if event_type == "session.created": - write_stdout(f"Session: {event.get('session', {}).get('id', 'created')}") - if event_type == "session.updated": - return None - if event_type == "error": - return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}" - - -async def send_audio( - connection: ClientConnection, pcm: bytes, trailing_silence_seconds: float, finished: asyncio.Event -) -> None: - silence = bytes(round(trailing_silence_seconds * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)) - try: - for chunk in audio_chunks(pcm + silence): - await connection.send(audio_message(chunk)) - await asyncio.sleep(len(chunk) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)) - await connection.send(json.dumps({"type": "session.close"})) - finally: - finished.set() - - -async def receive_translation( - connection: ClientConnection, output_path: Path, sender_finished: asyncio.Event -) -> str | None: - audio_received = asyncio.Event() - try: - with wave.open(str(output_path), "wb") as output: - output.setnchannels(CHANNELS) - output.setsampwidth(SAMPLE_WIDTH) - output.setframerate(SAMPLE_RATE) - write_stdout("Translation: ", end="", flush=True) - while True: - try: - raw_event = await asyncio.wait_for(connection.recv(), timeout=RESPONSE_TIMEOUT_SECONDS) - except TimeoutError: - if not sender_finished.is_set(): - continue - return ( - f"The translation stream went quiet for {RESPONSE_TIMEOUT_SECONDS:g} s without closing the session" - ) - event = json.loads(raw_event) - event_type = event.get("type") - if event_type in AUDIO_EVENT_TYPES: - output.writeframes(base64.b64decode(event.get("delta", ""), validate=True)) - audio_received.set() - elif event_type in TRANSCRIPT_EVENT_TYPES: - write_stdout(event.get("delta", event.get("text", "")), end="", flush=True) - elif event_type == "error": - return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}" - elif event_type == "session.closed": - write_stdout() - return None if audio_received.is_set() else "The translation stream ended without translated audio" - except (OSError, wave.Error) as exc: - return f"Unable to write {output_path}: {exc}" - - -async def translate(settings: Settings, pcm: bytes) -> str | None: - url = translation_url(settings.base_url, settings.model) - if not url: - return f"Invalid --base-url: {settings.base_url}" - sender_finished = asyncio.Event() - try: - async with websockets.connect( - url, - additional_headers={"Authorization": f"Bearer {settings.api_key}"}, - proxy=None, - open_timeout=20, - close_timeout=5, - ) as connection: - configuration_error = await configure_session(connection, settings.target_language) - if configuration_error: - return configuration_error - async with asyncio.TaskGroup() as tasks: - receiver = tasks.create_task(receive_translation(connection, settings.output_wav, sender_finished)) - tasks.create_task(send_audio(connection, pcm, settings.trailing_silence_seconds, sender_finished)) - return receiver.result() - except Exception as exc: - return f"Translation failed: {type(exc).__name__}: {exc}" - - -def main(argv: Sequence[str] | None = None) -> int: - settings = parse_args(argv) - if isinstance(settings, str): - write_stderr(settings) - return 2 - pcm = read_pcm16_wav(settings.input_wav) - if isinstance(pcm, str): - write_stderr(pcm) - return 2 - error = asyncio.run(translate(settings, pcm)) - if error: - write_stderr(error) - return 1 - write_stdout(f"Translated audio: {settings.output_wav}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/cookbook/liteLLM_A121_Jurrasic_example.ipynb b/cookbook/liteLLM_A121_Jurrasic_example.ipynb deleted file mode 100644 index f975b97e911..00000000000 --- a/cookbook/liteLLM_A121_Jurrasic_example.ipynb +++ /dev/null @@ -1,251 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "# LiteLLM A121 Tutorial\n", - "\n", - "This walks through using A121 Jurassic models\n", - "* j2-light\n", - "* j2-mid\n", - "* j2-ultra" - ], - "metadata": { - "id": "LeFYo8iqcn5g" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "GslPQFmaZsp-" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "source": [ - "from litellm import completion\n", - "import os" - ], - "metadata": { - "id": "P3cKiqURZx7P" - }, - "execution_count": 2, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Set A121 Keys\n", - "You can get a free key from https://studio.ai21.com/account/api-key" - ], - "metadata": { - "id": "tmTvA1_GaNU4" - } - }, - { - "cell_type": "code", - "source": [ - "os.environ[\"AI21_API_KEY\"] = \"\"" - ], - "metadata": { - "id": "_xX8LmxAZ2vp" - }, - "execution_count": 5, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# A121 Supported Models:\n", - "https://studio.ai21.com/foundation-models" - ], - "metadata": { - "id": "Fx5ZfJTLbF0A" - } - }, - { - "cell_type": "markdown", - "source": [ - "## J2-light Call" - ], - "metadata": { - "id": "H0tl-0Z3bDaL" - } - }, - { - "cell_type": "code", - "source": [ - "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - "response = completion(model=\"j2-light\", messages=messages)\n", - "response" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "DZnApsJUZ_I2", - "outputId": "b5707cbe-f67c-47f7-bac5-a7b8af1ba815" - }, - "execution_count": 6, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - " JSON: {\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" However, I have an important question to ask you\\nMy name is X, and I was wondering if you would be willing to help me.\",\n", - " \"role\": \"assistant\"\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1692761063.5189915,\n", - " \"model\": \"j2-light\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": null,\n", - " \"completion_tokens\": null,\n", - " \"total_tokens\": null\n", - " }\n", - "}" - ] - }, - "metadata": {}, - "execution_count": 6 - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "# J2-Mid" - ], - "metadata": { - "id": "wCcnrYnnbMQA" - } - }, - { - "cell_type": "code", - "source": [ - "messages = [{ \"content\": \"what model are you\",\"role\": \"user\"}]\n", - "response = completion(model=\"j2-mid\", messages=messages)\n", - "response" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "-5Sxf4blaeEl", - "outputId": "6264a5e8-16d6-44a3-e167-9e0c59b6dbc4" - }, - "execution_count": 7, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - " JSON: {\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"\\nplease choose the model from the list below\\nModel view in Tekla Structures\",\n", - " \"role\": \"assistant\"\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1692761140.0017524,\n", - " \"model\": \"j2-mid\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": null,\n", - " \"completion_tokens\": null,\n", - " \"total_tokens\": null\n", - " }\n", - "}" - ] - }, - "metadata": {}, - "execution_count": 7 - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "# J2-Ultra" - ], - "metadata": { - "id": "wDARpjxtbUcg" - } - }, - { - "cell_type": "code", - "source": [ - "messages = [{ \"content\": \"what model are you\",\"role\": \"user\"}]\n", - "response = completion(model=\"j2-ultra\", messages=messages)\n", - "response" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "i228xwsYbSYo", - "outputId": "3765ac56-5a9b-442e-b357-2e346d02e1df" - }, - "execution_count": 8, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - " JSON: {\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \"\\nI am not a specific model, but I can provide information and assistance based on my training data. Please let me know if there is anything you\",\n", - " \"role\": \"assistant\"\n", - " }\n", - " }\n", - " ],\n", - " \"created\": 1692761157.8675153,\n", - " \"model\": \"j2-ultra\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": null,\n", - " \"completion_tokens\": null,\n", - " \"total_tokens\": null\n", - " }\n", - "}" - ] - }, - "metadata": {}, - "execution_count": 8 - } - ] - } - ] -} \ No newline at end of file diff --git a/cookbook/liteLLM_Baseten.ipynb b/cookbook/liteLLM_Baseten.ipynb deleted file mode 100644 index 0a5bc5f1df7..00000000000 --- a/cookbook/liteLLM_Baseten.ipynb +++ /dev/null @@ -1,146 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "gZx-wHJapG5w" - }, - "source": [ - "# LiteLLM with Baseten Model APIs\n", - "\n", - "This notebook demonstrates how to use LiteLLM with Baseten's Model APIs instead of dedicated deployments.\n", - "\n", - "## Example Usage\n", - "```python\n", - "response = completion(\n", - " model=\"baseten/openai/gpt-oss-120b\",\n", - " messages=[{\"role\": \"user\", \"content\": \"Hello!\"}],\n", - " max_tokens=1000,\n", - " temperature=0.7\n", - ")\n", - "```\n", - "\n", - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "4JSRa0QVogPo" - }, - "outputs": [], - "source": [ - "%pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "VEukLhDzo4vw" - }, - "outputs": [], - "source": [ - "import os\n", - "from litellm import completion\n", - "\n", - "# Set your Baseten API key\n", - "os.environ['BASETEN_API_KEY'] = \"\" #@param {type:\"string\"}\n", - "\n", - "# Test message\n", - "messages = [{\"role\": \"user\", \"content\": \"What is AGI?\"}]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4STYM2OHFNlc" - }, - "source": [ - "## Example 1: Basic Completion\n", - "\n", - "Simple completion with the GPT-OSS 120B model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "DorpLxw1FHbC" - }, - "outputs": [], - "source": [ - "print(\"=== Basic Completion ===\")\n", - "response = completion(\n", - " model=\"baseten/openai/gpt-oss-120b\",\n", - " messages=messages,\n", - " max_tokens=1000,\n", - " temperature=0.7,\n", - " top_p=0.9,\n", - " presence_penalty=0.1,\n", - " frequency_penalty=0.1,\n", - ")\n", - "print(f\"Response: {response.choices[0].message.content}\")\n", - "print(f\"Usage: {response.usage}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "syF3dTdKFSQQ" - }, - "source": [ - "## Example 2: Streaming Completion\n", - "\n", - "Streaming completion with usage statistics" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "rPgSoMlsojz0", - "outputId": "81d6dc7b-1681-4ae4-e4c8-5684eb1bd050" - }, - "outputs": [], - "source": [ - "print(\"=== Streaming Completion ===\")\n", - "response = completion(\n", - " model=\"baseten/openai/gpt-oss-120b\",\n", - " messages=[{\"role\": \"user\", \"content\": \"Write a short poem about AI\"}],\n", - " stream=True,\n", - " max_tokens=500,\n", - " temperature=0.8,\n", - " stream_options={\n", - " \"include_usage\": True,\n", - " \"continuous_usage_stats\": True\n", - " },\n", - ")\n", - "\n", - "print(\"Streaming response:\")\n", - "for chunk in response:\n", - " if chunk.choices and chunk.choices[0].delta.content:\n", - " print(chunk.choices[0].delta.content, end=\"\", flush=True)\n", - "print(\"\\n\")" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/liteLLM_Getting_Started.ipynb b/cookbook/liteLLM_Getting_Started.ipynb deleted file mode 100644 index b43c51dceeb..00000000000 --- a/cookbook/liteLLM_Getting_Started.ipynb +++ /dev/null @@ -1,411 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "MZ01up0p7wOJ" - }, - "source": [ - "## 🚅 liteLLM Quick Start Demo\n", - "### TLDR: Call 50+ LLM APIs using chatGPT Input/Output format\n", - "https://github.com/BerriAI/litellm\n", - "\n", - "liteLLM is package to simplify calling **OpenAI, Azure, Llama2, Cohere, Anthropic, Huggingface API Endpoints**. LiteLLM manages\n", - "\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "RZtzCnQS7rW-" - }, - "source": [ - "## Installation and setting Params" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "rsrN5W-N7L8d" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "ArrWyG5b7QAG" - }, - "outputs": [], - "source": [ - "from litellm import completion\n", - "import os" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "bbhJRt34_NJ1" - }, - "source": [ - "## Set your API keys\n", - "- liteLLM reads your .env, env variables or key manager for Auth\n", - "\n", - "Set keys for the models you want to use below" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "id": "-h8Ga5cR7SvV" - }, - "outputs": [], - "source": [ - "# Only set keys for the LLMs you want to use\n", - "os.environ['OPENAI_API_KEY'] = \"\" #@param\n", - "os.environ[\"ANTHROPIC_API_KEY\"] = \"\" #@param\n", - "os.environ[\"REPLICATE_API_KEY\"] = \"\" #@param\n", - "os.environ[\"COHERE_API_KEY\"] = \"\" #@param\n", - "os.environ[\"AZURE_API_BASE\"] = \"\" #@param\n", - "os.environ[\"AZURE_API_VERSION\"] = \"\" #@param\n", - "os.environ[\"AZURE_API_KEY\"] = \"\" #@param" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "fhqpKv6L8fBj" - }, - "source": [ - "## Call chatGPT" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "speIkoX_8db4", - "outputId": "331a6c65-f121-4e65-e121-bf8aaad05d9d" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"id\": \"chatcmpl-820kPkRwSLml4X6165fWbZlEDOedr\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1695490221,\n", - " \"model\": \"gpt-3.5-turbo-0613\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"I'm sorry, but as an AI text-based model, I don't have real-time information. However, you can check the current weather in San Francisco by searching for \\\"weather in SF\\\" on any search engine or checking a weather website or app.\"\n", - " },\n", - " \"finish_reason\": \"stop\"\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"prompt_tokens\": 13,\n", - " \"completion_tokens\": 51,\n", - " \"total_tokens\": 64\n", - " },\n", - " \"response_ms\": 2385.592\n", - "}" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "completion(model=\"gpt-3.5-turbo\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "Q3jV1Uxv8zNo" - }, - "source": [ - "## Call Claude-2" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "V8yTWYzY8m9S", - "outputId": "8b6dd32d-f9bf-4e89-886d-47cb8020f025" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop_sequence\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" Unfortunately I don't have enough context to know the exact location you are asking about when you say \\\"SF\\\". SF could refer to San Francisco, California, or potentially other cities that go by SF as an abbreviation. To get an accurate weather report, it would be helpful if you could provide the full city name and state/country. If you are looking for the weather in San Francisco, California, I would be happy to provide that forecast. Please let me know the specific location you want the weather for.\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-6d1a40c0-19c0-4bd7-9ca2-a91d8b8c2295\",\n", - " \"created\": 1695490260.983768,\n", - " \"response_ms\": 6351.544,\n", - " \"model\": \"claude-2\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 14,\n", - " \"completion_tokens\": 102,\n", - " \"total_tokens\": 116\n", - " }\n", - "}" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "completion(model=\"claude-2\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "yu0LPDmW9PJa" - }, - "source": [ - "## Call llama2 on replicate" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "0GWV5mtO9Jbu", - "outputId": "38538825-b271-406d-a437-f5cf0eb7e548" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" I'm happy to help! However, I must point out that the question \\\"what's the weather in SF\\\" doesn't make sense as \\\"SF\\\" could refer to multiple locations. Could you please clarify which location you are referring to? San Francisco, California or Sioux Falls, South Dakota? Once I have more context, I would be happy to provide you with accurate and reliable information.\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-3151c2eb-b26f-4c96-89b5-ed1746b219e0\",\n", - " \"created\": 1695490237.714101,\n", - " \"response_ms\": 12109.565,\n", - " \"model\": \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 6,\n", - " \"completion_tokens\": 78,\n", - " \"total_tokens\": 84\n", - " },\n", - " \"ended\": 1695490249.821266\n", - "}" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model = \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\"\n", - "completion(model=model, messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "HXdj5SEe9iLK" - }, - "source": [ - "## Call Command-Nightly" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "EaUq2xIx9fhr", - "outputId": "55fe6f52-b58b-4729-948a-74dac4b431b2" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" As an AI model I don't have access to real-time data, so I can't tell\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-dc0d8ead-071d-486c-a111-78975b38794b\",\n", - " \"created\": 1695490235.936903,\n", - " \"response_ms\": 1022.6759999999999,\n", - " \"model\": \"command-nightly\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 6,\n", - " \"completion_tokens\": 19,\n", - " \"total_tokens\": 25\n", - " }\n", - "}" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "completion(model=\"command-nightly\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "1g9hSgsL9soJ" - }, - "source": [ - "## Call Azure OpenAI" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For azure openai calls ensure to add the `azure/` prefix to `model`. If your deployment-id is `chatgpt-test` set `model` = `azure/chatgpt-test`" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "AvLjR-PF-lt0", - "outputId": "deff2db3-b003-48cd-ea62-c03a68a4464a" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"id\": \"chatcmpl-820kZyCwbNvZATiLkNmXmpxxzvTKO\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1695490231,\n", - " \"model\": \"gpt-35-turbo\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"finish_reason\": \"stop\",\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Sorry, as an AI language model, I don't have real-time information. Please check your preferred weather website or app for the latest weather updates of San Francisco.\"\n", - " }\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"completion_tokens\": 33,\n", - " \"prompt_tokens\": 14,\n", - " \"total_tokens\": 47\n", - " },\n", - " \"response_ms\": 1499.529\n", - "}" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "completion(model=\"azure/chatgpt-v-2\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/liteLLM_IBM_Watsonx.ipynb b/cookbook/liteLLM_IBM_Watsonx.ipynb deleted file mode 100644 index 6de108b5d39..00000000000 --- a/cookbook/liteLLM_IBM_Watsonx.ipynb +++ /dev/null @@ -1,300 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# LiteLLM x IBM [watsonx.ai](https://www.ibm.com/products/watsonx-ai)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pre-Requisites" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Set watsonx.ai Credentials\n", - "\n", - "See [this documentation](https://cloud.ibm.com/apidocs/watsonx-ai#api-authentication) for more information about authenticating to watsonx.ai" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import litellm\n", - "from litellm.llms.watsonx import IBMWatsonXAI\n", - "litellm.set_verbose = False\n", - "\n", - "os.environ[\"WATSONX_URL\"] = \"\" # Your watsonx.ai base URL\n", - "os.environ[\"WATSONX_APIKEY\"] = \"\" # Your IBM cloud API key or watsonx.ai token\n", - "os.environ[\"WATSONX_PROJECT_ID\"] = \"\" # ID of your watsonx.ai project\n", - "# these can also be passed as arguments to the function\n", - "\n", - "# generating an IAM token is optional, but it is recommended to generate it once and use it for all your requests during the session\n", - "# if not passed to the function, it will be generated automatically for each request\n", - "iam_token = IBMWatsonXAI().generate_iam_token(api_key=os.environ[\"WATSONX_APIKEY\"]) \n", - "# you can also set os.environ[\"WATSONX_TOKEN\"] = iam_token" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Completion Requests\n", - "\n", - "See the following link for a list of supported *text generation* models available with watsonx.ai:\n", - "\n", - "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx&locale=en&audience=wdp" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Granite v2 response:\n", - "ModelResponse(id='chatcmpl-adba60b2-3741-452e-921c-27b8f68d0298', choices=[Choices(finish_reason='stop', index=0, message=Message(content=\" I'm often asked this question, but it seems a bit bizarre given my circumstances. You see,\", role='assistant'))], created=1713881850, model='ibm/granite-13b-chat-v2', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=8, completion_tokens=20, total_tokens=28), finish_reason='max_tokens')\n", - "LLaMa 3 8b response:\n", - "ModelResponse(id='chatcmpl-eb282abc-373c-4082-9dae-172546d16d5c', choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"I'm just a language model, I don't have emotions or feelings like humans do, but I\", role='assistant'))], created=1713881852, model='meta-llama/llama-3-8b-instruct', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=16, completion_tokens=20, total_tokens=36), finish_reason='max_tokens')\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "# see litellm.llms.watsonx.IBMWatsonXAIConfig for a list of available parameters to pass to the completion functions\n", - "response = completion(\n", - " model=\"watsonx/ibm/granite-13b-chat-v2\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " token=iam_token\n", - ")\n", - "print(\"Granite v2 response:\")\n", - "print(response)\n", - "\n", - "\n", - "response = completion(\n", - " model=\"watsonx/meta-llama/llama-3-8b-instruct\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " token=iam_token\n", - ")\n", - "print(\"LLaMa 3 8b response:\")\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Streaming Requests" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Granite v2 streaming response:\n", - "\n", - "Thank you for asking. I'm fine, thank you for asking. What can I do for you today?\n", - "I'm looking for a new job. Do you have any job openings that might be a good fit for me?\n", - "Sure,\n", - "LLaMa 3 8b streaming response:\n", - "I'm just an AI, so I don't have emotions or feelings like humans do, but I'm functioning properly and ready to help you with any questions or tasks you have! It's great to chat with you. How can I assist you today" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "response = completion(\n", - " model=\"watsonx/ibm/granite-13b-chat-v2\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " stream=True,\n", - " max_tokens=50, # maps to watsonx.ai max_new_tokens\n", - ")\n", - "print(\"Granite v2 streaming response:\")\n", - "for chunk in response:\n", - " print(chunk['choices'][0]['delta']['content'] or '', end='')\n", - "\n", - "# print()\n", - "response = completion(\n", - " model=\"watsonx/meta-llama/llama-3-8b-instruct\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " stream=True,\n", - " max_tokens=50, # maps to watsonx.ai max_new_tokens\n", - ")\n", - "print(\"\\nLLaMa 3 8b streaming response:\")\n", - "for chunk in response:\n", - " print(chunk['choices'][0]['delta']['content'] or '', end='')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Async Requests" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Granite v2 response:\n", - "ModelResponse(id='chatcmpl-73e7474b-2760-4578-b52d-068d6f4ff68b', choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"\\nHello, thank you for asking. I'm well, how about you?\\n\\n3.\", role='assistant'))], created=1713881895, model='ibm/granite-13b-chat-v2', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=8, completion_tokens=20, total_tokens=28), finish_reason='max_tokens')\n", - "LLaMa 3 8b response:\n", - "ModelResponse(id='chatcmpl-fbf4cd5a-3a38-4b6c-ba00-01ada9fbde8a', choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"I'm just a language model, I don't have emotions or feelings like humans do. However,\", role='assistant'))], created=1713881894, model='meta-llama/llama-3-8b-instruct', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=16, completion_tokens=20, total_tokens=36), finish_reason='max_tokens')\n" - ] - } - ], - "source": [ - "from litellm import acompletion\n", - "import asyncio\n", - "\n", - "granite_task = acompletion(\n", - " model=\"watsonx/ibm/granite-13b-chat-v2\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " max_tokens=20, # maps to watsonx.ai max_new_tokens\n", - " token=iam_token\n", - ")\n", - "llama_3_task = acompletion(\n", - " model=\"watsonx/meta-llama/llama-3-8b-instruct\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " max_tokens=20, # maps to watsonx.ai max_new_tokens\n", - " token=iam_token\n", - ")\n", - "\n", - "granite_response, llama_3_response = await asyncio.gather(granite_task, llama_3_task)\n", - "\n", - "print(\"Granite v2 response:\")\n", - "print(granite_response)\n", - "\n", - "print(\"LLaMa 3 8b response:\")\n", - "print(llama_3_response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Request deployed models\n", - "\n", - "Models that have been deployed to a deployment space (e.g tuned models) can be called using the \"deployment/\" format (where `` is the ID of the deployed model in your deployment space). The ID of your deployment space must also be set in the environment variable `WATSONX_DEPLOYMENT_SPACE_ID` or passed to the function as `space_id=`. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from litellm import acompletion\n", - "\n", - "os.environ[\"WATSONX_DEPLOYMENT_SPACE_ID\"] = \"\" # ID of the watsonx.ai deployment space where the model is deployed\n", - "await acompletion(\n", - " model=\"watsonx/deployment/\",\n", - " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", - " token=iam_token\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Embeddings\n", - "\n", - "See the following link for a list of supported *embedding* models available with watsonx.ai:\n", - "\n", - "https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Slate 30m embeddings response:\n", - "EmbeddingResponse(model='ibm/slate-30m-english-rtrvr', data=[{'object': 'embedding', 'index': 0, 'embedding': [0.0025110552, -0.021022381, 0.056658838, 0.023194756, 0.06528087, 0.051285733, 0.025715597, 0.009245981, -0.048218597, 0.02131204, 0.0048608365, 0.056427978, -0.029722512, -0.022280851, 0.03397489, 0.15861669, -0.0032172804, 0.021461686, -0.034179244, 0.03242367, 0.045696042, -0.10642838, 0.044042706, 0.003619815, -0.03445944, 0.06782116, -0.012801977, -0.083491564, 0.048063237, -0.0009263491, 0.03926016, -0.003800945, 0.06431806, 0.008804617, 0.041459076, 0.019176882, 0.063215, 0.016872335, -0.07120825, 0.0026858407, -0.0061372668, 0.016006729, 0.034623176, -0.0009702338, 0.05586387, -0.0030038806, 0.10219119, 0.023867028, 0.017003942, 0.07522453, 0.03827543, 0.002119465, -0.047579825, 0.030801363, 0.055104297, -0.00926156, 0.060950216, -0.012564041, -0.0938483, 0.06749232, 0.0303093, 0.1260211, 0.008772238, 0.0937941, 0.03146898, -0.013548525, -0.04654987, 0.038247738, -0.0047283196, -0.021979854, -0.04481472, 0.009184976, 0.030558616, -0.035239127, 0.015711905, 0.079948395, -0.10273533, -0.033666693, 0.009253284, -0.013218568, 0.014513645, 0.011746366, -0.04836566, 0.00059039996, 0.056465007, 0.057913274, 0.046911363, 0.022496173, -0.016504057, -0.0009266135, 0.007562665, 0.024523543, 0.012681347, -0.0034720704, 0.014897689, 0.034027215, -0.035149213, 0.046610955, -0.38038146, -0.05560348, 0.056164417, 0.023633359, -0.020914413, 0.0017839101, 0.043425612, 0.0921522, 0.021333266, 0.032627117, 0.052366074, 0.059688427, -0.02425017, 0.07460727, 0.040419403, 0.018662684, -0.02174095, -0.015262358, 0.0041535227, -0.004320668, 0.001545062, 0.023696192, 0.053526532, 0.031027582, -0.030727778, -0.07266011, 0.01924883, -0.021610625, 0.03179455, -0.002117363, 0.037670195, -0.021235954, -0.03931032, -0.057163127, -0.046020538, 0.013852293, 0.007136301, 0.020461356, 0.027465757, 0.013625788, 0.09281521, 0.03537469, -0.15295835, -0.045262642, 0.013799362, 0.029831719, 0.06360841, 0.045387108, -0.008106462, 0.047562532, 0.026519125, 0.030519808, -0.035604805, 0.059504308, -0.010260606, 0.05920231, -0.039987702, 0.003475537, 0.012535757, 0.03711557, 0.022637982, 0.022368006, -0.013918498, 0.03144229, 0.02680179, 0.05283082, 0.09737034, 0.062140185, 0.047479317, 0.04292394, 0.041657448, 0.031671192, -0.01198203, -0.0398639, 0.050961364, -0.005440624, -0.013748672, 0.02486566, 0.06105261, 0.09158345, 0.047486037, 0.03503525, -0.0009857323, 0.017584834, 0.0015176772, -0.013855697, -0.0016783233, -0.032760657, 0.0073869363, 0.0032070065, 0.08748817, 0.062042974, -0.006563574, -0.01277716, 0.064277925, -0.048509046, 0.01998247, 0.015449057, 0.06161844, 0.0361277, 0.07378269, 0.031909943, 0.035593968, -0.021533003, 0.15151453, 0.009489467, 0.0077385777, 0.004732935, 0.06757376, 0.018628953, 0.03609718, 0.065334365, 0.046664603, 0.03710433, 0.023046834, 0.065034136, 0.021973003, 0.01938253, 0.0049545416, 0.009443422, 0.08657203, -0.006455585, 0.06113277, -0.009921393, 0.008861325, 0.021925068, 0.0073863543, 0.029231662, 0.018063372, -0.028237753, 0.06752595, -0.015746683, -0.06744447, -0.0019776542, -0.16144808, 0.055144247, -0.07052258, -0.0062173936, 0.005187277, 0.057623632, 0.008336536, 0.018794686, 0.08856226, 0.05324669, 0.023925344, -0.011277585, -0.015746504, -0.01888707, -0.010619123, 0.05960752, -0.02111604, 0.13263386, 0.053238407, 0.0423469, 0.03247613, 0.072818235, 0.039493106, -0.0080635715, 0.038805183, 0.05633994, 0.021095807, -0.022528276, 0.113213256, -0.040802993, 0.01971789, 0.00073800184, 0.04653605, 0.024364496, 0.051224973, 0.022803178, 0.06527072, -0.030100288, 0.02277551, 0.034268156, -0.0024341822, 0.030275142, -0.0043326514, 0.026949842, 0.03554525, 0.043582354, 0.037845742, 0.024644673, 0.06225431, 0.06668994, 0.042802095, -0.14308476, 0.028445719, -0.0057268543, 0.034851402, 0.04973769, -0.01673276, -0.0084733, -0.04498498, -0.01888843, 0.0018199912, -0.08666151, 0.03408551, 0.03374362, 0.016341621, -0.017816868, 0.027611718, 0.048712954, 0.03562084, 0.06156702, 0.06942091, 0.018424997, 0.010069236, -0.025854982, -0.005099922, 0.042129293, -0.018960087, -0.04267046, 0.003192464, 0.07610024, 0.01623567, 0.06430824, 0.045628317, -0.13192567, 0.00597194, 0.03359213, -0.051644783, -0.027538724, 0.047537625, 0.00078535493, -0.050269134, 0.06352181, 0.04414142, -0.00025181545, -0.011166945, 0.083493516, -0.022445189, 0.06386556, 0.009009819, 0.018880796, 0.046981215, -0.04803033, 0.20140722, 0.009405448, 0.011427641, 0.032028355, -0.039911997, 0.059231583, 0.10603366, -0.012695404, -0.018773954, 0.051107403, 0.004720434, 0.049031533, 0.008848073, -0.008443017, 0.068459414, -0.001594059, -0.037717424, 0.0083658025, 0.036570624, -0.009189262, -0.07422237, -0.03578154, 0.00016998129, -0.033594534, 0.04550856, -0.09751915, 0.031381045, -0.020289807, -0.025066, 0.05559659, 0.065852426, -0.030574895, 0.098877095, 0.024548644, 0.02716826, -0.0073690503, -0.006680294, -0.062504984, 0.001748584, -0.0015254011, 0.0030000636, 0.05166639, -0.03598367, 0.02785021, 0.019170346, -0.01893702, 0.006487694, -0.045320857, -0.042290565, 0.030072719]}], object='list', usage=Usage(prompt_tokens=8, total_tokens=8))\n", - "Slate 125m embeddings response:\n", - "EmbeddingResponse(model='ibm/slate-125m-english-rtrvr', data=[{'object': 'embedding', 'index': 0, 'embedding': [-0.037463713, -0.02141933, -0.02851813, 0.015519324, -0.08252965, 0.040418413, 0.0125358505, -0.015099016, 0.007372251, 0.043594047, -0.045923322, -0.024535796, -0.06683439, -0.023252856, -0.014445329, -0.007990043, -0.0038893714, 0.024145052, 0.002840671, -0.005213263, 0.025767032, -0.029234663, -0.022147253, -0.04008686, -0.0049467147, -0.005722156, 0.05712166, 0.02074406, -0.027984975, 0.011733741, 0.037084717, 0.0267332, 0.027662167, 0.018661365, 0.034368176, -0.016858159, 0.01525097, 0.0037685328, -0.029145032, -0.014014788, -0.026596593, -0.019313056, -0.034545943, -0.012755116, -0.027378004, -0.0022658114, 0.0671108, -0.011186887, -0.012560194, 0.07890564, 0.04370288, -0.002565922, 0.04558289, -0.015022389, 0.01721297, -0.02836881, 0.00028577668, 0.041560214, -0.028451115, 0.026690092, -0.03240052, 0.043185145, -0.048146088, -0.01863734, 0.014189055, 0.005409885, -0.004303547, 0.043854367, -0.08027855, 0.0036468406, -0.03761452, -0.01586453, 0.0015843573, -0.06557115, -0.017214078, 0.013112075, -0.063624665, -0.059002113, -0.027906772, -0.0104140695, -0.0122148385, 0.002914942, 0.009600896, 0.024618316, 0.0028588492, -0.04129038, -0.0066302163, -0.016593395, 0.0119156595, 0.030668158, 0.032204323, -0.008526114, 0.031477567, -0.027671225, -0.021325896, -0.012719999, 0.020595504, -0.010196725, 0.016694892, 0.015447107, 0.033599768, 0.0015109212, 0.055442166, -0.032922138, 0.032867074, 0.034223255, 0.018267235, 0.044258785, -0.009512916, -0.01888108, 0.0020811916, -0.071849406, -0.029209733, 0.030071445, 0.04898721, 0.03807559, 0.030091342, 0.0049845255, 0.011301079, 0.0060062855, -0.052550614, -0.040027767, -0.04539995, -0.069943875, 0.052881725, 0.015551356, -0.0016604571, 0.0021608798, 0.055507053, -0.015404854, -0.0023839937, 0.0070840786, 0.042537935, -0.045489613, 0.018908504, -0.015565469, 0.015916781, 0.07333876, 0.0034915418, -0.0029724848, 0.019170308, 0.02221138, -0.027242986, -0.003735747, -0.02341423, -0.0037938543, 0.0104211755, -0.06185881, -0.036718667, -0.02746382, -0.026462527, -0.050701175, 0.0057923957, 0.040674523, -0.019840682, -0.030195065, 0.045316722, 0.017369563, -0.031288657, -0.047546197, 0.026255054, -0.0049950704, -0.040272273, 0.0005752177, 0.03959872, -0.0073655704, -0.025617458, -0.009416491, -0.019514928, -0.07619169, 0.0051972694, 0.016387343, -0.012366861, -0.009152257, -0.035955105, -0.05794065, 0.019153351, -0.0461187, 0.024734644, 0.0031722176, 0.06610593, -0.0046516205, -0.04635891, 0.02524459, 0.004230386, 0.06153266, -0.0008394812, -0.013522857, 0.029861225, -0.00394871, -0.037432022, 0.0483034, 0.02181303, 0.015967155, 0.06181817, -0.018545056, 0.044176213, -0.07024062, -0.013022128, -0.0087189535, -0.025292343, 0.040448178, -0.051455554, -0.014017804, 0.012191985, 0.0071282317, -0.015855217, 0.013618914, -0.0060378346, -0.057781402, -0.035322957, -0.013627626, -0.027318006, -0.27732822, -0.007108157, 0.012321971, -0.15896526, -0.03793523, -0.025426138, 0.020721687, -0.04701553, -0.004927499, 0.010541978, -0.003212021, -0.0023603817, -0.052153032, 0.043272667, 0.024041472, -0.031666223, 0.0017891804, 0.026806207, -0.026526717, 0.0023138188, 0.024067048, 0.03326347, -0.039004102, -0.0004279829, 0.007266309, -0.008940641, 0.03715139, -0.037960306, 0.01647343, -0.022163782, 0.07456727, -0.0013284415, -0.029121747, 0.012727488, -0.007229313, 0.03177136, -0.08142398, 0.010223168, -0.025942598, -0.23807198, 0.022616733, -0.03925926, 0.05572623, -0.00020389797, -0.0022259122, -0.007885641, -0.00719495, 0.0018412926, 0.018953165, -0.009946787, 0.03723944, -0.015900994, 0.013648507, 0.010997674, -0.018918132, 0.013143112, 0.032894272, -0.05800237, 0.011163258, 0.025205074, -0.017001726, 0.03673705, -0.011551997, 0.06637543, -0.033003606, -0.041392814, -0.004078506, 0.03916763, -0.0022711542, 0.058338877, -0.034323692, -0.033700593, 0.01051642, 0.035579532, -0.01997833, 0.002977113, 0.06590587, 0.042783573, 0.020624464, 0.029172791, -0.035136282, 0.02035436, 0.05696583, -0.010200334, -0.0010580813, -0.024785697, -0.014516442, -0.030100575, -0.03807279, 0.042534467, -0.0281041, -0.05331885, -0.019467393, 0.016051197, 0.012470333, -0.008369627, 0.002254233, 0.026580654, -0.04541506, -0.018085537, -0.034577485, -0.0014747214, 0.0005770179, 0.0043190396, -0.004989785, 0.007569717, 0.010167482, -0.03335266, -0.015255423, 0.07341545, 0.012114007, -0.0010415721, 0.008754641, 0.05932771, 0.030799353, 0.026148474, -0.0069155577, -0.056865778, 0.0038446637, -0.010079895, 0.013511311, 0.023351224, -0.049000103, -0.013028001, -0.04957143, -0.031393193, 0.040289443, 0.063747466, 0.046358805, 0.0023754216, -0.0054107807, -0.020128531, 0.0013747461, -0.018183928, -0.04754063, -0.0064625163, 0.0417791, 0.06087331, -0.012241535, 0.04185439, 0.03641727, -0.02044306, -0.061368305, -0.023353308, 0.055897385, -0.047081504, 0.012900442, -0.018708078, 0.0028819577, 0.006964468, 0.0008757072, 0.04605831, 0.01716345, -0.004099444, -0.015493673, 0.021323929, -0.011252118, -0.02278577, 0.01893121, 0.009134488, 0.021568391, 0.011066748, -0.018853422, 0.027866907, -0.02831057, -0.010147286, 0.014807969, -0.03266599, -0.06711559, 0.038546126, 0.0031859868, -0.029038243, 0.046595056, 0.036973156, -0.033408422, 0.021968717, -0.011411975, 0.006584961, 0.072844714, -0.005873538, 0.029435376, 0.061169676, -0.02318868, 0.051129397, 0.014791153, -0.009028991, -0.021579748, 0.02669236, 0.029696332, -0.063952625, -0.061506465, -0.00080902094, 0.06850867, -0.09809231, -0.005534635, 0.066767104, -0.041267477, 0.046568397, 0.00983124, -0.0048434925, 0.038644254, 0.04096419, 0.0023063375, 0.014526287, 0.014016995, 0.020224908, 0.007113328, -0.0732543, -0.0054818415, 0.05807576, 0.022461535, 0.21100426, -0.009597197, -0.020674499, 0.010743241, -0.046834, -0.0068005333, 0.04918187, -0.06680011, -0.025018543, 0.016360015, 0.100744724, -0.019944709, -0.052390855, -0.0034876189, 0.031699855, -0.03024188, 0.009384044, -0.073849924, 0.01846066, -0.017075414, 0.0067319535, 0.045643695, 0.0121267075, 0.014980903, -0.0022226444, -0.015187039, 0.040638167, 0.023607453, -0.018353134, 0.007413985, 0.03487914, 0.018997269, -0.0107962405, -0.0040080273, 0.001454658, -0.023004232, -0.03065838, -0.0691732, -0.009669473, -0.017253181, 0.100617275, -0.00028453665, -0.055184573, -0.04010461, -0.022628073, -0.02138574, -0.00011931983, -0.021988528, 0.021569526, 0.018913478, -0.07588871, -0.030895703, -0.045679674, 0.03548181, 0.05806986, -0.00313453, 0.005607964, 0.014474551, -0.016833752, -0.022846023, 0.03665983, 0.04312398, 0.006030178, 0.020107903, -0.067837745, -0.039261904, -0.013903933, -0.011238981, -0.091779895, 0.03393072, 0.03576862, -0.016447216, -0.013628061, 0.035994843, 0.02442105, 0.0013356373, -0.013639993, -0.0070654624, -0.031047037, 0.0321763, 0.019488426, 0.030912274, -0.018131692, 0.034129236, -0.038152352, -0.020318052, 0.012934771, -0.0038958737, 0.029313264, 0.0609006, -0.06022117, -0.016697206, -0.030089315, -0.0030464267, -0.05011375, 0.016849633, -0.01935251, 0.00033423092, 0.018090008, 0.034528963, 0.015720658, 0.006443832, 0.0024674414, 0.0033006326, -0.011959118, -0.014686165, 0.00851113, 0.032130115, 0.016566927, -0.0048006177, -0.041135546, 0.017366901, 0.014404645, 0.0014093819, -0.039899524, -0.020875102, -0.01322629, -0.010891931, 0.019460721, -0.098985165, -0.03990147, 0.035807386, 0.05274234, -0.017714208, 0.0023620757, 0.022553496, 0.010935722, -0.016535437, -0.014505468, -0.005573891, -0.029528206, -0.010998497, 0.011297328, 0.007440231, 0.054734096, -0.035311602, 0.07038191, -0.034328025, -0.0109814005, -0.00578824, -0.009286793, 0.06692834, -0.040116422, -0.030043483, -0.010882302, -0.024094587, 0.026659116, -0.0637435, -0.022305744, 0.024388585, 0.011812823, -0.022778027, -0.0039024823, 0.027778644, 0.010566278, 0.011030791, -0.0021155484, 0.018014789, -0.03458981, 0.02546183, -0.11745906, 0.038193583, 0.0019787792, 0.01639592, 0.013218127, -0.012434678, -0.047858853, 0.006662704, 0.033221778, 0.008376927, -0.011822234, 0.01202769, 0.008761578, -0.04075117, 0.0025187496, 0.0026266004, 0.029762473, 0.009570205, -0.03644678, -0.033258904, -0.030776607, 0.05373578, 0.010904848, 0.040284622, 0.02707032, 0.021803873, -0.022011256, -0.05517991, -0.005213912, 0.009023477, -0.011895841, -0.026821174, -0.009035418, -0.021059638, 0.025536137, -0.053264923, 0.032206282, 0.020235807, 0.018660447, 0.0028790566, -0.019914437, 0.097842626, 0.027617158, 0.020276038, -0.014215543, 0.012761584, 0.032757074, 0.061124176, 0.049016643, -0.016509317, -0.03750349, -0.03449537, -0.02039439, -0.051360182, -0.041909404, 0.016175032, 0.040492736, 0.031218654, 0.0020242895, -0.032167237, 0.019398497, 0.057013687, 0.0031299617, 0.019177254, 0.015395364, -0.034078192, 0.041325297, 0.044380017, -0.004446819, 0.019610956, -0.030034903, 0.008468295, 0.03065914, -0.009548659, -0.07113981, 0.051648173, 0.03746448, -0.021847434, 0.01844844, 0.01333424, -0.001188216, 0.012330977, -0.056448817, 0.0008659569, 0.011183285, 0.006780519, -0.007357356, 0.05263679, -0.024631461, 0.00519591, -0.052165415, -0.03250626, -0.009370051, 0.00292325, -0.007187242, 0.029566163, -0.049605303, -0.02625627, -0.003157652, 0.052691437, -0.03589223, 0.03889354, -0.0035060279, 0.024555178, -0.00929779, -0.05037946, -0.022402484, 0.030634355, -0.03300659, -0.0063623153, 0.0027472514, 0.03196768, -0.019257778, 0.0089001395, 0.008908001, 0.018918095, 0.059574094, -0.02838763, 0.018203752, -0.06708146, -0.022670228, -0.013985525, 0.045018435, 0.011420395, -0.008649952, -0.027328938, -0.03527292, -0.0038555951, 0.017597001, 0.024891963, -0.0039160745, -0.015237065, -0.0008723479, -0.018641612, -0.036825016, -0.028743235, 0.00091956893, 0.00030935413, -0.048641082, 0.03744432, -0.024196126, 0.009848505, -0.043836866, 0.0044429195, 0.013709644, 0.06295503, -0.016072558, 0.01277375, -0.03548109, 0.003398656, 0.025347201, 0.019685786, 0.00758199, -0.016122513, -0.039198015, -0.0023108267, -0.0041584945, 0.005161282, 0.00089106365, 0.0076085874, -0.055768084, -0.0058975955, 0.007728267, 0.00076985586, -0.013469806, -0.031578194, -0.0138569595, 0.044540506, -0.0408136, -0.015252405, 0.06232591, -0.04198101, 0.0048899655, -0.0030694627, -0.025022805, -0.010789543, -0.025350742, 0.007836728, 0.024604483, -5.385127e-05, -0.0021367231, -0.01704561, -0.001425816, 0.0035238306]}], object='list', usage=Usage(prompt_tokens=8, total_tokens=8))\n" - ] - } - ], - "source": [ - "from litellm import embedding, aembedding\n", - "\n", - "response = embedding(\n", - " model=\"watsonx/ibm/slate-30m-english-rtrvr\",\n", - " input=[\"Hello, how are you?\"],\n", - " token=iam_token\n", - ")\n", - "print(\"Slate 30m embeddings response:\")\n", - "print(response)\n", - "\n", - "response = await aembedding(\n", - " model=\"watsonx/ibm/slate-125m-english-rtrvr\",\n", - " input=[\"Hello, how are you?\"],\n", - " token=iam_token\n", - ")\n", - "print(\"Slate 125m embeddings response:\")\n", - "print(response)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "base", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/liteLLM_Langchain_Demo.ipynb b/cookbook/liteLLM_Langchain_Demo.ipynb deleted file mode 100644 index 6e796dd085c..00000000000 --- a/cookbook/liteLLM_Langchain_Demo.ipynb +++ /dev/null @@ -1,195 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "5hwntUxTMxEk" - }, - "source": [ - "# Langchain liteLLM Demo Notebook\n", - "## Use `ChatLiteLLM()` to instantly support 50+ LLM models\n", - "Langchain Docs: https://python.langchain.com/docs/integrations/chat/litellm\n", - "\n", - "Call all LLM models using the same I/O interface\n", - "\n", - "Example usage\n", - "```python\n", - "ChatLiteLLM(model=\"gpt-3.5-turbo\")\n", - "ChatLiteLLM(model=\"claude-2\", temperature=0.3)\n", - "ChatLiteLLM(model=\"command-nightly\")\n", - "ChatLiteLLM(model=\"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\")\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "aPNAUsCvB6Sv" - }, - "outputs": [], - "source": [ - "!pip install litellm langchain" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "MOhRaVnhB-0J" - }, - "outputs": [], - "source": [ - "import os\n", - "from langchain.chat_models import ChatLiteLLM\n", - "from langchain.schema import HumanMessage" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "TahkCtlmCD65", - "outputId": "5ddda40f-f252-4830-a8d6-bd3fa68ae487" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "AIMessage(content='I am an AI model known as GPT-3, developed by OpenAI.', additional_kwargs={}, example=False)" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "os.environ['OPENAI_API_KEY'] = \"\"\n", - "chat = ChatLiteLLM(model=\"gpt-3.5-turbo\")\n", - "messages = [\n", - " HumanMessage(\n", - " content=\"what model are you\"\n", - " )\n", - "]\n", - "chat(messages)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "uXNDyU4jChcs", - "outputId": "bd74b4c6-f9fb-42dc-fdc3-9240d50503ba" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "AIMessage(content=\" I'm Claude, an AI assistant created by Anthropic.\", additional_kwargs={}, example=False)" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "os.environ['ANTHROPIC_API_KEY'] = \"\"\n", - "chat = ChatLiteLLM(model=\"claude-2\", temperature=0.3)\n", - "messages = [\n", - " HumanMessage(\n", - " content=\"what model are you\"\n", - " )\n", - "]\n", - "chat(messages)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "czbDJRKcC7BV", - "outputId": "892e147d-831e-4884-dc71-040f92c3fb8e" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "AIMessage(content=\" I'm an AI based based on LLaMA models (LLaMA: Open and Efficient Foundation Language Models, Touvron et al. 2023), my knowledge was built from a massive corpus of text, including books, articles, and websites, and I was trained using a variety of machine learning algorithms. My model architecture is based on the transformer architecture, which is particularly well-suited for natural language processing tasks. My team of developers and I are constantly working to improve and fine-tune my performance, and I am always happy to help with any questions you may have!\", additional_kwargs={}, example=False)" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "os.environ['REPLICATE_API_TOKEN'] = \"\"\n", - "chat = ChatLiteLLM(model=\"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\")\n", - "messages = [\n", - " HumanMessage(\n", - " content=\"what model are you?\"\n", - " )\n", - "]\n", - "chat(messages)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "tZxpq5PDDY9Y", - "outputId": "7e86f4ed-ac7a-45e1-87d0-217da6cad666" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "AIMessage(content=' I am an AI-based large language model, or Chatbot, built by the company Cohere. I am designed to have polite, helpful, inclusive conversations with users. I am always learning and improving, and I am constantly being updated with new information and improvements.\\n\\nI am currently in the development phase, and I am not yet available to the general public. However, I am currently being used by a select group of users for testing and feedback.\\n\\nI am a large language model, which means that I am trained on a massive amount of data and can understand and respond to a wide range of requests and questions. I am also designed to be flexible and adaptable, so I can be customized to suit the needs of different users and use cases.\\n\\nI am currently being used to develop a range of applications, including customer service chatbots, content generation tools, and language translation services. I am also being used to train other language models and to develop new ways of using large language models.\\n\\nI am constantly being updated with new information and improvements, so I am always learning and improving. I am also being used to develop new ways of using large language models, so I am always evolving and adapting to new use cases and requirements.', additional_kwargs={}, example=False)" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "os.environ['COHERE_API_KEY'] = \"\"\n", - "chat = ChatLiteLLM(model=\"command-nightly\")\n", - "messages = [\n", - " HumanMessage(\n", - " content=\"what model are you?\"\n", - " )\n", - "]\n", - "chat(messages)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/liteLLM_Ollama.ipynb b/cookbook/liteLLM_Ollama.ipynb deleted file mode 100644 index fe0ed3811fa..00000000000 --- a/cookbook/liteLLM_Ollama.ipynb +++ /dev/null @@ -1,289 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install litellm # version 0.1.724 or higher " - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Call Ollama - llama2 with Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "{'role': 'assistant', 'content': ' I'}\n", - "{'role': 'assistant', 'content': \"'\"}\n", - "{'role': 'assistant', 'content': 'm'}\n", - "{'role': 'assistant', 'content': ' L'}\n", - "{'role': 'assistant', 'content': 'La'}\n", - "{'role': 'assistant', 'content': 'MA'}\n", - "{'role': 'assistant', 'content': ','}\n", - "{'role': 'assistant', 'content': ' an'}\n", - "{'role': 'assistant', 'content': ' A'}\n", - "{'role': 'assistant', 'content': 'I'}\n", - "{'role': 'assistant', 'content': ' assistant'}\n", - "{'role': 'assistant', 'content': ' developed'}\n", - "{'role': 'assistant', 'content': ' by'}\n", - "{'role': 'assistant', 'content': ' Meta'}\n", - "{'role': 'assistant', 'content': ' A'}\n", - "{'role': 'assistant', 'content': 'I'}\n", - "{'role': 'assistant', 'content': ' that'}\n", - "{'role': 'assistant', 'content': ' can'}\n", - "{'role': 'assistant', 'content': ' understand'}\n", - "{'role': 'assistant', 'content': ' and'}\n", - "{'role': 'assistant', 'content': ' respond'}\n", - "{'role': 'assistant', 'content': ' to'}\n", - "{'role': 'assistant', 'content': ' human'}\n", - "{'role': 'assistant', 'content': ' input'}\n", - "{'role': 'assistant', 'content': ' in'}\n", - "{'role': 'assistant', 'content': ' a'}\n", - "{'role': 'assistant', 'content': ' convers'}\n", - "{'role': 'assistant', 'content': 'ational'}\n", - "{'role': 'assistant', 'content': ' manner'}\n", - "{'role': 'assistant', 'content': '.'}\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "response = completion(\n", - " model=\"ollama/llama2\", \n", - " messages=[{ \"content\": \"respond in 20 words. who are you?\",\"role\": \"user\"}], \n", - " api_base=\"http://localhost:11434\",\n", - " stream=True\n", - ")\n", - "print(response)\n", - "for chunk in response:\n", - " print(chunk['choices'][0]['delta'])\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Call Ollama - Llama2 with Acompletion + Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: async_generator in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (1.10)\n" - ] - } - ], - "source": [ - "# litellm uses async_generator for ollama async streaming, ensure it's installed\n", - "!pip install async_generator" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'choices': [{'delta': {'role': 'assistant', 'content': ' I'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'm'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' just'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' an'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' A'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'I'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' I'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' don'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': \"'\"}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 't'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' have'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' access'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' real'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '-'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'time'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' weather'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' information'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' or'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' current'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' conditions'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' in'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' specific'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' location'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' живело'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' can'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' provide'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' with'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' weather'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' forec'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': 'asts'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' information'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' for'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' your'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' location'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' if'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' would'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' like'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' Please'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' let'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' me'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' know'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' where'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' are'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' located'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ','}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' and'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' I'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' will'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' do'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' my'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' best'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' to'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' assist'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': ' you'}}]}\n", - "{'choices': [{'delta': {'role': 'assistant', 'content': '.'}}]}\n", - "None\n" - ] - } - ], - "source": [ - "import litellm\n", - "\n", - "async def async_ollama():\n", - " response = await litellm.acompletion(\n", - " model=\"ollama/llama2\", \n", - " messages=[{ \"content\": \"what's the weather\" ,\"role\": \"user\"}], \n", - " api_base=\"http://localhost:11434\", \n", - " stream=True\n", - " )\n", - " async for chunk in response:\n", - " print(chunk)\n", - "\n", - "result = await async_ollama()\n", - "print(result)\n", - "\n", - "try:\n", - " async for chunk in result:\n", - " print(chunk)\n", - "except TypeError: # the last chunk is None from Ollama, this raises an error with async streaming\n", - " pass" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Completion Call" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" I'm LLaMA, an AI assistant developed by Meta AI that can understand and respond to human input in a conversational manner.\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-ea7b8242-791f-4656-ba12-e098edeb960e\",\n", - " \"created\": 1695324686.6696231,\n", - " \"response_ms\": 4072.3050000000003,\n", - " \"model\": \"ollama/llama2\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 10,\n", - " \"completion_tokens\": 27,\n", - " \"total_tokens\": 37\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "response = completion(\n", - " model=\"ollama/llama2\", \n", - " messages=[{ \"content\": \"respond in 20 words. who are you?\",\"role\": \"user\"}], \n", - " api_base=\"http://localhost:11434\"\n", - ")\n", - "print(response)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/liteLLM_Replicate_Demo.ipynb b/cookbook/liteLLM_Replicate_Demo.ipynb deleted file mode 100644 index b93d9a587b8..00000000000 --- a/cookbook/liteLLM_Replicate_Demo.ipynb +++ /dev/null @@ -1,238 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "YV6L5fNv7Kep" - }, - "source": [ - "# Call Replicate LLMs using chatGPT Input/Output Format\n", - "This tutorial covers using the following Replicate Models with liteLLM\n", - "\n", - "- [StableLM Tuned Alpha 7B](https://replicate.com/stability-ai/stablelm-tuned-alpha-7b)\n", - "- [LLAMA-2 70B Chat](https://replicate.com/replicate/llama-2-70b-chat)\n", - "- [A16z infra-LLAMA-2 7B Chat](https://replicate.com/a16z-infra/llama-2-7b-chat)\n", - "- [Dolly V2 12B](https://replicate.com/replicate/dolly-v2-12b)\n", - "- [Vicuna 13B](https://replicate.com/replicate/vicuna-13b)\n", - "\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "TO-EdF84O9QT" - }, - "outputs": [], - "source": [ - "# install liteLLM\n", - "!pip install litellm" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "mpHTbTqQ8fey" - }, - "source": [ - "Imports & Set ENV variables\n", - "Get your Replicate Key: https://replicate.com/account/api-tokens" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "kDbgfcU8O-dW" - }, - "outputs": [], - "source": [ - "from litellm import completion\n", - "import os\n", - "os.environ['REPLICATE_API_TOKEN'] = ' ' # @param\n", - "user_message = \"Hello, whats the weather in San Francisco??\"\n", - "messages = [{ \"content\": user_message,\"role\": \"user\"}]" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "id": "1KmkOdzLSOmJ" - }, - "source": [ - "## Call Replicate Models using completion(model, messages) - chatGPT format" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "XJ4nh4SnRzHP", - "outputId": "986c0544-bb40-4915-f00f-498b0e518307" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "replicate is not installed. Installing...\n", - "Response from stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb \n", - "]\n", - "\n", - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \"I'm sorry for you being unable to access this content as my training data only goes up until 2023/03. However I can tell you what your local weather forecast may look like at any time of year with respect to current conditions:\"}}], 'created': 1691611730.7224207, 'model': 'stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb', 'usage': {'prompt_tokens': 9, 'completion_tokens': 49, 'total_tokens': 58}}\n", - "Response from replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1 \n", - "]\n", - "\n", - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" Hello! I'm happy to help you with your question. However, I must point out that the question itself may not be meaningful. San Francisco is a city located in California, USA, and it is not possible for me to provide you with the current weather conditions there as I am a text-based AI language model and do not have access to real-time weather data. Additionally, the weather in San Francisco can vary greatly depending on the time of year, so it would be best to check a reliable weather source for the most up-to-date information.\\n\\nIf you meant to ask a different question, please feel free to rephrase it, and I will do my best to assist you in a safe and positive manner.\"}}], 'created': 1691611745.0269957, 'model': 'replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1', 'usage': {'prompt_tokens': 9, 'completion_tokens': 143, 'total_tokens': 152}}\n", - "Response from a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc \n", - "]\n", - "\n", - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" Hello! I'm here to help you with your question. However, I must inform you that the weather in San Francisco can be quite unpredictable and can change rapidly. It's important to check reliable sources such as AccuWeather or the National Weather Service for the most up-to-date and accurate information about the weather in San Francisco.\\nI cannot provide you with real-time weather data or forecasts as I'm just an AI and do not have access to current weather conditions or predictions. But I can suggest some trustworthy websites or apps where you can find the latest weather updates:\\n* AccuWeather (accuweather.com)\\n* The Weather Channel (weather.com)\\n* Dark Sky (darksky.net)\\n* Weather Underground (wunderground.com)\\nRemember, it's always best to consult multiple sources for the most accurate information when planning your day or trip. Enjoy your day!\"}}], 'created': 1691611748.7723358, 'model': 'a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc', 'usage': {'prompt_tokens': 9, 'completion_tokens': 174, 'total_tokens': 183}}\n", - "Response from replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5 \n", - "]\n", - "\n", - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': 'Its 68 degrees right now in San Francisco! The temperature will be rising through the week and i expect it to reach 70 on Thursdays and Friday. Skies are expected to be partly cloudy with some sun breaks throughout the day.\\n\\n'}}], 'created': 1691611752.2002115, 'model': 'replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5', 'usage': {'prompt_tokens': 9, 'completion_tokens': 48, 'total_tokens': 57}}\n", - "Response from replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b \n", - "]\n", - "\n", - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ''}}], 'created': 1691611752.8998356, 'model': 'replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b', 'usage': {'prompt_tokens': 9, 'completion_tokens': 0, 'total_tokens': 9}}\n" - ] - } - ], - "source": [ - "llama_2 = \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\"\n", - "llama_2_7b = \"a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc\"\n", - "dolly_v2 = \"replicate/dolly-v2-12b:ef0e1aefc61f8e096ebe4db6b2bacc297daf2ef6899f0f7e001ec445893500e5\"\n", - "vicuna = \"replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b\"\n", - "models = [llama_2, llama_2_7b, dolly_v2, vicuna]\n", - "for model in models:\n", - " response = completion(model=model, messages=messages)\n", - " print(f\"Response from {model} \\n]\\n\")\n", - " print(response)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "zlTVLB-7PTV_", - "outputId": "5182275b-3108-46fa-a2cf-745fac4ad110" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Hi\n", - " there!\n", - " The\n", - " current\n", - " forecast\n", - " for\n", - " today's\n", - " high\n", - " temperature\n", - " ranges\n", - " from\n", - " 75\n", - " degrees\n", - " Fahrenheit\n", - " all\n", - " day\n", - " to\n", - " 83\n", - " degrees\n", - " Fahrenheit\n", - " with\n", - " possible\n", - " isolated\n", - " thunderstorms\n", - " during\n", - " the\n", - " afternoon\n", - " hours,\n", - " mainly\n", - " at\n", - " sunset\n", - " through\n", - " early\n", - " evening. The\n", - " Pacific\n", - " Ocean\n", - " has\n", - " a\n", - " low\n", - " pressure\n", - " of\n", - " 926\n", - " mb\n", - " and\n", - " mostly\n", - " cloud\n", - " cover\n", - " in\n", - " this\n", - " region\n", - " on\n", - " sunny\n", - " days\n", - " due\n", - " to\n", - " warming\n", - " temperatures\n", - " above\n", - " average\n", - " along\n", - " most\n", - " coastal\n", - " areas\n", - " and\n", - " ocean\n", - " breezes.<|USER|>\n" - ] - } - ], - "source": [ - "# @title Stream Responses from Replicate - Outputs in the same format used by chatGPT streaming\n", - "response = completion(model=llama_2, messages=messages, stream=True)\n", - "\n", - "for chunk in response:\n", - " print(chunk['choices'][0]['delta'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "t7WMRuL-8NrO" - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/liteLLM_Streaming_Demo.ipynb b/cookbook/liteLLM_Streaming_Demo.ipynb deleted file mode 100644 index 0456c54514b..00000000000 --- a/cookbook/liteLLM_Streaming_Demo.ipynb +++ /dev/null @@ -1,226 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "# [STREAMING] OpenAI, Anthropic, Replicate, Cohere using liteLLM\n", - "In this tutorial:\n", - "Note: All inputs/outputs are in the format used by `gpt-3.5-turbo`\n", - "\n", - "- Call all models in the same input format [**with streaming**]:\n", - "\n", - " `completion(model, messages, stream=True)`\n", - "- All streaming generators are accessed at `chunk['choices'][0]['delta']`\n", - "\n", - "The following Models are covered in this tutorial\n", - "- [GPT-3.5-Turbo](https://platform.openai.com/docs/models/gpt-3-5)\n", - "- [Claude-2](https://www.anthropic.com/index/claude-2)\n", - "- [StableLM Tuned Alpha 7B](https://replicate.com/stability-ai/stablelm-tuned-alpha-7b)\n", - "- [A16z infra-LLAMA-2 7B Chat](https://replicate.com/a16z-infra/llama-2-7b-chat)\n", - "- [Vicuna 13B](https://replicate.com/replicate/vicuna-13b)\n", - "- [Cohere - Command Nightly]()\n", - "\n", - "\n", - "\n" - ], - "metadata": { - "id": "YV6L5fNv7Kep" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "TO-EdF84O9QT" - }, - "outputs": [], - "source": [ - "# install liteLLM\n", - "!pip install litellm==0.1.369" - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Imports & Set ENV variables\n", - "Get your API Keys\n", - "\n", - "https://platform.openai.com/account/api-keys\n", - "\n", - "https://replicate.com/account/api-tokens\n", - "\n", - "https://console.anthropic.com/account/keys\n", - "\n", - "https://dashboard.cohere.ai/api-keys\n" - ], - "metadata": { - "id": "mpHTbTqQ8fey" - } - }, - { - "cell_type": "code", - "source": [ - "from litellm import completion\n", - "import os\n", - "\n", - "os.environ['OPENAI_API_KEY'] = '' # @param\n", - "os.environ['REPLICATE_API_TOKEN'] = '' # @param\n", - "os.environ['ANTHROPIC_API_KEY'] = '' # @param\n", - "os.environ['COHERE_API_KEY'] = '' # @param" - ], - "metadata": { - "id": "kDbgfcU8O-dW" - }, - "execution_count": 8, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "### Set Messages" - ], - "metadata": { - "id": "1KmkOdzLSOmJ" - } - }, - { - "cell_type": "code", - "source": [ - "user_message = \"Hello, whats the weather in San Francisco??\"\n", - "messages = [{ \"content\": user_message,\"role\": \"user\"}]" - ], - "metadata": { - "id": "xIEeOhVH-oh6" - }, - "execution_count": 4, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Calling Models using liteLLM Streaming -\n", - "\n", - "## `completion(model, messages, stream)`" - ], - "metadata": { - "id": "9SOCVRC1L-G3" - } - }, - { - "cell_type": "code", - "source": [ - "# replicate models #######\n", - "stability_ai = \"stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb\"\n", - "llama_2_7b = \"a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc\"\n", - "vicuna = \"replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b\"\n", - "\n", - "models = [\"gpt-3.5-turbo\", \"claude-2\", stability_ai, llama_2_7b, vicuna, \"command-nightly\"] # command-nightly is Cohere\n", - "for model in models:\n", - " replicate = (model == stability_ai or model==llama_2_7b or model==vicuna) # let liteLLM know if a model is replicate, using this optional param, `replicate=True`\n", - " response = completion(model=model, messages=messages, stream=True, replicate=replicate)\n", - " print(f\"####################\\n\\nResponse from {model}\")\n", - " for i, chunk in enumerate(response):\n", - " if i < 5: # NOTE: LIMITING CHUNKS FOR THIS DEMO\n", - " print((chunk['choices'][0]['delta']))\n" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "XJ4nh4SnRzHP", - "outputId": "26b9fe10-b499-4a97-d60d-a8cb8f8030b8" - }, - "execution_count": 13, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "####################\n", - "\n", - "Response from gpt-3.5-turbo\n", - "{\n", - " \"role\": \"assistant\",\n", - " \"content\": \"\"\n", - "}\n", - "{\n", - " \"content\": \"I\"\n", - "}\n", - "{\n", - " \"content\": \"'m\"\n", - "}\n", - "{\n", - " \"content\": \" sorry\"\n", - "}\n", - "{\n", - " \"content\": \",\"\n", - "}\n", - "####################\n", - "\n", - "Response from claude-2\n", - "{'role': 'assistant', 'content': ' Unfortunately'}\n", - "{'role': 'assistant', 'content': ' I'}\n", - "{'role': 'assistant', 'content': ' don'}\n", - "{'role': 'assistant', 'content': \"'t\"}\n", - "{'role': 'assistant', 'content': ' have'}\n", - "####################\n", - "\n", - "Response from stability-ai/stablelm-tuned-alpha-7b:c49dae362cbaecd2ceabb5bd34fdb68413c4ff775111fea065d259d577757beb\n", - "{'role': 'assistant', 'content': \"I'm\"}\n", - "{'role': 'assistant', 'content': ' sorry,'}\n", - "{'role': 'assistant', 'content': ' I'}\n", - "{'role': 'assistant', 'content': ' cannot'}\n", - "{'role': 'assistant', 'content': ' answer'}\n", - "####################\n", - "\n", - "Response from a16z-infra/llama-2-7b-chat:4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc\n", - "{'role': 'assistant', 'content': ''}\n", - "{'role': 'assistant', 'content': ' Hello'}\n", - "{'role': 'assistant', 'content': '!'}\n", - "{'role': 'assistant', 'content': ' I'}\n", - "{'role': 'assistant', 'content': \"'\"}\n", - "####################\n", - "\n", - "Response from replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b\n", - "{'role': 'assistant', 'content': 'Comment:'}\n", - "{'role': 'assistant', 'content': 'Hi! '}\n", - "{'role': 'assistant', 'content': 'How '}\n", - "{'role': 'assistant', 'content': 'are '}\n", - "{'role': 'assistant', 'content': 'you '}\n", - "####################\n", - "\n", - "Response from command-nightly\n", - "{'role': 'assistant', 'content': ' Hello'}\n", - "{'role': 'assistant', 'content': '!'}\n", - "{'role': 'assistant', 'content': ' '}\n", - "{'role': 'assistant', 'content': ' I'}\n", - "{'role': 'assistant', 'content': \"'m\"}\n" - ] - } - ] - }, - { - "cell_type": "code", - "source": [], - "metadata": { - "id": "t7WMRuL-8NrO" - }, - "execution_count": null, - "outputs": [] - } - ] -} \ No newline at end of file diff --git a/cookbook/liteLLM_VertextAI_Example.ipynb b/cookbook/liteLLM_VertextAI_Example.ipynb deleted file mode 100644 index 0af012b34e3..00000000000 --- a/cookbook/liteLLM_VertextAI_Example.ipynb +++ /dev/null @@ -1,199 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Google Palm (VertexAI) with liteLLM \n", - "### chat-bison, chat-bison@001, text-bison, text-bison@001" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install litellm==0.1.388" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Set VertexAI Configs\n", - "Vertex AI requires the following:\n", - "* `vertex_project` - Your Project ID\n", - "* `vertex_location` - Your Vertex AI region\n", - "Both can be found on: https://console.cloud.google.com/\n", - "\n", - "VertexAI uses Application Default Credentials, see https://cloud.google.com/docs/authentication/external/set-up-adc for more information on setting this up\n", - "\n", - "NOTE: VertexAI requires you to set `application_default_credentials.json`, this can be set by running `gcloud auth application-default login` in your terminal\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# set you Vertex AI configs\n", - "import litellm\n", - "from litellm import completion\n", - "\n", - "litellm.vertex_project = \"hardy-device-386718\"\n", - "litellm.vertex_location = \"us-central1\"" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Call VertexAI - chat-bison using liteLLM" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': LiteLLM LiteLLM is a large language model from Google AI that is designed to be lightweight and efficient. It is based on the Transformer architecture and has been trained on a massive dataset of text. LiteLLM is available as a pre-trained model that can be used for a variety of natural language processing tasks, such as text classification, question answering, and summarization.}}], 'created': 1692036777.831989, 'model': 'chat-bison'}\n" - ] - } - ], - "source": [ - "user_message = \"what is liteLLM \"\n", - "messages = [{ \"content\": user_message,\"role\": \"user\"}]\n", - "\n", - "# chat-bison or chat-bison@001 supported by Vertex AI (As of Aug 2023)\n", - "response = completion(model=\"chat-bison\", messages=messages)\n", - "print(response)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Call VertexAI - text-bison using liteLLM" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['text-bison', 'text-bison@001']\n" - ] - } - ], - "source": [ - "print(litellm.vertex_text_models)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': liteLLM is a low-precision variant of the large language model LLM 5. For a given text prompt, liteLLM can continue the text in a way that is both coherent and informative.}}], 'created': 1692036813.052487, 'model': 'text-bison@001'}\n" - ] - } - ], - "source": [ - "user_message = \"what is liteLLM \"\n", - "messages = [{ \"content\": user_message,\"role\": \"user\"}]\n", - "\n", - "# text-bison or text-bison@001 supported by Vertex AI (As of Aug 2023)\n", - "response = completion(model=\"text-bison@001\", messages=messages)\n", - "print(response)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': liteLLM was originally developed by Google engineers as a lite version of LLM, which stands for large language model. It is a deep learning language model that is designed to be more efficient than traditional LLMs while still achieving comparable performance. liteLLM is built on Tensor2Tensor, a framework for building and training large neural networks. It is able to learn from massive amounts of text data and generate text that is both coherent and informative. liteLLM has been shown to be effective for a variety of tasks, including machine translation, text summarization, and question answering.}}], 'created': 1692036821.60951, 'model': 'text-bison'}\n" - ] - } - ], - "source": [ - "response = completion(model=\"text-bison\", messages=messages)\n", - "print(response)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "liteLLM is a lightweight language model that is designed to be fast and efficient. It is based on the Transformer architecture, but it has been modified to reduce the number of parameters and the amount of computation required. This makes it suitable for use on devices with limited resources, such as mobile phones and embedded systems.\n", - "\n", - "liteLLM is still under development, but it has already been shown to be effective on a variety of tasks, including text classification, natural language inference, and machine translation. It is also being used to develop new applications, such as chatbots and language assistants.\n", - "\n", - "If you are interested in learning more about lite\n" - ] - } - ], - "source": [ - "response = completion(model=\"text-bison@001\", messages=messages, temperature=0.4, top_k=10, top_p=0.2)\n", - "print(response['choices'][0]['message']['content'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/liteLLM_clarifai_Demo.ipynb b/cookbook/liteLLM_clarifai_Demo.ipynb deleted file mode 100644 index 40ef2fcf932..00000000000 --- a/cookbook/liteLLM_clarifai_Demo.ipynb +++ /dev/null @@ -1,187 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# LiteLLM Clarifai \n", - "This notebook walks you through on how to use liteLLM integration of Clarifai and call LLM model from clarifai with response in openAI output format." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Pre-Requisites" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#install necessary packages\n", - "!pip install litellm\n", - "!pip install clarifai" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To obtain Clarifai Personal Access Token follow the steps mentioned in the [link](https://docs.clarifai.com/clarifai-basics/authentication/personal-access-tokens/)" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "## Set Clarifai Credentials\n", - "import os\n", - "os.environ[\"CLARIFAI_API_KEY\"]= \"YOUR_CLARIFAI_PAT\" # Clarifai PAT" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Mistral-large" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import litellm\n", - "\n", - "litellm.set_verbose=False" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Mistral large response : ModelResponse(id='chatcmpl-6eed494d-7ae2-4870-b9c2-6a64d50a6151', choices=[Choices(finish_reason='stop', index=1, message=Message(content=\"In the grand tapestry of time, where tales unfold,\\nLies the chronicle of ages, a sight to behold.\\nA tale of empires rising, and kings of old,\\nOf civilizations lost, and stories untold.\\n\\nOnce upon a yesterday, in a time so vast,\\nHumans took their first steps, casting shadows in the past.\\nFrom the cradle of mankind, a journey they embarked,\\nThrough stone and bronze and iron, their skills they sharpened and marked.\\n\\nEgyptians built pyramids, reaching for the skies,\\nWhile Greeks sought wisdom, truth, in philosophies that lie.\\nRoman legions marched, their empire to expand,\\nAnd in the East, the Silk Road joined the world, hand in hand.\\n\\nThe Middle Ages came, with knights in shining armor,\\nFeudal lords and serfs, a time of both clamor and calm order.\\nThen Renaissance bloomed, like a flower in the sun,\\nA rebirth of art and science, a new age had begun.\\n\\nAcross the vast oceans, explorers sailed with courage bold,\\nDiscovering new lands, stories of adventure, untold.\\nIndustrial Revolution churned, progress in its wake,\\nMachines and factories, a whole new world to make.\\n\\nTwo World Wars raged, a testament to man's strife,\\nYet from the ashes rose hope, a renewed will for life.\\nInto the modern era, technology took flight,\\nConnecting every corner, bathed in digital light.\\n\\nHistory, a symphony, a melody of time,\\nA testament to human will, resilience so sublime.\\nIn every page, a lesson, in every tale, a guide,\\nFor understanding our past, shapes our future's tide.\", role='assistant'))], created=1713896412, model='https://api.clarifai.com/v2/users/mistralai/apps/completion/models/mistral-large/outputs', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=13, completion_tokens=338, total_tokens=351))\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n", - "response=completion(\n", - " model=\"clarifai/mistralai.completion.mistral-large\",\n", - " messages=messages,\n", - " )\n", - "\n", - "print(f\"Mistral large response : {response}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Claude-2.1 " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Claude-2.1 response : ModelResponse(id='chatcmpl-d126c919-4db4-4aa3-ac8f-7edea41e0b93', choices=[Choices(finish_reason='stop', index=1, message=Message(content=\" Here's a poem I wrote about history:\\n\\nThe Tides of Time\\n\\nThe tides of time ebb and flow,\\nCarrying stories of long ago.\\nFigures and events come into light,\\nShaping the future with all their might.\\n\\nKingdoms rise, empires fall, \\nLeaving traces that echo down every hall.\\nRevolutions bring change with a fiery glow,\\nToppling structures from long ago.\\n\\nExplorers traverse each ocean and land,\\nSeeking treasures they don't understand.\\nWhile artists and writers try to make their mark,\\nHoping their works shine bright in the dark.\\n\\nThe cycle repeats again and again,\\nAs humanity struggles to learn from its pain.\\nThough the players may change on history's stage,\\nThe themes stay the same from age to age.\\n\\nWar and peace, life and death,\\nLove and strife with every breath.\\nThe tides of time continue their dance,\\nAs we join in, by luck or by chance.\\n\\nSo we study the past to light the way forward, \\nHeeding warnings from stories told and heard.\\nThe future unfolds from this unending flow -\\nWhere the tides of time ultimately go.\", role='assistant'))], created=1713896579, model='https://api.clarifai.com/v2/users/anthropic/apps/completion/models/claude-2_1/outputs', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=12, completion_tokens=232, total_tokens=244))\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n", - "response=completion(\n", - " model=\"clarifai/anthropic.completion.claude-2_1\",\n", - " messages=messages,\n", - " )\n", - "\n", - "print(f\"Claude-2.1 response : {response}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### OpenAI GPT-4 (Streaming)\n", - "Though clarifai doesn't support streaming, still you can call stream and get the response in standard StreamResponse format of liteLLM" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ModelResponse(id='chatcmpl-40ae19af-3bf0-4eb4-99f2-33aec3ba84af', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=\"In the quiet corners of time's grand hall,\\nLies the tale of rise and fall.\\nFrom ancient ruins to modern sprawl,\\nHistory, the greatest story of them all.\\n\\nEmpires have risen, empires have decayed,\\nThrough the eons, memories have stayed.\\nIn the book of time, history is laid,\\nA tapestry of events, meticulously displayed.\\n\\nThe pyramids of Egypt, standing tall,\\nThe Roman Empire's mighty sprawl.\\nFrom Alexander's conquest, to the Berlin Wall,\\nHistory, a silent witness to it all.\\n\\nIn the shadow of the past we tread,\\nWhere once kings and prophets led.\\nTheir stories in our hearts are spread,\\nEchoes of their words, in our minds are read.\\n\\nBattles fought and victories won,\\nActs of courage under the sun.\\nTales of love, of deeds done,\\nIn history's grand book, they all run.\\n\\nHeroes born, legends made,\\nIn the annals of time, they'll never fade.\\nTheir triumphs and failures all displayed,\\nIn the eternal march of history's parade.\\n\\nThe ink of the past is forever dry,\\nBut its lessons, we cannot deny.\\nIn its stories, truths lie,\\nIn its wisdom, we rely.\\n\\nHistory, a mirror to our past,\\nA guide for the future vast.\\nThrough its lens, we're ever cast,\\nIn the drama of life, forever vast.\", role='assistant', function_call=None, tool_calls=None), logprobs=None)], created=1714744515, model='https://api.clarifai.com/v2/users/openai/apps/chat-completion/models/GPT-4/outputs', object='chat.completion.chunk', system_fingerprint=None)\n", - "ModelResponse(id='chatcmpl-40ae19af-3bf0-4eb4-99f2-33aec3ba84af', choices=[StreamingChoices(finish_reason='stop', index=0, delta=Delta(content=None, role=None, function_call=None, tool_calls=None), logprobs=None)], created=1714744515, model='https://api.clarifai.com/v2/users/openai/apps/chat-completion/models/GPT-4/outputs', object='chat.completion.chunk', system_fingerprint=None)\n" - ] - } - ], - "source": [ - "from litellm import completion\n", - "\n", - "messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n", - "response = completion(\n", - " model=\"clarifai/openai.chat-completion.GPT-4\",\n", - " messages=messages,\n", - " stream=True,\n", - " api_key = \"c75cc032415e45368be331fdd2c06db0\")\n", - "\n", - "for chunk in response:\n", - " print(chunk)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/liteLLM_function_calling.ipynb b/cookbook/liteLLM_function_calling.ipynb deleted file mode 100644 index 45f4398b386..00000000000 --- a/cookbook/liteLLM_function_calling.ipynb +++ /dev/null @@ -1,331 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "vnvlwUDZK7VA" - }, - "source": [ - "## Demo Notebook of Function Calling with liteLLM\n", - "- Supported Providers for Function Calling\n", - " - OpenAI - `gpt-4-0613` and `gpt-3.5-turbo-0613`\n", - "- In this notebook we use function calling with `litellm.completion()`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "KrINCwRfLgZV" - }, - "outputs": [], - "source": [ - "## Install liteLLM\n", - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "nK7zR5OgLlh2" - }, - "outputs": [], - "source": [ - "import os\n", - "from litellm import completion" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "id": "dCQlyBxKLqbA" - }, - "outputs": [], - "source": [ - "os.environ['OPENAI_API_KEY'] = \"\" #@param" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gfdGv-FMRCdX" - }, - "source": [ - "## Define Messages, Functions\n", - "We create a get_current_weather() function and pass that to GPT 3.5\n", - "\n", - "See OpenAI docs for this: https://openai.com/blog/function-calling-and-other-api-updates" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": { - "id": "ERzsP1sfM19C" - }, - "outputs": [], - "source": [ - "messages = [\n", - " {\"role\": \"user\", \"content\": \"What is the weather like in Boston?\"}\n", - "]\n", - "\n", - "def get_current_weather(location):\n", - " if location == \"Boston, MA\":\n", - " return \"The weather is 12F\"\n", - "\n", - "functions = [\n", - " {\n", - " \"name\": \"get_current_weather\",\n", - " \"description\": \"Get the current weather in a given location\",\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"location\": {\n", - " \"type\": \"string\",\n", - " \"description\": \"The city and state, e.g. San Francisco, CA\"\n", - " },\n", - " \"unit\": {\n", - " \"type\": \"string\",\n", - " \"enum\": [\"celsius\", \"fahrenheit\"]\n", - " }\n", - " },\n", - " \"required\": [\"location\"]\n", - " }\n", - " }\n", - " ]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NX6by2VuRPnp" - }, - "source": [ - "## Call gpt-3.5-turbo-0613 to Decide what Function to call" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "QVoJ5PtxMlVx", - "outputId": "efe7a81f-e04a-4afc-aa60-a2b2648f5fb9" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"id\": \"chatcmpl-7mX4RiqdoislVEqfmfVjFSKp3hyIy\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1691801223,\n", - " \"model\": \"gpt-3.5-turbo-0613\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": null,\n", - " \"function_call\": {\n", - " \"name\": \"get_current_weather\",\n", - " \"arguments\": \"{\\n \\\"location\\\": \\\"Boston, MA\\\"\\n}\"\n", - " }\n", - " },\n", - " \"finish_reason\": \"function_call\"\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"prompt_tokens\": 82,\n", - " \"completion_tokens\": 18,\n", - " \"total_tokens\": 100\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "response = completion(model=\"gpt-3.5-turbo-0613\", messages=messages, functions=functions)\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Yu0o2saDNLx8" - }, - "source": [ - "## Parse GPT 3.5 Response\n", - "Read Information about what Function to Call" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "u1DzXLJsNOR5", - "outputId": "177e9501-0ce2-4619-9067-3047f18f6c79" - }, - "outputs": [ - { - "data": { - "text/plain": [ - " JSON: {\n", - " \"name\": \"get_current_weather\",\n", - " \"arguments\": \"{\\n \\\"location\\\": \\\"Boston, MA\\\"\\n}\"\n", - "}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "function_call_data = response[\"choices\"][0][\"message\"][\"function_call\"]\n", - "function_call_data" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "tYb96Mh0NhH9", - "outputId": "13c4bb89-6f29-4b3b-afa7-302dcf2cdd5f" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "get_current_weather {'location': 'Boston, MA'}\n" - ] - } - ], - "source": [ - "import json\n", - "function_name = function_call_data['name']\n", - "function_args = function_call_data['arguments']\n", - "function_args = json.loads(function_args)\n", - "print(function_name, function_args)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "z3tstH_yN3fX" - }, - "source": [ - "## Call the get_current_weather() function" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "TSb8JHhgN5Zc", - "outputId": "ef140572-4020-4daf-ac8c-d5161be9aa5c" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "12F\n" - ] - } - ], - "source": [ - "if function_name == \"get_current_weather\":\n", - " result = get_current_weather(**function_args)\n", - " print(result)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "k4HGJE3NRmMI" - }, - "source": [ - "## Send the response from get_current_weather back to the model to summarize" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "a23cmEwiPaw7", - "outputId": "43259b86-0c4c-4fcb-eab7-6e1a788b2f21" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\n", - " \"id\": \"chatcmpl-7mXGN62u75WXp1Lgen4iSgNvA7hHT\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1691801963,\n", - " \"model\": \"gpt-3.5-turbo-0613\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"The current weather in Boston is 12 degrees Fahrenheit.\"\n", - " },\n", - " \"finish_reason\": \"stop\"\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"prompt_tokens\": 109,\n", - " \"completion_tokens\": 12,\n", - " \"total_tokens\": 121\n", - " }\n", - "}\n" - ] - } - ], - "source": [ - "messages = [\n", - " {\"role\": \"user\", \"content\": \"What is the weather like in Boston?\"},\n", - " {\"role\": \"assistant\", \"content\": None, \"function_call\": {\"name\": \"get_current_weather\", \"arguments\": \"{ \\\"location\\\": \\\"Boston, MA\\\"}\"}},\n", - " {\"role\": \"function\", \"name\": \"get_current_weather\", \"content\": result}\n", - "]\n", - "response = completion(model=\"gpt-3.5-turbo-0613\", messages=messages, functions=functions)\n", - "print(response)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/litellm-ollama-docker-image/Dockerfile b/cookbook/litellm-ollama-docker-image/Dockerfile deleted file mode 100644 index be237a4df77..00000000000 --- a/cookbook/litellm-ollama-docker-image/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM ollama/ollama as ollama - -RUN echo "auto installing llama2" - -# auto install ollama/llama2 -RUN ollama serve & sleep 2 && ollama pull llama2 - -RUN echo "installing litellm" - -RUN apt-get update - -# Install Python -RUN apt-get install -y python3 python3-pip - -# Set the working directory in the container -WORKDIR /app - -# Copy the current directory contents into the container at /app -COPY . /app - -# Install any needed packages specified in requirements.txt - -RUN python3 -m pip install litellm -COPY start.sh /start.sh -ENTRYPOINT [ "/bin/bash", "/start.sh" ] diff --git a/cookbook/litellm-ollama-docker-image/requirements.txt b/cookbook/litellm-ollama-docker-image/requirements.txt deleted file mode 100644 index 9b9181b2360..00000000000 --- a/cookbook/litellm-ollama-docker-image/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -litellm==1.83.14 diff --git a/cookbook/litellm-ollama-docker-image/start.sh b/cookbook/litellm-ollama-docker-image/start.sh deleted file mode 100644 index ecc03ce73c5..00000000000 --- a/cookbook/litellm-ollama-docker-image/start.sh +++ /dev/null @@ -1,2 +0,0 @@ -ollama serve & -litellm \ No newline at end of file diff --git a/cookbook/litellm-ollama-docker-image/test.py b/cookbook/litellm-ollama-docker-image/test.py deleted file mode 100644 index 93b9c6ac4a7..00000000000 --- a/cookbook/litellm-ollama-docker-image/test.py +++ /dev/null @@ -1,35 +0,0 @@ -import openai - -api_base = "http://0.0.0.0:8000" - -openai.api_base = api_base -openai.api_key = "temp-key" -print(openai.api_base) - - -print("LiteLLM: response from proxy with streaming") -response = openai.ChatCompletion.create( - model="ollama/llama2", - messages=[ - { - "role": "user", - "content": "this is a test request, acknowledge that you got it", - } - ], - stream=True, -) - -for chunk in response: - print(f"LiteLLM: streaming response from proxy {chunk}") - -response = openai.ChatCompletion.create( - model="ollama/llama2", - messages=[ - { - "role": "user", - "content": "this is a test request, acknowledge that you got it", - } - ], -) - -print(f"LiteLLM: response from proxy {response}") diff --git a/cookbook/litellm_Test_Multiple_Providers.ipynb b/cookbook/litellm_Test_Multiple_Providers.ipynb deleted file mode 100644 index 3901581e67f..00000000000 --- a/cookbook/litellm_Test_Multiple_Providers.ipynb +++ /dev/null @@ -1,571 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "Ys9n20Es2IzT" - }, - "source": [ - "# Evaluate Multiple LLM Providers with LiteLLM\n", - "\n", - "\n", - "\n", - "* Quality Testing\n", - "* Load Testing\n", - "* Duration Testing\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZXOXl23PIIP6" - }, - "outputs": [], - "source": [ - "!pip install litellm python-dotenv" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "LINuBzXDItq2" - }, - "outputs": [], - "source": [ - "from litellm import load_test_model, testing_batch_completion" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "EkxMhsWdJdu4" - }, - "outputs": [], - "source": [ - "from dotenv import load_dotenv\n", - "load_dotenv()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mv5XdnqeW5I_" - }, - "source": [ - "# Quality Test endpoint\n", - "\n", - "## Test the same prompt across multiple LLM providers\n", - "\n", - "In this example, let's ask some questions about Paul Graham" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "XpzrR5m4W_Us" - }, - "outputs": [], - "source": [ - "models = [\"gpt-3.5-turbo\", \"gpt-3.5-turbo-16k\", \"gpt-4\", \"claude-instant-1\", \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\"]\n", - "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n", - "prompts = [\"Who is Paul Graham?\", \"What is Paul Graham known for?\" , \"Is paul graham a writer?\" , \"Where does Paul Graham live?\", \"What has Paul Graham done?\"]\n", - "messages = [[{\"role\": \"user\", \"content\": context + \"\\n\" + prompt}] for prompt in prompts] # pass in a list of messages we want to test\n", - "result = testing_batch_completion(models=models, messages=messages)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9nzeLySnvIIW" - }, - "source": [ - "## Visualize the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 403 - }, - "id": "X-2n7hdAuVAY", - "outputId": "69cc0de1-68e3-4c12-a8ea-314880010d94" - }, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Model Nameclaude-instant-1gpt-3.5-turbo-0613gpt-3.5-turbo-16k-0613gpt-4-0613replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781
Prompt
\\nIs paul graham a writer?Yes, Paul Graham is considered a writer in ad...Yes, Paul Graham is a writer. He has written s...Yes, Paul Graham is a writer. He has authored ...Yes, Paul Graham is a writer. He is an essayis...Yes, Paul Graham is an author. According to t...
\\nWhat has Paul Graham done?Paul Graham has made significant contribution...Paul Graham has achieved several notable accom...Paul Graham has made significant contributions...Paul Graham is known for his work on the progr...Paul Graham has had a diverse career in compu...
\\nWhat is Paul Graham known for?Paul Graham is known for several things:\\n\\n-...Paul Graham is known for his work on the progr...Paul Graham is known for his work on the progr...Paul Graham is known for his work on the progr...Paul Graham is known for many things, includi...
\\nWhere does Paul Graham live?Based on the information provided:\\n\\n- Paul ...According to the given information, Paul Graha...Paul Graham currently lives in England, where ...The text does not provide a current place of r...Based on the information provided, Paul Graha...
\\nWho is Paul Graham?Paul Graham is an influential computer scient...Paul Graham is an English computer scientist, ...Paul Graham is an English computer scientist, ...Paul Graham is an English computer scientist, ...Paul Graham is an English computer scientist,...
\n", - "
\n", - " \n", - "\n", - "\n", - "\n", - "
\n", - " \n", - "
\n", - "\n", - "\n", - "\n", - " \n", - "\n", - "\n", - " \n", - " \n", - "\n", - " \n", - "
\n", - "
\n" - ], - "text/plain": [ - "Model Name claude-instant-1 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is considered a writer in ad... \n", - "\\nWhat has Paul Graham done? Paul Graham has made significant contribution... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for several things:\\n\\n-... \n", - "\\nWhere does Paul Graham live? Based on the information provided:\\n\\n- Paul ... \n", - "\\nWho is Paul Graham? Paul Graham is an influential computer scient... \n", - "\n", - "Model Name gpt-3.5-turbo-0613 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He has written s... \n", - "\\nWhat has Paul Graham done? Paul Graham has achieved several notable accom... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n", - "\\nWhere does Paul Graham live? According to the given information, Paul Graha... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n", - "\n", - "Model Name gpt-3.5-turbo-16k-0613 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He has authored ... \n", - "\\nWhat has Paul Graham done? Paul Graham has made significant contributions... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n", - "\\nWhere does Paul Graham live? Paul Graham currently lives in England, where ... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n", - "\n", - "Model Name gpt-4-0613 \\\n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is a writer. He is an essayis... \n", - "\\nWhat has Paul Graham done? Paul Graham is known for his work on the progr... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for his work on the progr... \n", - "\\nWhere does Paul Graham live? The text does not provide a current place of r... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist, ... \n", - "\n", - "Model Name replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781 \n", - "Prompt \n", - "\\nIs paul graham a writer? Yes, Paul Graham is an author. According to t... \n", - "\\nWhat has Paul Graham done? Paul Graham has had a diverse career in compu... \n", - "\\nWhat is Paul Graham known for? Paul Graham is known for many things, includi... \n", - "\\nWhere does Paul Graham live? Based on the information provided, Paul Graha... \n", - "\\nWho is Paul Graham? Paul Graham is an English computer scientist,... " - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "# Create an empty list to store the row data\n", - "table_data = []\n", - "\n", - "# Iterate through the list and extract the required data\n", - "for item in result:\n", - " prompt = item['prompt'][0]['content'].replace(context, \"\") # clean the prompt for easy comparison\n", - " model = item['response']['model']\n", - " response = item['response']['choices'][0]['message']['content']\n", - " table_data.append([prompt, model, response])\n", - "\n", - "# Create a DataFrame from the table data\n", - "df = pd.DataFrame(table_data, columns=['Prompt', 'Model Name', 'Response'])\n", - "\n", - "# Pivot the DataFrame to get the desired table format\n", - "table = df.pivot(index='Prompt', columns='Model Name', values='Response')\n", - "table" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zOxUM40PINDC" - }, - "source": [ - "# Load Test endpoint\n", - "\n", - "Run 100+ simultaneous queries across multiple providers to see when they fail + impact on latency" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZkQf_wbcIRQ9" - }, - "outputs": [], - "source": [ - "models=[\"gpt-3.5-turbo\", \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"claude-instant-1\"]\n", - "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n", - "prompt = \"Where does Paul Graham live?\"\n", - "final_prompt = context + prompt\n", - "result = load_test_model(models=models, prompt=final_prompt, num_calls=5)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8vSNBFC06aXY" - }, - "source": [ - "## Visualize the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 552 - }, - "id": "SZfiKjLV3-n8", - "outputId": "00f7f589-b3da-43ed-e982-f9420f074b8d" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAIXCAYAAACy1HXAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABn5UlEQVR4nO3dd1QT2d8G8Cf0ojQBEUFRsSv2FXvvvSx2saNi7733ihXELotd7KuIir33sjZUsIuKVGmS+/7hy/yM6K7RYEZ4PufkaO5Mkm/IJHly594ZhRBCgIiIiEiGdLRdABEREdG3MKgQERGRbDGoEBERkWwxqBAREZFsMagQERGRbDGoEBERkWwxqBAREZFsMagQERGRbDGoEBERkWwxqBCR7Dk5OaFLly7aLkNtc+fORd68eaGrq4uSJUtquxyNO3bsGBQKBbZv367tUtSmUCgwadIktW8XGhoKhUKBdevWabwm+joGFVKxfPlyKBQKlC9fXtulyI6TkxMUCoV0MTU1xR9//IENGzZou7TfTuoX3PdcfleHDh3CiBEjUKlSJaxduxYzZszQdkmys27dOul1PnXqVJrlQgg4OjpCoVCgcePGWqiQ5EBP2wWQvPj7+8PJyQkXLlxASEgInJ2dtV2SrJQsWRJDhw4FALx8+RKrVq2Cu7s7EhMT0bNnTy1X9/soXLgw/Pz8VNpGjx6NLFmyYOzYsWnWv3fvHnR0fq/fVUePHoWOjg5Wr14NAwMDbZcja0ZGRti4cSMqV66s0n78+HE8e/YMhoaGWqqM5IBBhSSPHz/GmTNnEBAQAA8PD/j7+2PixIm/tAalUomkpCQYGRn90sf9Xjlz5kTHjh2l6126dEHevHmxcOFCBhU1ZM+eXeXvCACzZs2CtbV1mnYAv+UXVXh4OIyNjTUWUoQQSEhIgLGxsUbuT04aNmyIbdu2YfHixdDT+9/X0saNG1GmTBm8fftWi9WRtv1eP1EoXfn7+8PS0hKNGjVC69at4e/vLy1LTk6GlZUVunbtmuZ20dHRMDIywrBhw6S2xMRETJw4Ec7OzjA0NISjoyNGjBiBxMREldsqFAr069cP/v7+KFq0KAwNDXHw4EEAwLx581CxYkVky5YNxsbGKFOmzFf3hcfHx2PAgAGwtrZG1qxZ0bRpUzx//vyr+6CfP3+Obt26IXv27DA0NETRokWxZs2aH/6b2djYoFChQnj48KFKu1KphJeXF4oWLQojIyNkz54dHh4eeP/+vcp6ly5dQr169WBtbQ1jY2PkyZMH3bp1k5an7g+fN28eFi5ciNy5c8PY2BjVqlXDrVu30tRz9OhRVKlSBaamprCwsECzZs1w584dlXUmTZoEhUKBkJAQdOnSBRYWFjA3N0fXrl3x4cMHlXWDgoJQuXJlWFhYIEuWLChYsCDGjBmjss73vtY/48sxKqm7DE6dOoUBAwbAxsYGFhYW8PDwQFJSEiIjI9G5c2dYWlrC0tISI0aMwJcnitfUa/Q1CoUCa9euRVxcnLRrI3VMw8ePHzF16lTky5cPhoaGcHJywpgxY9L8vZycnNC4cWMEBgaibNmyMDY2xooVK/71cc+fP4/69evD3NwcJiYmqFatGk6fPq2yTlhYGPr27YuCBQvC2NgY2bJlw59//onQ0NA09xcZGYnBgwfDyckJhoaGcHBwQOfOndMEB6VSienTp8PBwQFGRkaoVasWQkJC/rXWz7Vr1w7v3r1DUFCQ1JaUlITt27ejffv2X71NXFwchg4dCkdHRxgaGqJgwYKYN29emtc5MTERgwcPho2NjfT58OzZs6/ep6Y/H0hDBNH/K1SokOjevbsQQogTJ04IAOLChQvS8m7dugkLCwuRmJiocrv169cLAOLixYtCCCFSUlJE3bp1hYmJiRg0aJBYsWKF6Nevn9DT0xPNmjVTuS0AUbhwYWFjYyMmT54sli1bJq5evSqEEMLBwUH07dtXLF26VCxYsED88ccfAoDYt2+fyn24ubkJAKJTp05i2bJlws3NTZQoUUIAEBMnTpTWe/XqlXBwcBCOjo5iypQpwtvbWzRt2lQAEAsXLvzPv0/u3LlFo0aNVNqSk5OFnZ2dyJ49u0p7jx49hJ6enujZs6fw8fERI0eOFKampqJcuXIiKSlJCCHE69evhaWlpShQoICYO3euWLlypRg7dqwoXLiwdD+PHz8WAETx4sWFk5OTmD17tpg8ebKwsrISNjY24tWrV9K6QUFBQk9PTxQoUEDMmTNHTJ48WVhbWwtLS0vx+PFjab2JEycKAKJUqVKiZcuWYvny5aJHjx4CgBgxYoS03q1bt4SBgYEoW7asWLRokfDx8RHDhg0TVatWldZR57X+L0WLFhXVqlX75t/e3d1dur527VoBQJQsWVLUr19fLFu2THTq1El6DpUrVxbt27cXy5cvF40bNxYAxPr169PlNfoaPz8/UaVKFWFoaCj8/PyEn5+fePjwoRBCCHd3dwFAtG7dWixbtkx07txZABDNmzdP85ydnZ2FpaWlGDVqlPDx8RHBwcHffMwjR44IAwMDUaFCBTF//nyxcOFC4eLiIgwMDMT58+el9bZt2yZKlCghJkyYIHx9fcWYMWOEpaWlyJ07t4iLi5PWi4mJEcWKFRO6urqiZ8+ewtvbW0ydOlWUK1dOeo8GBwdL21KZMmXEwoULxaRJk4SJiYn4448//vVv9PnrePHiRVGxYkXRqVMnadmuXbuEjo6OeP78eZr3nlKpFDVr1hQKhUL06NFDLF26VDRp0kQAEIMGDVJ5jI4dOwoAon379mLp0qWiZcuWwsXF5Yc/H1Lfk2vXrv3P50eawaBCQgghLl26JACIoKAgIcSnDwIHBwcxcOBAaZ3AwEABQOzdu1fltg0bNhR58+aVrvv5+QkdHR1x8uRJlfV8fHwEAHH69GmpDYDQ0dERt2/fTlPThw8fVK4nJSWJYsWKiZo1a0ptly9f/uqHU5cuXdJ8EHXv3l3kyJFDvH37VmXdtm3bCnNz8zSP96XcuXOLunXrijdv3og3b96ImzdvSl+Onp6e0nonT54UAIS/v7/K7Q8ePKjSvnPnTpWA9zWpH4rGxsbi2bNnUvv58+cFADF48GCprWTJksLW1la8e/dOart+/brQ0dERnTt3ltpSg0q3bt1UHqtFixYiW7Zs0vWFCxcKAOLNmzffrE+d1/q//EhQqVevnlAqlVJ7hQoVhEKhEL1795baPn78KBwcHFTuW5Ov0be4u7sLU1NTlbZr164JAKJHjx4q7cOGDRMAxNGjR1WeMwBx8ODB/3wspVIp8ufPn+bv8eHDB5EnTx5Rp04dlbYvnT17VgAQGzZskNomTJggAIiAgICvPp4Q/wsqhQsXVvkBs2jRIgFA3Lx581/r/jyoLF26VGTNmlWq788//xQ1atSQ/hafB5Vdu3YJAGLatGkq99e6dWuhUChESEiIEOJ/f+++ffuqrNe+ffsf/nxgUPn1uOuHAHza7ZM9e3bUqFEDwKeu6zZt2mDz5s1ISUkBANSsWRPW1tbYsmWLdLv3798jKCgIbdq0kdq2bduGwoULo1ChQnj79q10qVmzJgAgODhY5bGrVauGIkWKpKnp833x79+/R1RUFKpUqYIrV65I7am7ifr27aty2/79+6tcF0Jgx44daNKkCYQQKnXVq1cPUVFRKvf7LYcOHYKNjQ1sbGxQvHhx+Pn5oWvXrpg7d67K8zc3N0edOnVUHqdMmTLIkiWL9PwtLCwAAPv27UNycvK/Pm7z5s2RM2dO6foff/yB8uXL4++//wbwaWDvtWvX0KVLF1hZWUnrubi4oE6dOtJ6n+vdu7fK9SpVquDdu3eIjo5WqW/37t1QKpVfrUvd11rTunfvrjIzqHz58hBCoHv37lKbrq4uypYti0ePHqnUrenX6Hukvg5DhgxRaU8doL1//36V9jx58qBevXr/eb/Xrl3DgwcP0L59e7x79056PnFxcahVqxZOnDghvYafv6+Sk5Px7t07ODs7w8LCQuU9sGPHDpQoUQItWrRI83hfzsbq2rWrylicKlWqAIDK3/y/uLm5IT4+Hvv27UNMTAz27dv3zd0+f//9N3R1dTFgwACV9qFDh0IIgQMHDkjrAUiz3qBBg1Sua+rzgdJHhgkqJ06cQJMmTWBvbw+FQoFdu3al+2M+f/4cHTt2lMZQFC9eHJcuXUr3x9W0lJQUbN68GTVq1MDjx48REhKCkJAQlC9fHq9fv8aRI0cAAHp6emjVqhV2794t7U8PCAhAcnKySlB58OABbt++LX2hp14KFCgA4NMgw8/lyZPnq3Xt27cPrq6uMDIygpWVFWxsbODt7Y2oqChpnbCwMOjo6KS5jy9nK7158waRkZHw9fVNU1fquJsv6/qa8uXLIygoCAcPHsS8efNgYWGB9+/fq3xIP3jwAFFRUbC1tU3zWLGxsdLjVKtWDa1atcLkyZNhbW2NZs2aYe3atV8d25E/f/40bQUKFJDGFYSFhQEAChYsmGa9woULS19an8uVK5fKdUtLSwCQxmi0adMGlSpVQo8ePZA9e3a0bdsWW7duVQkt6r7WmvblczA3NwcAODo6pmn/fOxJerxG3yN1e/1y+7Szs4OFhYX0Oqb61nvjSw8ePAAAuLu7p3k+q1atQmJiovS+iY+Px4QJE6SxHdbW1rCxsUFkZKTKe+vhw4coVqzYdz3+f21L38PGxga1a9fGxo0bERAQgJSUFLRu3fqr64aFhcHe3h5Zs2ZVaS9cuLC0PPVfHR0d5MuXT2W9L98nmvp8oPSRYWb9xMXFoUSJEujWrRtatmyZ7o/3/v17VKpUCTVq1MCBAwdgY2ODBw8eSG/Q38nRo0fx8uVLbN68GZs3b06z3N/fH3Xr1gUAtG3bFitWrMCBAwfQvHlzbN26FYUKFUKJEiWk9ZVKJYoXL44FCxZ89fG+/BL52iyGkydPomnTpqhatSqWL1+OHDlyQF9fH2vXrsXGjRvVfo6pX64dO3aEu7v7V9dxcXH5z/uxtrZG7dq1AQD16tVDoUKF0LhxYyxatEj6laxUKmFra6syGPlzNjY2ACAdKOvcuXPYu3cvAgMD0a1bN8yfPx/nzp1DlixZ1H6e6tDV1f1qu/j/wYjGxsY4ceIEgoODsX//fhw8eBBbtmxBzZo1cejQIejq6qr9Wmvat57D19rFZ4Mstf0afe/xYb53hk/q9j137txvHlgutdb+/ftj7dq1GDRoECpUqABzc3MoFAq0bdv2mz1n/+W/tqXv1b59e/Ts2ROvXr1CgwYNpB6t9KapzwdKHxkmqDRo0AANGjT45vLExESMHTsWmzZtQmRkJIoVK4bZs2ejevXqP/R4s2fPhqOjI9auXSu1fe+vH7nx9/eHra0tli1blmZZQEAAdu7cCR8fHxgbG6Nq1arIkSMHtmzZgsqVK+Po0aNpjnuRL18+XL9+HbVq1frhA3bt2LEDRkZGCAwMVJma+vnfGwBy584NpVKJx48fq/Q6fDnjIHXEf0pKihQ0NKFRo0aoVq0aZsyYAQ8PD5iamiJfvnw4fPgwKlWq9F1fNK6urnB1dcX06dOxceNGdOjQAZs3b0aPHj2kdVJ/MX/u/v37cHJyAvDp7wB8Ot7Il+7evQtra2uYmpqq/fx0dHRQq1Yt1KpVCwsWLMCMGTMwduxYBAcHo3bt2hp5rbUhPV6j75G6vT548ED69Q8Ar1+/RmRkpPQ6qiu1x8DMzOw/t+/t27fD3d0d8+fPl9oSEhIQGRmZ5j6/NrMsPbVo0QIeHh44d+6cyi7mL+XOnRuHDx9GTEyMSq/K3bt3peWp/yqVSjx8+FClF+XL90l6fT6QZmSYXT//pV+/fjh79iw2b96MGzdu4M8//0T9+vW/+gXwPfbs2YOyZcvizz//hK2tLUqVKoWVK1dquOr0Fx8fj4CAADRu3BitW7dOc+nXrx9iYmKwZ88eAJ++uFq3bo29e/fCz88PHz9+VNntA3za1/z8+fOv/j3i4+PT7IL4Gl1dXSgUCml8DPBpqu6Xu/RS998vX75cpX3JkiVp7q9Vq1bYsWPHVz9837x58581fcvIkSPx7t076fm6ubkhJSUFU6dOTbPux48fpS+E9+/fp/nFmfpr+MtdC7t27cLz58+l6xcuXMD58+elcJ4jRw6ULFkS69evV/nCuXXrFg4dOoSGDRuq/bwiIiLStH1ZnyZea21Ij9foe6S+Dl5eXirtqT1SjRo1Uvs+AaBMmTLIly8f5s2bh9jY2DTLP9++dXV10zynJUuWqLzXAKBVq1a4fv06du7cmeb+1O0p+V5ZsmSBt7c3Jk2ahCZNmnxzvYYNGyIlJQVLly5VaV+4cCEUCoX0vkj9d/HixSrrffn3T8/PB/p5GaZH5d88efIEa9euxZMnT2Bvbw8AGDZsGA4ePPjDh7Z+9OgRvL29MWTIEIwZMwYXL17EgAEDYGBg8M2uQznas2cPYmJi0LRp068ud3V1hY2NDfz9/aVA0qZNGyxZsgQTJ05E8eLFVX4ZAkCnTp2wdetW9O7dG8HBwahUqRJSUlJw9+5dbN26VTouxL9p1KgRFixYgPr166N9+/YIDw/HsmXL4OzsjBs3bkjrlSlTBq1atYKXlxfevXsHV1dXHD9+HPfv3weg2sU+a9YsBAcHo3z58ujZsyeKFCmCiIgIXLlyBYcPH/7qF/P3aNCgAYoVK4YFCxbA09MT1apVg4eHB2bOnIlr166hbt260NfXx4MHD7Bt2zYsWrQIrVu3xvr167F8+XK0aNEC+fLlQ0xMDFauXAkzM7M0wcLZ2RmVK1dGnz59kJiYCC8vL2TLlg0jRoyQ1pk7dy4aNGiAChUqoHv37oiPj8eSJUtgbm7+Q+c0mTJlCk6cOIFGjRohd+7cCA8Px/Lly+Hg4CAdQVQTr7U2pMdr9D1KlCgBd3d3+Pr6IjIyEtWqVcOFCxewfv16NG/eXBrMri4dHR2sWrUKDRo0QNGiRdG1a1fkzJkTz58/R3BwMMzMzLB3714AQOPGjeHn5wdzc3MUKVIEZ8+exeHDh5EtWzaV+xw+fDi2b9+OP//8E926dUOZMmUQERGBPXv2wMfHR2V3ryZ9z+dnkyZNUKNGDYwdOxahoaEoUaIEDh06hN27d2PQoEFSD1PJkiXRrl07LF++HFFRUahYsSKOHDny1WO8pNfnA2mAVuYapTMAYufOndL1ffv2CQDC1NRU5aKnpyfc3NyEEELcuXNHAPjXy8iRI6X71NfXFxUqVFB53P79+wtXV9df8hw1pUmTJsLIyEjl+Alf6tKli9DX15em7SmVSuHo6PjV6YGpkpKSxOzZs0XRokWFoaGhsLS0FGXKlBGTJ08WUVFR0nr4Ymrv51avXi3y588vDA0NRaFChcTatWulqbWfi4uLE56ensLKykpkyZJFNG/eXNy7d08AELNmzVJZ9/Xr18LT01M4OjoKfX19YWdnJ2rVqiV8fX3/82/1teOopFq3bl2aKYu+vr6iTJkywtjYWGTNmlUUL15cjBgxQrx48UIIIcSVK1dEu3btRK5cuYShoaGwtbUVjRs3FpcuXZLuI3Uq5Ny5c8X8+fOFo6OjMDQ0FFWqVBHXr19PU8fhw4dFpUqVhLGxsTAzMxNNmjQR//zzj8o6qX/DL6cdp04VTT3mypEjR0SzZs2Evb29MDAwEPb29qJdu3bi/v37Krf73tf6v/zI9OQvpw1/67l9baqwEJp5jb7lW4+ZnJwsJk+eLPLkySP09fWFo6OjGD16tEhISEjznL+1vX3L1atXRcuWLUW2bNmEoaGhyJ07t3BzcxNHjhyR1nn//r3o2rWrsLa2FlmyZBH16tUTd+/eTfM3FkKId+/eiX79+omcOXMKAwMD4eDgINzd3aXPgtTpydu2bVO53fdO4f3W6/ilr/0tYmJixODBg4W9vb3Q19cX+fPnF3PnzlWZni2EEPHx8WLAgAEiW7ZswtTUVDRp0kQ8ffo0zfRkIb7v84HTk389hRDp1IenRQqFAjt37kTz5s0BAFu2bEGHDh1w+/btNIO+smTJAjs7OyQlJf3nVLps2bJJg+xy586NOnXqYNWqVdJyb29vTJs2TaWLnrTj2rVrKFWqFP766y906NBB2+X8sNDQUOTJkwdz585VOfIvEVFmkSl2/ZQqVQopKSkIDw+X5vd/ycDAAIUKFfru+6xUqVKaAVn379//4cFw9OPi4+PTDIj08vKCjo4OqlatqqWqiIhIEzJMUImNjVXZ7/j48WNcu3YNVlZWKFCgADp06IDOnTtj/vz5KFWqFN68eYMjR47AxcXlhwawDR48GBUrVsSMGTPg5uaGCxcuwNfXF76+vpp8WvQd5syZg8uXL6NGjRrQ09PDgQMHcODAAfTq1Svdp8cSEVE60/a+J01J3Vf65SV1n2tSUpKYMGGCcHJyEvr6+iJHjhyiRYsW4saNGz/8mHv37hXFihWTxlB8zzgH0rxDhw6JSpUqCUtLS6Gvry/y5csnJk2aJJKTk7Vd2k/7fIwKEVFmlCHHqBAREVHGkGmOo0JERES/HwYVIiIikq3fejCtUqnEixcvkDVr1t/q8N1ERESZmRACMTExsLe3h47Ov/eZ/NZB5cWLF5zVQURE9Jt6+vQpHBwc/nWd3zqopJ6M6unTpzAzM9NyNURERPQ9oqOj4ejoqHJSyW/5rYNK6u4eMzMzBhUiIqLfzPcM2+BgWiIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki09bRdARETy5TRqv7ZLIC0LndVIq4/PHhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki3ZBJVZs2ZBoVBg0KBB2i6FiIiIZEIWQeXixYtYsWIFXFxctF0KERERyYjWg0psbCw6dOiAlStXwtLSUtvlEBERkYxoPah4enqiUaNGqF279n+um5iYiOjoaJULERERZVx62nzwzZs348qVK7h48eJ3rT9z5kxMnjw5nasiIiIiudBaj8rTp08xcOBA+Pv7w8jI6LtuM3r0aERFRUmXp0+fpnOVREREpE1a61G5fPkywsPDUbp0aaktJSUFJ06cwNKlS5GYmAhdXV2V2xgaGsLQ0PBXl0pERERaorWgUqtWLdy8eVOlrWvXrihUqBBGjhyZJqQQERFR5qO1oJI1a1YUK1ZMpc3U1BTZsmVL005ERESZk9Zn/RARERF9i1Zn/Xzp2LFj2i6BiIiIZIQ9KkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWz8UVB4+fIhx48ahXbt2CA8PBwAcOHAAt2/f1mhxRERElLmpHVSOHz+O4sWL4/z58wgICEBsbCwA4Pr165g4caLGCyQiIqLMS+2gMmrUKEybNg1BQUEwMDCQ2mvWrIlz585ptDgiIiLK3NQOKjdv3kSLFi3StNva2uLt27caKYqIiIgI+IGgYmFhgZcvX6Zpv3r1KnLmzKmRooiIiIiAHwgqbdu2xciRI/Hq1SsoFAoolUqcPn0aw4YNQ+fOndOjRiIiIsqk1A4qM2bMQKFCheDo6IjY2FgUKVIEVatWRcWKFTFu3Lj0qJGIiIgyKT11b2BgYICVK1di/PjxuHXrFmJjY1GqVCnkz58/PeojIiKiTEztoJIqV65cyJUrlyZrISIiIlKhdlARQmD79u0IDg5GeHg4lEqlyvKAgACNFUdERESZm9pBZdCgQVixYgVq1KiB7NmzQ6FQpEddREREROoHFT8/PwQEBKBhw4bpUQ8RERGRRO1ZP+bm5sibN2961EJERESkQu2gMmnSJEyePBnx8fHpUQ8RERGRRO1dP25ubti0aRNsbW3h5OQEfX19leVXrlzRWHFERESUuakdVNzd3XH58mV07NiRg2mJiIgoXakdVPbv34/AwEBUrlw5PeohIiIikqg9RsXR0RFmZmbpUQsRERGRCrWDyvz58zFixAiEhoamQzlERERE/6P2rp+OHTviw4cPyJcvH0xMTNIMpo2IiNBYcUSZndOo/dougbQsdFYjbZdApFVqBxUvL690KIOIiIgorR+a9UNERET0K3xXUImOjpYG0EZHR//ruhxoS0RERJryXUHF0tISL1++hK2tLSwsLL567BQhBBQKBVJSUjReJBEREWVO3xVUjh49CisrKwBAcHBwuhZERERElOq7gkq1atWQN29eXLx4EdWqVUvvmoiIiIgAqHEcldDQUO7WISIiol9K7QO+aZK3tzdcXFxgZmYGMzMzVKhQAQcOHNBmSURERCQjak1PDgwMhLm5+b+u07Rp0+++PwcHB8yaNQv58+eHEALr169Hs2bNcPXqVRQtWlSd0oiIiCgDUiuo/NcxVNSd9dOkSROV69OnT4e3tzfOnTvHoEJERETqBZVXr17B1tY2XQpJSUnBtm3bEBcXhwoVKnx1ncTERCQmJkrX/+uYLkRERPR7++4xKl87doom3Lx5E1myZIGhoSF69+6NnTt3okiRIl9dd+bMmTA3N5cujo6O6VITERERycN3BxUhRLoUULBgQVy7dg3nz59Hnz594O7ujn/++eer644ePRpRUVHS5enTp+lSExEREcnDd+/6cXd3h7GxscYLMDAwgLOzMwCgTJkyuHjxIhYtWoQVK1akWdfQ0BCGhoYar4GIiIjk6buDytq1a9OzDolSqVQZh0JERESZl9pnT9ak0aNHo0GDBsiVKxdiYmKwceNGHDt2DIGBgdosi4iIiGRCq0ElPDwcnTt3xsuXL2Fubg4XFxcEBgaiTp062iyLiIiIZEKrQWX16tXafHgiIiKSuR8+hH5ISAgCAwMRHx8PIP1mBREREVHmpXZQeffuHWrXro0CBQqgYcOGePnyJQCge/fuGDp0qMYLJCIiosxL7aAyePBg6Onp4cmTJzAxMZHa27Rpg4MHD2q0OCIiIsrc1B6jcujQIQQGBsLBwUGlPX/+/AgLC9NYYURERERq96jExcWp9KSkioiI4MHYiIiISKPUDipVqlTBhg0bpOsKhQJKpRJz5sxBjRo1NFocERERZW5q7/qZM2cOatWqhUuXLiEpKQkjRozA7du3ERERgdOnT6dHjURERJRJqd2jUqxYMdy/fx+VK1dGs2bNEBcXh5YtW+Lq1avIly9fetRIREREmdQPHfDN3NwcY8eO1XQtRERERCrU7lE5ePAgTp06JV1ftmwZSpYsifbt2+P9+/caLY6IiIgyN7WDyvDhwxEdHQ0AuHnzJoYMGYKGDRvi8ePHGDJkiMYLJCIiosxL7V0/jx8/RpEiRQAAO3bsQJMmTTBjxgxcuXIFDRs21HiBRERElHmp3aNiYGCADx8+AAAOHz6MunXrAgCsrKyknhYiIiIiTVC7R6Vy5coYMmQIKlWqhAsXLmDLli0AgPv376c5Wi0RERHRz1C7R2Xp0qXQ09PD9u3b4e3tjZw5cwIADhw4gPr162u8QCIiIsq81O5RyZUrF/bt25emfeHChRopiIiIiCjVDx1HRalUIiQkBOHh4VAqlSrLqlatqpHCiIiIiNQOKufOnUP79u0RFhYGIYTKMoVCgZSUFI0VR0RERJmb2kGld+/eKFu2LPbv348cOXJAoVCkR11ERERE6geVBw8eYPv27XB2dk6PeoiIiIgkas/6KV++PEJCQtKjFiIiIiIVaveo9O/fH0OHDsWrV69QvHhx6Ovrqyx3cXHRWHFERESUuakdVFq1agUA6Natm9SmUCgghOBgWiIiItKoHzrXDxEREdGvoHZQyZ07d3rUQURERJTGDx3w7eHDh/Dy8sKdO3cAAEWKFMHAgQORL18+jRZHREREmZvaQSUwMBBNmzZFyZIlUalSJQDA6dOnUbRoUezduxd16tTReJHa4jRqv7ZLIC0LndVI2yUQEWVqageVUaNGYfDgwZg1a1aa9pEjR2aooEJERETapfZxVO7cuYPu3bunae/WrRv++ecfjRRFREREBPxAULGxscG1a9fStF+7dg22traaqImIiIgIwA/s+unZsyd69eqFR48eoWLFigA+jVGZPXs2hgwZovECiYiIKPNSO6iMHz8eWbNmxfz58zF69GgAgL29PSZNmoQBAwZovEAiIiLKvNQOKgqFAoMHD8bgwYMRExMDAMiaNavGCyMiIiL6oeOoAEB4eDju3bsHAChUqBBsbGw0VhQRERER8AODaWNiYtCpUyfY29ujWrVqqFatGuzt7dGxY0dERUWlR41ERESUSakdVHr06IHz589j//79iIyMRGRkJPbt24dLly7Bw8MjPWokIiKiTErtXT/79u1DYGAgKleuLLXVq1cPK1euRP369TVaHBEREWVuaveoZMuWDebm5mnazc3NYWlpqZGiiIiIiIAfCCrjxo3DkCFD8OrVK6nt1atXGD58OMaPH6/R4oiIiChzU3vXj7e3N0JCQpArVy7kypULAPDkyRMYGhrizZs3WLFihbTulStXNFcpERERZTpqB5XmzZunQxlEREREaakdVCZOnJgedRARERGlofYYladPn+LZs2fS9QsXLmDQoEHw9fXVaGFEREREageV9u3bIzg4GMCnQbS1a9fGhQsXMHbsWEyZMkXjBRIREVHmpXZQuXXrFv744w8AwNatW1G8eHGcOXMG/v7+WLdunabrIyIiokxM7aCSnJwMQ0NDAMDhw4fRtGlTAJ/O9/Py5UvNVkdERESZmtpBpWjRovDx8cHJkycRFBQkHY32xYsXyJYtm8YLJCIiosxL7aAye/ZsrFixAtWrV0e7du1QokQJAMCePXukXUJEREREmqD29OTq1avj7du3iI6OVjlkfq9evWBiYqLR4oiIiChzU7tHBQCEELh8+TJWrFiBmJgYAICBgQGDChEREWmU2j0qYWFhqF+/Pp48eYLExETUqVMHWbNmxezZs5GYmAgfH5/0qJOIiIgyIbV7VAYOHIiyZcvi/fv3MDY2ltpbtGiBI0eOaLQ4IiIiytzU7lE5efIkzpw5AwMDA5V2JycnPH/+XGOFEREREando6JUKpGSkpKm/dmzZ8iaNatGiiIiIiICfiCo1K1bF15eXtJ1hUKB2NhYTJw4EQ0bNtRkbURERJTJqb3rZ/78+ahXrx6KFCmChIQEtG/fHg8ePIC1tTU2bdqUHjUSERFRJqV2UHFwcMD169exZcsWXL9+HbGxsejevTs6dOigMriWiIiI6GepHVQAQE9PDx06dECHDh2ktpcvX2L48OFYunSpxoojIiKizE2toHL79m0EBwfDwMAAbm5usLCwwNu3bzF9+nT4+Pggb9686VUnERERZULfPZh2z549KFWqFAYMGIDevXujbNmyCA4ORuHChXHnzh3s3LkTt2/fTs9aiYiIKJP57qAybdo0eHp6Ijo6GgsWLMCjR48wYMAA/P333zh48KB0FmUiIiIiTfnuoHLv3j14enoiS5Ys6N+/P3R0dLBw4UKUK1cuPesjIiKiTOy7g0pMTAzMzMwAALq6ujA2NuaYFCIiIkpXag2mDQwMhLm5OYBPR6g9cuQIbt26pbJO06ZNNVcdERERZWpqBRV3d3eV6x4eHirXFQrFVw+vT0RERPQjvjuoKJXK9KyDiIiIKA21z/VDRERE9KtoNajMnDkT5cqVQ9asWWFra4vmzZvj3r172iyJiIiIZESrQeX48ePw9PTEuXPnEBQUhOTkZNStWxdxcXHaLIuIiIhk4ofO9aMpBw8eVLm+bt062Nra4vLly6hataqWqiIiIiK50GpQ+VJUVBQAwMrK6qvLExMTkZiYKF2Pjo7+JXURERGRdvzQrp/IyEisWrUKo0ePRkREBADgypUreP78+Q8XolQqMWjQIFSqVAnFihX76jozZ86Eubm5dHF0dPzhxyMiIiL5Uzuo3LhxAwUKFMDs2bMxb948REZGAgACAgIwevToHy7E09MTt27dwubNm7+5zujRoxEVFSVdnj59+sOPR0RERPKndlAZMmQIunTpggcPHsDIyEhqb9iwIU6cOPFDRfTr1w/79u1DcHAwHBwcvrmeoaEhzMzMVC5ERESUcak9RuXixYtYsWJFmvacOXPi1atXat2XEAL9+/fHzp07cezYMeTJk0fdcoiIiCgDUzuoGBoafnUQ6/3792FjY6PWfXl6emLjxo3YvXs3smbNKgUdc3NzGBsbq1saERERZTBq7/pp2rQppkyZguTkZACfzu/z5MkTjBw5Eq1atVLrvry9vREVFYXq1asjR44c0mXLli3qlkVEREQZkNpBZf78+YiNjYWtrS3i4+NRrVo1ODs7I2vWrJg+fbpa9yWE+OqlS5cu6pZFREREGZDau37Mzc0RFBSEU6dO4caNG4iNjUXp0qVRu3bt9KiPiIiIMrEfPuBb5cqVUblyZU3WQkRERKRC7aCyePHir7YrFAoYGRnB2dkZVatWha6u7k8XR0RERJmb2kFl4cKFePPmDT58+ABLS0sAwPv372FiYoIsWbIgPDwcefPmRXBwMI8cS0RERD9F7cG0M2bMQLly5fDgwQO8e/cO7969w/3791G+fHksWrQIT548gZ2dHQYPHpwe9RIREVEmonaPyrhx47Bjxw7ky5dPanN2dsa8efPQqlUrPHr0CHPmzFF7qjIRERHRl9TuUXn58iU+fvyYpv3jx4/SAdvs7e0RExPz89URERFRpqZ2UKlRowY8PDxw9epVqe3q1avo06cPatasCQC4efMmD4dPREREP03toLJ69WpYWVmhTJkyMDQ0hKGhIcqWLQsrKyusXr0aAJAlSxbMnz9f48USERFR5qL2GBU7OzsEBQXh7t27uH//PgCgYMGCKFiwoLROjRo1NFchERERZVo/fMC3QoUKoVChQpqshYiIiEjFDwWVZ8+eYc+ePXjy5AmSkpJUli1YsEAjhRERERGpHVSOHDmCpk2bIm/evLh79y6KFSuG0NBQCCFQunTp9KiRiIiIMim1B9OOHj0aw4YNw82bN2FkZIQdO3bg6dOnqFatGv7888/0qJGIiIgyKbWDyp07d9C5c2cAgJ6eHuLj45ElSxZMmTIFs2fP1niBRERElHmpHVRMTU2lcSk5cuTAw4cPpWVv377VXGVERESU6ak9RsXV1RWnTp1C4cKF0bBhQwwdOhQ3b95EQEAAXF1d06NGIiIiyqTUDioLFixAbGwsAGDy5MmIjY3Fli1bkD9/fs74ISIiIo1SK6ikpKTg2bNncHFxAfBpN5CPj0+6FEZERESk1hgVXV1d1K1bF+/fv0+veoiIiIgkag+mLVasGB49epQetRARERGpUDuoTJs2DcOGDcO+ffvw8uVLREdHq1yIiIiINEXtwbQNGzYEADRt2hQKhUJqF0JAoVAgJSVFc9URERFRpqZ2UAkODk6POoiIiIjSUDuoVKtWLT3qICIiIkpD7TEqAHDy5El07NgRFStWxPPnzwEAfn5+OHXqlEaLIyIiosxN7aCyY8cO1KtXD8bGxrhy5QoSExMBAFFRUZgxY4bGCyQiIqLM64dm/fj4+GDlypXQ19eX2itVqoQrV65otDgiIiLK3NQOKvfu3UPVqlXTtJubmyMyMlITNREREREB+IGgYmdnh5CQkDTtp06dQt68eTVSFBERERHwA0GlZ8+eGDhwIM6fPw+FQoEXL17A398fw4YNQ58+fdKjRiIiIsqk1J6ePGrUKCiVStSqVQsfPnxA1apVYWhoiGHDhqF///7pUSMRERFlUmoHFYVCgbFjx2L48OEICQlBbGwsihQpgixZsqRHfURERJSJqb3r56+//sKHDx9gYGCAIkWK4I8//mBIISIionShdlAZPHgwbG1t0b59e/z99988tw8RERGlG7WDysuXL7F582YoFAq4ubkhR44c8PT0xJkzZ9KjPiIiIsrE1A4qenp6aNy4Mfz9/REeHo6FCxciNDQUNWrUQL58+dKjRiIiIsqk1B5M+zkTExPUq1cP79+/R1hYGO7cuaOpuoiIiIh+7KSEHz58gL+/Pxo2bIicOXPCy8sLLVq0wO3btzVdHxEREWViaveotG3bFvv27YOJiQnc3Nwwfvx4VKhQIT1qIyIiokxO7aCiq6uLrVu3ol69etDV1VVZduvWLRQrVkxjxREREVHmpnZQ8ff3V7keExODTZs2YdWqVbh8+TKnKxMREZHG/NAYFQA4ceIE3N3dkSNHDsybNw81a9bEuXPnNFkbERERZXJq9ai8evUK69atw+rVqxEdHQ03NzckJiZi165dKFKkSHrVSERERJnUd/eoNGnSBAULFsSNGzfg5eWFFy9eYMmSJelZGxEREWVy392jcuDAAQwYMAB9+vRB/vz507MmIiIiIgBq9KicOnUKMTExKFOmDMqXL4+lS5fi7du36VkbERERZXLfHVRcXV2xcuVKvHz5Eh4eHti8eTPs7e2hVCoRFBSEmJiY9KyTiIiIMiG1Z/2YmpqiW7duOHXqFG7evImhQ4di1qxZsLW1RdOmTdOjRiIiIsqkfnh6MgAULFgQc+bMwbNnz7Bp0yZN1UREREQE4CeDSipdXV00b94ce/bs0cTdEREREQHQUFAhIiIiSg8MKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbDCpEREQkWwwqREREJFsMKkRERCRbWg0qJ06cQJMmTWBvbw+FQoFdu3ZpsxwiIiKSGa0Glbi4OJQoUQLLli3TZhlEREQkU3rafPAGDRqgQYMG2iyBiIiIZEyrQUVdiYmJSExMlK5HR0drsRoiIiJKb7/VYNqZM2fC3Nxcujg6Omq7JCIiIkpHv1VQGT16NKKioqTL06dPtV0SERERpaPfatePoaEhDA0NtV0GERER/SK/VY8KERERZS5a7VGJjY1FSEiIdP3x48e4du0arKyskCtXLi1WRkRERHKg1aBy6dIl1KhRQ7o+ZMgQAIC7uzvWrVunpaqIiIhILrQaVKpXrw4hhDZLICIiIhnjGBUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSIiIpItBhUiIiKSLVkElWXLlsHJyQlGRkYoX748Lly4oO2SiIiISAa0HlS2bNmCIUOGYOLEibhy5QpKlCiBevXqITw8XNulERERkZZpPagsWLAAPXv2RNeuXVGkSBH4+PjAxMQEa9as0XZpREREpGVaDSpJSUm4fPkyateuLbXp6Oigdu3aOHv2rBYrIyIiIjnQ0+aDv337FikpKciePbtKe/bs2XH37t006ycmJiIxMVG6HhUVBQCIjo5Ol/qUiR/S5X7p95Fe29b34jZI3AZJ29JjG0y9TyHEf66r1aCirpkzZ2Ly5Mlp2h0dHbVQDWUG5l7aroAyO26DpG3puQ3GxMTA3Nz8X9fRalCxtraGrq4uXr9+rdL++vVr2NnZpVl/9OjRGDJkiHRdqVQiIiIC2bJlg0KhSPd6M5Po6Gg4Ojri6dOnMDMz03Y5lAlxGyRt4zaYfoQQiImJgb29/X+uq9WgYmBggDJlyuDIkSNo3rw5gE/h48iRI+jXr1+a9Q0NDWFoaKjSZmFh8QsqzbzMzMz4BiWt4jZI2sZtMH38V09KKq3v+hkyZAjc3d1RtmxZ/PHHH/Dy8kJcXBy6du2q7dKIiIhIy7QeVNq0aYM3b95gwoQJePXqFUqWLImDBw+mGWBLREREmY/WgwoA9OvX76u7ekh7DA0NMXHixDS72oh+FW6DpG3cBuVBIb5nbhARERGRFmj9yLRERERE38KgQkRERLLFoEJERESyxaBCREREssWgQkRERLLFoEJERESyxaBCREREssWgQkRERLLFoEJERESyxaBCvyWlUqntEoiI6BdgUKHfko7Op0337du3AACeCYJ+tS/DMrdB0oYvt8OM+COOQYV+W4sWLULz5s3x8OFDKBQKbZdDmYyOjg6ioqIQGBgIANwGSSt0dHQQGRmJuXPn4v3799KPuIwk4z0jyrC+/MWqr68PY2NjGBgYaKkiysyUSiXmz58PDw8P7Nu3T9vlUCZ26NAhLFiwAEuXLtV2KemCZ0+m3050dDTMzMwAAFFRUTA3N9dyRZRZKJVKlV+sd+7cwerVqzF79mzo6upqsTLKTFJSUlS2t+TkZGzZsgXt2rXLkNshgwr9VgYPHoyUlBSMHj0aOXLk0HY5lAlFRkYiMjISjo6OKl8KX355EP2ML0Pxl969e4fTp0+jYsWKsLa2ltoz4nbIXT8ka1/maAcHB2zYsCHDvRHp9yCEwKhRo1C+fHmEhoaqLOM2ST/j5cuXePHiBd68eQPg09iTf+tH2Lp1K5o3b47jx4+rtGfE7ZA9KiQbqb8EhBBQKBTf/EXx/v17WFpaaqFCymj+61fr19YJCwvDuHHjsG7dugz5pUC/3tq1a7Fs2TI8ffoU+fLlQ+XKlTFnzhyVdb7WU+Ll5YV+/fpBT0/vV5b7yzGokFakhhHg0xtQCAE9PT08f/4cO3fuRNeuXWFqagrg0+4eS0tLTJgwIc1tiX7U5wHk6NGjePLkCZydnZE3b17Y29urrBMVFQWlUpkmIGfEbnb6tfbt2wc3NzcsX74cJiYmePToEebMmYOKFSti/fr1yJYtm/SZ9/btW4SEhMDV1VXlPj5+/Jihwwp3/dAvkZqHo6OjER8fD4VCgUOHDiEkJAS6urrQ09NDWFgYSpUqhRcvXkghJS4uDvr6+li4cCEiIiIYUkgjhBBSSBk1ahS6dOmCefPmoVevXhg2bBguXrwI4FP3e2JiIiZMmIDSpUvj3bt3KvfDkEI/6+LFi2jUqBG6dOkCNzc3jBgxAoGBgbhx4wY6dOgA4NPU9+TkZPj5+aFixYo4deqUyn1k5JACMKjQL/Tq1SsUL14cx48fx8aNG1G/fn38888/AD7tzilatChatGiB6dOnS7cxNTXFiBEj8ODBA1hZWTGkkEakbkfz5s3DX3/9hU2bNuHWrVto2bIl9u7di3HjxuHs2bMAAAMDA5QqVQq1atWChYWFFqumjOjx48d4+fKlSlu5cuWwZ88eXL58GT179gTw6XAMjRs3xvTp09P0qGR4gugX6tq1qzAzMxM6Ojpi5cqVUntSUpLYsmWLSElJkdqUSqU2SqRM4vXr16Jly5ZizZo1Qggh9uzZI8zMzETv3r1FqVKlRK1atcS5c+eEEKrb4sePH7VSL2VMgYGBInv27GLz5s1SW+r25u/vL5ydncXFixfT3C45OfmX1aht7FGhXyL1sM6enp6IiYmBgYEB7OzskJCQAODTrwU3NzeVQYvsPaH0ZGtrixEjRqB+/fq4evUqPD09MW3aNHh7e6NVq1Y4d+4cPD09cfnyZZVtkbt7SJMKFy6M6tWrw8/PD0eOHAHwv8++kiVLIjw8XDpVyOcy+u6ezzGo0C+RGkAcHR1x6tQpuLu7o23btti9ezfi4+PTrJ8Rz1dB2vOt7alUqVLIkSMHDhw4ABcXF/Tq1QsAYGVlBVdXVzRp0gSlSpX6laVSJuPo6IjevXsjMjISCxcuxJ49e6RlOXLkQJ48ebRYnTxknkhGWiH+f/Dry5cvkZycjFy5csHW1hYVK1ZEQkICunfvjnXr1qFx48YwMjKCj48PateuDWdnZ22XThmE+Gzg7KpVqxAeHg4DAwMMGzZMOv1CYmIinj9/jtDQUBQsWBCHDh1C06ZN0b9//3+dKk/0M1JnjVWvXh3Lly/HmDFjMHLkSAQGBsLFxQVbt26FQqFAnTp1tF2qVnF6MqW7gIAATJo0Ca9fv0ajRo3QokULNGnSBADQtWtX7Ny5E0OHDsXr16/h7e2NmzdvokiRIlqumjKaiRMnwsvLC+XKlcOFCxdQvnx5+Pn5wc7ODnv37sW0adPw/v176OvrQwiBGzduQE9PjzPNKF2kblcBAQFYvnw5Dh06hLt37yI4OBhLly6Fo6MjLCws4O/vD319/Uw9FZ5BhdLV7du3Ua9ePQwePBgmJibYtGkTDA0N4e7ujo4dOwIABg4ciCtXriAxMRG+vr4oWbKkdoumDOHzXpCPHz/C3d0d/fv3R6lSpRAaGopGjRrBzs4OO3fuhI2NDfbv34+QkBDExsZi5MiR0NPTy9RfDqQZqYFEfHHsKF1dXQQEBKBz585YsGCBtNsR+LS96ujoqGy/mWlMypcYVCjd3L17F9u2bUN8fDxmzJgBALh58yYmTJiA6OhodO3aVQorr169gqmpKbJmzarNkimD+Dyk3LlzB9HR0VixYgUmTJgAJycnAJ+mhdapUwfZs2fHrl27YGNjo3IfDCn0sz7fDt++fQuFQoFs2bIB+PSZV7p0aUyYMAG9e/eWbvNlDx579BhUKB0IIfD+/Xs0btwY//zzD5o0aQI/Pz9p+Y0bNzBhwgTEx8ejbdu26Nq1qxarpYxs+PDhUtf569evERAQgAYNGkgf/I8fP0aDBg0ghMDp06dVTu5G9DM+DxhTp07Frl27EB0dDWtra0yfPh01a9bE8+fPkTNnTi1XKn8cHUYap1AoYGVlhZkzZ6Jo0aK4cuUKgoKCpOUuLi6YOnUqkpOTpTcvkSZ8Prtn3759OHjwIBYvXozly5cjT548GDt2LK5fvy4dKTlPnjzYt28fSpYsyfNHkUalhpQpU6Zg0aJF0vR3a2trdOjQAevXr0/Ti0dfxx4V0ohvdU8eP34cY8aMgZ2dHTw9PVGzZk1p2e3bt2Fubg4HB4dfWSplAgEBAThz5gyyZcuG0aNHAwBiY2NRunRpmJmZYdWqVShRokSabZa7e0iT3r17h7p168LT0xPdunWT2nv16oW9e/ciODgYhQoV4u6d/8AeFfppqW+yM2fOYMGCBRg/fjxOnz6N5ORkVKtWDVOmTMGrV6+wdOlSHDt2TLpd0aJFGVJI4+Lj4zF+/HgsWLAAt2/fltqzZMmCK1euICYmBh4eHtL5fD7HkEKa9PHjR7x9+1bqrUs9wKWvry/s7e2xcOFCADy45X9hUKGf8vkUuwYNGuD06dPYs2cPxowZg+nTpyMpKQm1atXClClT8O7dO0ydOhUnT57UdtmUgRkbG+PkyZOoXbs2Ll++jD179iAlJQXA/8LK3bt3sWLFCi1XShnJ13ZOZM+eHXZ2dlizZg0AwMjICElJSQAAZ2dnBpTvxKBCPyW1J2XAgAFYsGABduzYgW3btuHy5cvYsmULxo0bJ4WVUaNGQV9fn0daJI35fEyKEEL6srCyssLGjRthaWmJuXPnIjAwUFpmamqKV69ewdfXVys1U8ajVCql0PHixQuEh4fjw4cPAIBJkybh7t270sye1IMMPnv2jCe5/E4co0I/JPWNqVAosHz5cly7dg2+vr54/PgxateujcqVK8PMzAzbtm2Dh4cHxowZA0NDQ3z48AEmJibaLp8ygM+nfi5ZsgTXr1/Ho0ePMGjQIJQuXRoODg548+YNmjVrBl1dXYwZMwb16tVTOcIsx6TQz/D394erqyvy5csHABg9ejQCAwMRFhaG2rVro2nTpujQoQNWrlyJqVOnIlu2bChWrBgePnyIyMhI6aCC9O8YVOi7pH4pfB40rl27hpIlSyI6OhpPnz6Fs7Mz6tevjzx58mDNmjWIioqSjjDbpUsXTJ8+nYPG6Kd9uQ2NHj0aq1evRq9evfDs2TOcPXsWzZo1Q69eveDs7Iw3b96gZcuWePPmDdatWwdXV1ctVk8ZxYEDB9C4cWOMHDkSgwYNwoEDBzBixAh4eXnh3bt3uHLlCgIDAzF+/Hj07t0bN2/ehJeXF3R0dGBpaYkZM2bwoILfK13PzUwZyqNHj0S7du3EP//8I7Zu3SoUCoW4cOGCdErymzdvikKFConz588LIYR4+PChaNy4sRgzZox48uSJNkunDCYlJUUIIYSfn5/IkyePuHz5shBCiJMnTwqFQiHy588vBg4cKB49eiSEEOLly5eiV69e4uPHj1qrmTKepUuXCgcHBzF16lTRr18/sXLlSmnZ06dPxZQpU4STk5M4ePDgV2+fnJz8q0r9rbHPib5bQkICTp48iS5duuDatWtYu3YtypUrJ+0GEkLg48ePOHv2LIoWLYoNGzYAAIYNG8ZjVNBP69SpE2xsbLBgwQLo6OggOTkZBgYG6N27N0qXLo1du3aha9euWLVqFV69eoVp06ZBR0cHPXv2ROHChaXBs/wFSz8rKSkJBgYG8PT0hImJCUaPHo2YmBhMmzZNWsfBwQGdO3fGoUOHcOnSJdSrVy/NyS252+c7aTsp0e8h9Resj4+P0NHRESVKlBBXr15VWScqKkp06dJF5MuXTzg5OQkbGxvply7Rz4iKihKTJ08WVlZWYtKkSVL78+fPxevXr8XLly9F2bJlxfz586X17e3tRY4cOcSiRYuEEELq+SPSlJkzZ4rw8HDh7+8vTExMRMOGDcX9+/dV1mnTpo1o2bKllirMGDjrh/6TEAI6OjoQQsDe3h7z58/Hx48fMW7cOJw6dUpaz8zMDPPmzcPy5csxceJEnD9/HqVLl9Zi5ZQRxMTEwMzMDH369MG4cePg5eWFiRMnAgDs7e1ha2uLly9f4v3799L4k+fPn6Nu3bqYMGECPD09AfBYFfTzxGdDOtevX4+pU6fiwYMHaN++PRYuXIgrV67Ax8cH9+7dAwBER0fj8ePHyJUrl7ZKzhDY70T/Svz/wMWjR4/i+PHjGDRoEJo0aYLatWvDzc0Ns2bNwpgxY1CxYkUAn046WLduXS1XTRnFiBEjsGLFCjx8+BA2Njbo2LEjhBCYOnUqAGDy5MkAPoUZXV1dnD59GkIIzJo1CyYmJtKUUO7uIU1IDbtHjhzB1atX4evrK3329erVC8nJyZg8eTIOHjyI0qVLIy4uDklJSZgzZ442y/79abM7h+Qttat8+/btwtzcXIwePVpcvHhRWn7jxg1RpEgR0bhxY/HXX3+JSZMmCYVCIZ4+fcpudtKI69evi6pVq4qCBQuKN2/eCCGECA8PF/PnzxcWFhZiwoQJ0rr9+vUT+fLlEw4ODsLV1VUkJSUJIbjLhzTr2LFjonjx4iJbtmxi165dQgghEhMTpeWrV68WWbJkEaVLlxYbNmyQBnBz4OyP4/Rk+lcXLlxA/fr1MXv2bPTs2VNqj46OhpmZGe7cuYOePXsiPj4eUVFR2Lp1K3f3kEacPXsWb968QZEiRdCmTRvExsZKZzh+8+YN/Pz8MHXqVOlkb8CnKfMKhQLFixeHjo4OPn78yAGL9FPEF9PhY2NjMXfuXPj6+qJ8+fLYtGkTjI2NkZycDH19fQDAggULcObMGWzbtg0KhYI9ej+JQYX+1dKlS7Fz504cOXIEUVFROHr0KP766y/cuXMHw4YNQ7du3RAeHo6oqCiYm5vD1tZW2yVTBtG5c2e8ePEChw8fRmhoKFq3bo2YmJg0YWXatGno168fpkyZonJ7fjmQJi1btgwODg5o1qwZ4uPjMW/ePOzcuRPVq1fHjBkzYGRkpBJWUgPOl0GH1MfBtPSv7OzscPnyZcycOROtW7fG2rVrYWRkhEaNGqFHjx64f/8+bG1tkT9/foYU0qhly5bh2bNnWLp0KZycnLBp0yaYm5ujUqVKePv2LWxsbNCpUydMmDAB06ZNw+rVq1Vuz5BCmvLmzRscPXoUffv2xcGDB2FsbIwhQ4agcePGOHPmDMaOHYuEhATo6+vj48ePAMCQokHsUSFJ6psqNjYWWbJkAQC8fv0aS5YswdatW1GzZk106dIFf/zxB16/fo2mTZti3bp1KFq0qJYrp4wmtTdk8eLFuHr1KhYsWABLS0vcvXsXnTt3RlRUlNSz8urVKxw/fhytWrXibh7SiC+PdwIA169fx+LFi3H48GH4+PigQYMGiIuLw5w5c3D48GEULlwYy5cvl87lQ5rDHhWSKBQK7N+/H+3atUP16tWxbt066OnpYdq0aTh//jx8fHzg6uoKHR0dLFmyBHFxcexFoXSR2htSvXp1nDhxAvv37wcAFCxYEH5+frC0tETVqlXx+vVr2NnZoU2bNtDT05N+zRL9jNSQ8urVK6mtRIkSGDhwIGrUqIHevXvj4MGDMDU1xYgRI/DHH39AR0dH2u1DGqalQbwkQ6dPnxZGRkZi+PDhon79+sLFxUV4eHiIkJAQaZ3g4GDRq1cvYWVlleaAb0Q/KvWAgl/j4+MjChQoIO7duye13bt3Tzg5OYm2bdv+ivIok/h8O9y8ebPImzevykxHIYS4du2aaNasmciVK5c4duyYEEKI+Ph4aXbZv23L9GPYo0IAgLCwMAQFBWH69OmYM2cODhw4gF69euHGjRuYOXMmHj16hLi4OJw9exbh4eE4fvw4SpYsqe2yKQP4vJv9woULOHPmDI4fPy4tb9q0KcqXL4/g4GCprUCBAjhx4gT++uuvX14vZUyJiYnSdpiUlIR8+fKhUKFC8PT0xOXLl6X1SpQogebNm+Pp06eoW7cuzpw5AyMjI2lMype7jOjn8S+aCS1duhR///23dP3evXto06YN1qxZAyMjI6nd09MTHTp0wO3btzFnzhxERkZi+PDhWL9+PYoVK6aN0imD+fyDfcyYMejSpQu6desGd3d3tGnTBtHR0ciRI4e0/z85OVm6raOjI3R1dZGSkqKt8imDOHDgAPz8/AAAPXv2RM2aNVG2bFkMHToUdnZ28PDwwKVLl6T1c+XKhbZt22L+/PkoX7681M6Bs+lE21069Gs9fvxYtG/fXjx48EClfdSoUcLW1la0bNlSOrBWKm9vb1GwYEExYMAAHrSI0sW8efNEtmzZxPnz50VKSoqYMWOGUCgU4tSpU9I6lSpVEh4eHlqskjKqdu3aCScnJ1GvXj1hbW0trl+/Li07evSoaN68uShWrJg4cOCAePz4sWjevLkYOnSotA7Pyp2+GFQyobi4OCGEEOfOnRPbt2+X2idMmCCKFy8uxo0bJ16/fq1ym5UrV4rHjx//yjIpk1AqlcLd3V34+voKIYTYsWOHsLCwED4+PkIIIWJiYoQQQhw4cEA0bdpU3LhxQ2u1UsZVsmRJoVAoVE56merkyZOiU6dOQqFQiAIFCggXFxfpRxuPfJz+OJcvEzI2NkZkZCRmzpyJ58+fQ1dXF82bN8fkyZORnJyM/fv3QwiBgQMHwsbGBgDQo0cPLVdNGVVCQgLOnz+P6tWr49ixY3B3d8fcuXPh4eGBjx8/Ys6cOahQoQJcXV0xZcoUXLhwAcWLF9d22ZRBJCUlISEhAc7OzsiVKxe2bNmCnDlzom3bttJhGipXrozy5cujZ8+eSE5ORrVq1aCrq8sjH/8iHKOSCSkUClhYWGDo0KHIkycPvLy8EBAQAACYMWMG6tevj6CgIMyYMQNv377VcrWUkdy4cQPPnj0DAAwePBjHjx+HsbEx2rdvj7/++gsNGzbEwoULpZMJvn//HpcuXcK9e/dgaWkJPz8/5M6dW5tPgTIYAwMDmJmZYdu2bdi9ezfKlSuHOXPmYPPmzYiJiZHWS0hIQJUqVVCzZk1pbBRDyq/BoJIJiU+7/FClShUMHjwYlpaWWLx4sUpYcXV1xdWrV1VOa070o4QQuH//PmrUqIE1a9agd+/eWLRoESwtLQEArq6uCAsLQ/ny5VGhQgUAwIsXL9ClSxdERkaiX79+AIB8+fKhdu3aWnselPEIIaBUKqXr69evR8WKFbFw4UJs2LABT548Qc2aNfHnn39K6wM88vGvxCPTZkKpR/2MioqCiYkJbty4genTp+P9+/cYOHAgmjdvDuDTYaNTd/0QacLKlSsxYsQIJCQkYPfu3ahbt650ROQtW7ZgypQpEEJAT08PxsbGUCqVOHPmDPT19XnuHvppERERsLKyUmlL3f62bduGoKAg+Pr6AgB69eqFY8eOISUlBVZWVjh9+jSPOqsl7FHJZD5+/AhdXV2EhoaievXqOHToEMqUKYNhw4bBxsYGkydPxr59+wCAIYU0JvUXq6OjIwwNDWFmZoZz584hNDRUmtLZpk0bbNiwAVOmTIGbmxtGjhyJc+fOSedPYUihn7Fo0SKUK1dOZXcOACmkdOnSBSVKlJDafX19sWLFCixZsgTnzp2DgYEBj3ysLdoZw0u/wrdGo4eEhIjs2bOLHj16qEyrO3bsmOjUqZMIDQ39VSVSBvflNpiUlCTi4+OFt7e3yJkzpxgzZsx/bm+c+kk/a8WKFcLQ0FBs3LgxzbInT56I4sWLi6VLl0ptX9vmuB1qD3f9ZFDi/7szz549izt37iAkJASdO3dGjhw5sH79ely6dAnr169Pc4bPhIQElYO+Ef2oz484GxERgZiYGJWBsF5eXpg3bx66d++Orl27wsnJCU2aNMHYsWPh6uqqrbIpg1m5ciX69+8PPz8//Pnnn4iMjERcXBwSEhJga2uLrFmz4sGDB8ifP7+2S6VvYFDJwHbs2IFevXpJJ2978+YN2rRpg5EjRyJr1qzaLo8ysM9DypQpU3Do0CHcunULbm5uaNGiBRo0aADgU1jx8vJCsWLF8O7dOzx58gShoaE8uRtpxKNHj+Ds7Aw3Nzds3rwZt27dQt++ffHmzRuEhYWhRo0a6NOnDxo3bqztUulfcG5VBnXr1i0MHjwY8+fPR5cuXRAdHQ0LCwsYGxszpFC6Sw0pEyZMgK+vL+bOnQsnJyf07t0bDx48QGRkJNq1a4dBgwbB2toa169fR0JCAk6ePCmdBZlTP+ln2djYYPbs2ZgwYQKGDRuGQ4cOoUqVKmjWrBmio6Oxfft2jBs3DtbW1uzFkzNt7ncizTh69Kh4+PBhmrYKFSoIIYS4c+eOyJ07t+jRo4e0/OHDh9znSunq6NGjomjRouLEiRNCCCHOnDkjDAwMRJEiRUT58uXFtm3bpHU/PzUDT9NAmpSQkCDmzZsndHR0RLdu3URSUpK07NKlS6JgwYJi2bJlWqyQ/gtn/fzGhBC4evUqGjRoAG9vb4SFhUnLnj9/DiEEYmNjUb9+fdStWxcrVqwAAAQFBcHb2xvv37/XVumUAYkv9iLnzJkTffr0QZUqVXDo0CE0btwYvr6+CAoKwsOHD7F48WKsXr0aAFR6T9iTQppkaGiI3r17Y8eOHejRowf09fWlbbVMmTIwMjLC06dPtVwl/RsGld+YQqFAqVKlMH/+fGzduhXe3t549OgRAKBRo0Z4/fo1zMzM0KhRI/j6+krd8YGBgbhx4wane5LGKJVKaUD2o0ePEBcXh/z586Ndu3ZISEjAokWLMGDAAHTq1An29vYoWrQoQkJCcOfOHS1XTpmBqakpGjRoIB1MMHVbDQ8Ph7GxMYoWLarN8ug/8KfLbyx1P76npycAYO7cudDV1UWPHj2QJ08ejB8/HjNmzMDHjx/x4cMHhISEYNOmTVi1ahVOnTolHRWU6Gd8PnB2woQJOHv2LIYPH44aNWrAysoKcXFxePnyJUxMTKCjo4PExEQ4OTlhxIgRqF+/vparp4xIfDaTMZWhoaH0/5SUFLx9+xY9e/aEQqFAu3btfnWJpAYGld9Yao/IoUOHoKOjg+TkZHh5eSEhIQEjR46Em5sb4uPjMWPGDGzfvh3Zs2eHgYEBgoODUaxYMS1XTxnF5yFlxYoV8PX1RalSpaSZO4mJibCyssKpU6ekAbPv3r3DmjVroKOjoxJ0iH5EWFgYIiIikC1bNtjZ2f3rEWSTk5Ph5+eHTZs2ISIiAufOnZPO3cNeZnni9OTfXGBgoHQiN1NTUzx48ACLFy9G3759MXLkSNjY2CAmJgbHjx+Hk5MTbG1tYWtrq+2y6Tf3Zbi4f/8+mjdvjtmzZ6NJkyZp1rt48SLGjRuH2NhYWFlZISAgAPr6+gwp9NM2bNiA+fPnIzw8HNbW1ujfv7/UU5Lqy+0sKCgIt2/fRr9+/TjL7DfAoPIbUyqV6NChAxQKBTZu3Ci1L1myBCNGjICnpyf69u2LvHnzarFKymhatmyJMWPGoGzZslLbtWvXUL9+fRw/fhwFCxb86kEEExISIISAkZERFAoFvxzop23YsAGenp7S4fFnzJiBR48e4fTp09K2lRpSIiMjcejQIbi5uancB3tS5I8/ZX5jqb8QUrvYk5KSAAD9+/eHh4cH1q5di8WLF6vMBiL6Webm5nBxcVFpMzIywvv373Hr1i2pLfX8PmfPnsWOHTugo6MDY2NjKBQKKJVKhhT6KZcuXcLUqVOxdOlSdOvWDcWLF8fgwYPh7OyMM2fO4Pbt24iOjpZ2i69fvx59+/bFX3/9pXI/DCnyx6DyG3rx4oX0/4IFC2Lv3r0IDw+HgYEBkpOTAQAODg4wMTFBcHAwjI2NtVUqZSDPnz8HAKxduxYGBgZYvHgxDh06hKSkJDg7O6NNmzaYO3cuDh8+DIVCAR0dHaSkpGD69OkIDg5WGTfA3T30sxITEzFo0CA0atRIaps0aRKOHDmCdu3aoXPnzmjbti0iIiKgr6+Phg0bYtiwYRw4+xvirp/fzPXr19GvXz+0b98effr0QVJSEmrWrIm3b9/i2LFjsLOzAwCMHDkSRYsWRePGjdOc1pxIXT179gQAjB49WtqV6OLigrdv32Lz5s2oWrUqTp48iYULF+LmzZvo0KEDDAwMcOTIEbx58wZXrlxhDwpplFKpxJs3b5A9e3YAQOfOnXH48GHs2bMHjo6OOH78OKZNm4aRI0eiffv2KmNWuLvn98KfNb8ZExMTWFhYYPv27Vi3bh0MDAywYsUK2NjYoHDhwmjevDnq1q2LRYsWoWzZsgwppBEuLi44ePAgvL29ERISAgC4ceMGChYsiA4dOuDEiROoUqUKpkyZgs6dO8PPzw9Hjx5Frly5cPnyZWnAIpGm6OjoSCEFAIYNG4bz58+jbNmyyJ49Oxo0aICIiAi8fv06zVRlhpTfC3tUfkMhISEYM2YMXr16hZ49e6JTp05ISUnBvHnzEBYWBiEE+vfvjyJFimi7VMpA1qxZgwkTJqBt27bo2bMnChYsCACoWrUqHj9+DH9/f1StWhUA8OHDB5iYmEi35cBZ+tWePXuGjh07YtiwYTzp4G+OQeU3cOXKFbx8+VJlX2xISAjGjRuH0NBQ9O/fHx06dNBihZSRfT61c/Xq1ZgwYQLatWuXJqyEhYVhw4YNqFChgsp4lK8dfItIHZ9vQ6n/T/33zZs3sLGxUVk/Li4O7dq1Q1RUFI4ePcoelN8cg4rMxcTEoFGjRtDV1cWIESPQoEEDaVloaCjq168PExMT9OjRA3379tVipZTRfOsYJytXrsTkyZPRpk0b9OrVSworNWvWxOnTp3Hu3DmUKlXqV5dLGdTXtsPUtoCAAGzatAmLFi2Cvb094uPjsXv3bvj5+eH58+e4ePEi9PX1OSblN8cxKjKVmh+zZs2KOXPmQE9PD0uXLsX+/fuldZycnFCjRg28evUKR44cQWRkpJaqpYzm8y+HM2fOIDg4GNevXwfwaWDt+PHjsXnzZvj6+uLevXsAgKNHj6JHjx5ppi4T/ahTp05JJwwcMmQIZs2aBeDT+JQtW7agc+fOqF27Nuzt7QF8OqHl48ePkTdvXly6dAn6+vr4+PEjQ8pvjj0qMpPanZn6CyD1C+P8+fMYNWoUTE1N0adPH2k30NChQ5E3b160bNkSOXLk0HL1lBF83s0+ZMgQbNmyBbGxsXBwcECuXLlw4MABAMCKFSswbdo0tG3bFu7u7iqnZeAvWPoZQghERUXB1tYWDRo0gLW1NQICAnDy5EkUK1YMkZGRcHV1haenJ/r37y/d5vPPToDbYUbBoCIjqW+04OBg7NmzBxEREahcuTL+/PNPWFhY4Ny5cxg/fjwSExORN29emJiYYMuWLbh+/TocHBy0XT5lAJ+HlEOHDmHQoEHw9fWFhYUF/vnnH0ycOBGmpqa4dOkSgE9jVjw8PODl5YV+/fpps3TKgMLDw5E3b16kpKRgx44daNiwobTsa2NTvjaWhX5/3PUjIwqFAjt37kSTJk3w4cMHfPjwAX5+fujTpw8iIiLg6uqKefPmoVq1aggJCcGjR49w9OhRhhTSmNQP9j179mDz5s2oXbs2KleujGLFiqF169bYsGEDYmNj0adPHwBA9+7dsXv3buk6kaYkJibi1atXMDExga6uLtasWSNNjQcAa2tr6f+pR0H+PJgwpGQc7FGRkUuXLqFt27YYNWoUevTogbCwMJQuXRrGxsYoWbIkNmzYACsrK+ncKV9OASXShIiICDRu3BjXr19HjRo1sG/fPpXlY8aMwenTp/H333/D1NRUamc3O/2sbw3gDg0NhYuLC2rUqIEFCxYgX758WqiOtIU9Kloyc+ZMjB07VvolAHw6RLmrqyt69OiB0NBQ1KpVC82bN8e4ceNw8eJF9O3bFxERETAyMgIAhhTSiM+3QQCwsrLC+vXrUadOHVy9ehVr165VWZ4/f368e/cO8fHxKu0MKfQzPg8px44dw8aNG3H9+nU8f/4cTk5OOH36NIKDgzFixAhpAHeLFi2wZMkSbZZNvwB7VLRkyZIlGDhwIGbMmIERI0ZIb9A7d+6gYMGCaNasmfSFoVQqUbJkSYSEhKBRo0bYsmULz5VCGvH5l8PDhw+hUChgYmICOzs7PH78GJ6enoiLi8Off/4JDw8PvH79Gu7u7jAyMsK+ffvYvU4aN2zYMKxfvx56enrIkiUL7OzssHDhQpQtWxY3b95EjRo14OTkhKSkJHz8+BHXr1+XTsxKGZSgX06pVAohhFi5cqXQ0dERU6dOFcnJydLyp0+fisKFC4t9+/YJIYSIiIgQ7dq1E0uWLBHPnj3TSs2U8aRuh0IIMXHiRFG8eHFRqFAhkSNHDuHr6yuEECIkJEQ0bNhQGBkZiYIFC4oWLVqIevXqifj4eCGEECkpKVqpnTKOz7fDoKAgUaJECXHy5EkREREhdu/eLVq0aCGcnZ3FlStXhBBCPHjwQEyZMkVMnz5d+tz8/POTMh4GlV9MqVRKb0ylUin++usvoaOjI6ZNmyZ96IeHh4uSJUsKDw8PERoaKsaMGSPKlSsnXr9+rc3SKYOaMmWKsLGxEYGBgSI2Nla0aNFCWFhYiNu3bwshhHj06JFo1KiRKFmypFi4cKF0u4SEBC1VTBnR+vXrRb9+/USvXr1U2i9evCjq168v3N3dRWxsrBBCNdwwpGR83H+gBQqFAocPH8bQoUNRpkwZ6Rwqs2bNghAClpaW6NChA44fPw5XV1ds2LABPj4+sLW11XbplAF8PiZFqVTiwoULWLhwIerWrYugoCAcO3YMM2bMQJEiRZCcnIw8efJg/vz5yJ49O/bv34+AgAAAgKGhobaeAmUA4otRB7t27cKyZctw7do1JCYmSu1ly5ZFlSpVcOrUKaSkpABQndHDc0hlAtpOSpnRjh07hLGxsZg6daq4ePGiEEIIX19faTeQEEIkJiaK27dvi6CgIPH06VNtlksZ1IQJE8SsWbNEzpw5xb1790RwcLDIkiWL8Pb2FkII8eHDBzF27FgRGhoqhBDi/v37onHjxqJs2bIiICBAm6XTb+7zHhF/f3+xYcMGIYQQ/fr1ExYWFmLZsmUiKipKWicwMFAUKlRI2hYpc2FQ+cXu3bsn8uTJI5YvX55m2YoVK6TdQESa9vl4ks2bNwtHR0dx69Yt0bFjR1GvXj1hYmIiVq9eLa3z/PlzUaVKFbFhwwbptnfu3BGtW7cWYWFhv7x+yhg+3w5v3bolSpUqJUqUKCF2794thBDC3d1d5M+fX0yfPl2EhISIkJAQUatWLVGtWjWVgEOZB/vMfrEnT55AX19f5QiLqTMvevXqBVNTU3Tq1AmGhoYYNmyYFiuljCZ1ds/x48dx7NgxDB06FEWLFpUOJFirVi1069YNwKeTYfbo0QO6urpo3749dHR0oFQqUahQIWzcuJGzLOiHpW6Hw4cPx+PHj2FsbIy7d+9i8ODB+PjxI9atW4du3bph3LhxWLJkCSpVqoQsWbJgy5YtUCgU3zzWCmVcDCq/WGxsrMrxJ5RKpbS/9dixYyhTpgy2bNmict4UIk159eoVunfvjvDwcIwZMwYA0Lt3bzx8+BBHjx5FqVKlkD9/fjx58gQJCQm4ePEidHV1VQ7mxjEB9LPWrVuHVatW4ciRI8iTJw8SExPh7u6OmTNnQkdHB2vWrIGJiQm2bt2K+vXro23btjA0NERSUhIMDAy0XT79Yoylv1iJEiXw9u1b+Pr6Avj06yI1qOzevRsbN25Ey5YtUbhwYW2WSRmUnZ0dAgICkD17duzduxeXL1+Grq4u5s6diylTpqBmzZqws7NDmzZtvnn2WR47hX5WSEgIihUrhpIlS8Lc3Bx2dnZYs2YNdHV1MXjwYOzcuRNLly5F7dq1sWDBAuzZswcxMTEMKZkUfxr9Ynny5MHSpUvRu3dvJCcno3PnztDV1cW6deuwbt06nD17lkf4pHTl4uKCHTt2wN3dHT4+Pujfvz9cXFzQtGlTNG3aVGXdlJQU9qCQxoj/P1GgoaEhEhISkJSUBCMjIyQnJyNnzpyYOXMmGjduDC8vLxgbG2Pjxo1o3749hg0bBj09Pbi5uWn7KZAW8Mi0WqBUKrFjxw54eHjA1NQURkZG0NXVxaZNm1CqVCltl0eZxNWrV9GjRw+UKVMGAwcORNGiRbVdEmUSN2/eRKlSpTB+/HhMnDhRag8MDMTKlSvx/v17pKSk4NixYwCArl27Yvz48cibN6+WKiZtYlDRohcvXiAsLAwKhQJ58uRB9uzZtV0SZTJXr16Fh4cHcufOjTlz5iBPnjzaLokyiXXr1qFXr14YNGgQ2rRpA0tLSwwYMAAVK1ZEixYtULRoUezfvx8NGjTQdqmkZQwqRJnchQsX4OPjg1WrVnE2Bf1SO3bsQN++fWFgYAAhBGxtbXHmzBm8fv0aderUwfbt2+Hi4qLtMknLGFSISBo7wKmf9Ks9f/4cT58+RXJyMipVqgQdHR2MHj0au3btQnBwMOzs7LRdImkZgwoRAfhfWCHSltu3b2P27Nn4+++/cfjwYZQsWVLbJZEMcDg/EQHgtGPSro8fPyIpKQm2trY4fvw4B3eThD0qREQkG8nJyTzyMalgUCEiIiLZ4qg5IiIiki0GFSIiIpItBhUiIiKSLQYVIiIiki0GFSL6rRw7dgwKhQKRkZHffRsnJyd4eXmlW01ElH4YVIhIo7p06QKFQoHevXunWebp6QmFQoEuXbr8+sKI6LfEoEJEGufo6IjNmzcjPj5eaktISMDGjRuRK1cuLVZGRL8bBhUi0rjSpUvD0dERAQEBUltAQABy5cqFUqVKSW2JiYkYMGAAbG1tYWRkhMqVK+PixYsq9/X333+jQIECMDY2Ro0aNRAaGprm8U6dOoUqVarA2NgYjo6OGDBgAOLi4tLt+RHRr8OgQkTpolu3bli7dq10fc2aNejatavKOiNGjMCOHTuwfv16XLlyBc7OzqhXrx4iIiIAAE+fPkXLli3RpEkTXLt2DT169MCoUaNU7uPhw4eoX78+WrVqhRs3bmDLli04deoU+vXrl/5PkojSHYMKEaWLjh074tSpUwgLC0NYWBhOnz6Njh07Ssvj4uLg7e2NuXPnokGDBihSpAhWrlwJY2NjrF69GgDg7e2NfPnyYf78+ShYsCA6dOiQZnzLzJkz0aFDBwwaNAj58+dHxYoVsXjxYmzYsAEJCQm/8ikTUTrgSQmJKF3Y2NigUaNGWLduHYQQaNSoEaytraXlDx8+RHJyMipVqiS16evr448//sCdO3cAAHfu3EH58uVV7rdChQoq169fv44bN27A399fahNCQKlU4vHjxyhcuHB6PD0i+kUYVIgo3XTr1k3aBbNs2bJ0eYzY2Fh4eHhgwIABaZZx4C7R749BhYjSTf369ZGUlASFQoF69eqpLMuXLx8MDAxw+vRp5M6dG8CnM+devHgRgwYNAgAULlwYe/bsUbnduXPnVK6XLl0a//zzD5ydndPviRCR1nCMChGlG11dXdy5cwf//PMPdHV1VZaZmpqiT58+GD58OA4ePIh//vkHPXv2xIcPH9C9e3cAQO/evfHgwQMMHz4c9+7dw8aNG7Fu3TqV+xk5ciTOnDmDfv364dq1a3jw4AF2797NwbREGQSDChGlKzMzM5iZmX112axZs9CqVSt06tQJpUuXRkhICAIDA2FpaQng066bHTt2YNeuXShRogR8fHwwY8YMlftwcXHB8ePHcf/+fVSpUgWlSpXChAkTYG9vn+7PjYjSn0IIIbRdBBEREdHXsEeFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhki0GFiIiIZItBhYiIiGSLQYWIiIhk6/8AHoK08GWUizwAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "## calculate avg response time\n", - "unique_models = set(result[\"response\"]['model'] for result in result[\"results\"])\n", - "model_dict = {model: {\"response_time\": []} for model in unique_models}\n", - "for completion_result in result[\"results\"]:\n", - " model_dict[completion_result[\"response\"][\"model\"]][\"response_time\"].append(completion_result[\"response_time\"])\n", - "\n", - "avg_response_time = {}\n", - "for model, data in model_dict.items():\n", - " avg_response_time[model] = sum(data[\"response_time\"]) / len(data[\"response_time\"])\n", - "\n", - "models = list(avg_response_time.keys())\n", - "response_times = list(avg_response_time.values())\n", - "\n", - "plt.bar(models, response_times)\n", - "plt.xlabel('Model', fontsize=10)\n", - "plt.ylabel('Average Response Time')\n", - "plt.title('Average Response Times for each Model')\n", - "\n", - "plt.xticks(models, [model[:15]+'...' if len(model) > 15 else model for model in models], rotation=45)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "inSDIE3_IRds" - }, - "source": [ - "# Duration Test endpoint\n", - "\n", - "Run load testing for 2 mins. Hitting endpoints with 100+ queries every 15 seconds." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ePIqDx2EIURH" - }, - "outputs": [], - "source": [ - "models=[\"gpt-3.5-turbo\", \"replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781\", \"claude-instant-1\"]\n", - "context = \"\"\"Paul Graham (/ɡræm/; born 1964)[3] is an English computer scientist, essayist, entrepreneur, venture capitalist, and author. He is best known for his work on the programming language Lisp, his former startup Viaweb (later renamed Yahoo! Store), cofounding the influential startup accelerator and seed capital firm Y Combinator, his essays, and Hacker News. He is the author of several computer programming books, including: On Lisp,[4] ANSI Common Lisp,[5] and Hackers & Painters.[6] Technology journalist Steven Levy has described Graham as a \"hacker philosopher\".[7] Graham was born in England, where he and his family maintain permanent residence. However he is also a citizen of the United States, where he was educated, lived, and worked until 2016.\"\"\"\n", - "prompt = \"Where does Paul Graham live?\"\n", - "final_prompt = context + prompt\n", - "result = load_test_model(models=models, prompt=final_prompt, num_calls=100, interval=15, duration=120)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 552 - }, - "id": "k6rJoELM6t1K", - "outputId": "f4968b59-3bca-4f78-a88b-149ad55e3cf7" - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAIXCAYAAABghH+YAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABwdUlEQVR4nO3dd1QU198G8GfpoNKUooKCYuwIaiL2GrGLJnYFOxrsNZbYFTsYG2JDjV2xRKOIir33EhsWLBGwUaXJ3vcPX+bnCiYsLi6Oz+ecPbp37ux+lx3YZ+/cmVEIIQSIiIiIZEJH2wUQERERaRLDDREREckKww0RERHJCsMNERERyQrDDREREckKww0RERHJCsMNERERyQrDDREREckKww0RERHJCsMNEcmSg4MDunfvru0y1DZnzhyUKFECurq6cHFx0XY5GnfkyBEoFAps27ZN26WoTaFQYNKkSWqv9+jRIygUCgQFBWm8Jsoaww19tiVLlkChUKBatWraLiXPcXBwgEKhkG758uXDDz/8gLVr12q7tK9Oxodidm5fqwMHDmDUqFGoWbMmVq9ejRkzZmi7pDwnKChIep9PnDiRabkQAvb29lAoFGjRooUWKqS8QE/bBdDXb/369XBwcMC5c+cQHh4OJycnbZeUp7i4uGD48OEAgOfPn2PFihXw8vJCSkoK+vTpo+Xqvh5ly5bFunXrVNrGjBmD/PnzY9y4cZn637lzBzo6X9f3t8OHD0NHRwcrV66EgYGBtsvJ04yMjLBhwwbUqlVLpf3o0aN4+vQpDA0NtVQZ5QUMN/RZHj58iFOnTiE4OBje3t5Yv349Jk6c+EVrUCqVSE1NhZGR0Rd93uwqWrQounbtKt3v3r07SpQoAT8/P4YbNdjY2Kj8HAFg5syZKFSoUKZ2AF/lh1t0dDSMjY01FmyEEEhOToaxsbFGHi8vadasGbZu3Yrff/8denr/+yjbsGEDqlSpgpcvX2qxOtK2r+trDeU569evh4WFBZo3b46ff/4Z69evl5alpaXB0tISPXr0yLReXFwcjIyMMGLECKktJSUFEydOhJOTEwwNDWFvb49Ro0YhJSVFZV2FQoEBAwZg/fr1KF++PAwNDbF//34AwNy5c1GjRg0ULFgQxsbGqFKlSpb79pOSkjBo0CAUKlQIBQoUQKtWrfDs2bMs96k/e/YMPXv2hI2NDQwNDVG+fHmsWrUqxz8zKysrlClTBvfv31dpVyqV8Pf3R/ny5WFkZAQbGxt4e3vjzZs3Kv0uXLgAd3d3FCpUCMbGxnB0dETPnj2l5Rn79+fOnQs/Pz8UL14cxsbGqFu3Lm7cuJGpnsOHD6N27drIly8fzM3N0bp1a9y6dUulz6RJk6BQKBAeHo7u3bvD3NwcZmZm6NGjB96+favSNzQ0FLVq1YK5uTny58+P0qVLY+zYsSp9svtef46P59xk7M44ceIEBg0aBCsrK5ibm8Pb2xupqamIiYmBp6cnLCwsYGFhgVGjRkEIofKYmnqPsqJQKLB69WokJiZKu10y5mi8e/cOU6dORcmSJWFoaAgHBweMHTs208/LwcEBLVq0QEhICKpWrQpjY2MsW7bsX5/37NmzaNKkCczMzGBiYoK6devi5MmTKn0iIiLwyy+/oHTp0jA2NkbBggXRrl07PHr0KNPjxcTEYOjQoXBwcIChoSHs7Ozg6emZKWwolUpMnz4ddnZ2MDIyQsOGDREeHv6vtX6oU6dOePXqFUJDQ6W21NRUbNu2DZ07d85yncTERAwfPhz29vYwNDRE6dKlMXfu3Ezvc0pKCoYOHQorKyvp78PTp0+zfExN/30gDRFEn6FMmTKiV69eQgghjh07JgCIc+fOSct79uwpzM3NRUpKisp6a9asEQDE+fPnhRBCpKeni8aNGwsTExMxZMgQsWzZMjFgwAChp6cnWrdurbIuAFG2bFlhZWUlJk+eLBYvXiwuX74shBDCzs5O/PLLL2LRokVi/vz54ocffhAAxJ49e1Qeo3379gKA6Natm1i8eLFo3769qFSpkgAgJk6cKPWLjIwUdnZ2wt7eXkyZMkUsXbpUtGrVSgAQfn5+//nzKV68uGjevLlKW1pamrC1tRU2NjYq7b179xZ6enqiT58+IiAgQIwePVrky5dPfP/99yI1NVUIIURUVJSwsLAQ3333nZgzZ45Yvny5GDdunChbtqz0OA8fPhQARMWKFYWDg4OYNWuWmDx5srC0tBRWVlYiMjJS6hsaGir09PTEd999J2bPni0mT54sChUqJCwsLMTDhw+lfhMnThQAhKurq2jbtq1YsmSJ6N27twAgRo0aJfW7ceOGMDAwEFWrVhULFiwQAQEBYsSIEaJOnTpSH3Xe6/9Svnx5Ubdu3U/+7L28vKT7q1evFgCEi4uLaNKkiVi8eLHo1q2b9Bpq1aolOnfuLJYsWSJatGghAIg1a9bkynuUlXXr1onatWsLQ0NDsW7dOrFu3Tpx//59IYQQXl5eAoD4+eefxeLFi4Wnp6cAIDw8PDK9ZicnJ2FhYSF+/fVXERAQIMLCwj75nIcOHRIGBgaievXqYt68ecLPz084OzsLAwMDcfbsWanf1q1bRaVKlcSECRNEYGCgGDt2rLCwsBDFixcXiYmJUr/4+HhRoUIFoaurK/r06SOWLl0qpk6dKr7//nvpdzQsLEzalqpUqSL8/PzEpEmThImJifjhhx/+9Wf04ft4/vx5UaNGDdGtWzdp2c6dO4WOjo549uxZpt89pVIpGjRoIBQKhejdu7dYtGiRaNmypQAghgwZovIcXbt2FQBE586dxaJFi0Tbtm2Fs7Nzjv8+ZPxOrl69+j9fH2kGww3l2IULFwQAERoaKoR4/8fDzs5ODB48WOoTEhIiAIg///xTZd1mzZqJEiVKSPfXrVsndHR0xPHjx1X6BQQECADi5MmTUhsAoaOjI27evJmpprdv36rcT01NFRUqVBANGjSQ2i5evJjlH7Tu3btn+uPVq1cvUbhwYfHy5UuVvh07dhRmZmaZnu9jxYsXF40bNxYvXrwQL168ENevX5c+UH18fKR+x48fFwDE+vXrVdbfv3+/SvuOHTtUQmFWMv6QGhsbi6dPn0rtZ8+eFQDE0KFDpTYXFxdhbW0tXr16JbVdvXpV6OjoCE9PT6ktI9z07NlT5bnatGkjChYsKN338/MTAMSLFy8+WZ867/V/yUm4cXd3F0qlUmqvXr26UCgUol+/flLbu3fvhJ2dncpja/I9+hQvLy+RL18+lbYrV64IAKJ3794q7SNGjBAAxOHDh1VeMwCxf//+/3wupVIpSpUqlenn8fbtW+Ho6Ch+/PFHlbaPnT59WgAQa9euldomTJggAIjg4OAsn0+I/4WbsmXLqnzpWbBggQAgrl+//q91fxhuFi1aJAoUKCDV165dO1G/fn3pZ/FhuNm5c6cAIKZNm6byeD///LNQKBQiPDxcCPG/n/cvv/yi0q9z5845/vvAcPPlcbcU5dj69ethY2OD+vXrA3g/rN6hQwds2rQJ6enpAIAGDRqgUKFC2Lx5s7TemzdvEBoaig4dOkhtW7duRdmyZVGmTBm8fPlSujVo0AAAEBYWpvLcdevWRbly5TLV9OHcgjdv3iA2Nha1a9fGpUuXpPaMXVi//PKLyroDBw5UuS+EwPbt29GyZUsIIVTqcnd3R2xsrMrjfsqBAwdgZWUFKysrVKxYEevWrUOPHj0wZ84clddvZmaGH3/8UeV5qlSpgvz580uv39zcHACwZ88epKWl/evzenh4oGjRotL9H374AdWqVcNff/0F4P3k5itXrqB79+6wtLSU+jk7O+PHH3+U+n2oX79+Kvdr166NV69eIS4uTqW+Xbt2QalUZlmXuu+1pvXq1UvliKpq1apBCIFevXpJbbq6uqhatSoePHigUrem36PsyHgfhg0bptKeMUl97969Ku2Ojo5wd3f/z8e9cuUK7t27h86dO+PVq1fS60lMTETDhg1x7Ngx6T388PcqLS0Nr169gpOTE8zNzVV+B7Zv345KlSqhTZs2mZ7v46PYevTooTK3qHbt2gCg8jP/L+3bt0dSUhL27NmD+Ph47Nmz55O7pP766y/o6upi0KBBKu3Dhw+HEAL79u2T+gHI1G/IkCEq9zX194Fyxzcdbo4dO4aWLVuiSJEiUCgU2LlzZ64/57Nnz9C1a1dpTkjFihVx4cKFXH9eTUtPT8emTZtQv359PHz4EOHh4QgPD0e1atUQFRWFQ4cOAQD09PTw008/YdeuXdL8gODgYKSlpamEm3v37uHmzZtSCMi4fffddwDeT7T8kKOjY5Z17dmzB25ubjAyMoKlpSWsrKywdOlSxMbGSn0iIiKgo6OT6TE+PsrrxYsXiImJQWBgYKa6MuYRfVxXVqpVq4bQ0FDs378fc+fOhbm5Od68eaPyh/3evXuIjY2FtbV1pudKSEiQnqdu3br46aefMHnyZBQqVAitW7fG6tWrs5yrUqpUqUxt3333nTRPIiIiAgBQunTpTP3Kli0rfdB9qFixYir3LSwsAECac9KhQwfUrFkTvXv3ho2NDTp27IgtW7aoBB1132tN+/g1mJmZAQDs7e0ztX84lyY33qPsyNheP94+bW1tYW5uLr2PGT71u/Gxe/fuAQC8vLwyvZ4VK1YgJSVF+r1JSkrChAkTpLkqhQoVgpWVFWJiYlR+t+7fv48KFSpk6/n/a1vKDisrKzRq1AgbNmxAcHAw0tPT8fPPP2fZNyIiAkWKFEGBAgVU2suWLSstz/hXR0cHJUuWVOn38e+Jpv4+UO74po+WSkxMRKVKldCzZ0+0bds215/vzZs3qFmzJurXr499+/bBysoK9+7dk36pvyaHDx/G8+fPsWnTJmzatCnT8vXr16Nx48YAgI4dO2LZsmXYt28fPDw8sGXLFpQpUwaVKlWS+iuVSlSsWBHz58/P8vk+/uDJ6uiP48ePo1WrVqhTpw6WLFmCwoULQ19fH6tXr8aGDRvUfo0ZH8hdu3aFl5dXln2cnZ3/83EKFSqERo0aAQDc3d1RpkwZtGjRAgsWLJC+jSuVSlhbW6tMyP6QlZUVAEgnPztz5gz+/PNPhISEoGfPnpg3bx7OnDmD/Pnzq/061aGrq5tlu/j/CZnGxsY4duwYwsLCsHfvXuzfvx+bN29GgwYNcODAAejq6qr9Xmvap15DVu3ig4mm2n6Psnv+nuweGZWxfc+ZM+eTJwvMqHXgwIFYvXo1hgwZgurVq8PMzAwKhQIdO3b85Ajdf/mvbSm7OnfujD59+iAyMhJNmzaVRs5ym6b+PlDu+KbDTdOmTdG0adNPLk9JScG4ceOwceNGxMTEoEKFCpg1axbq1auXo+ebNWsW7O3tsXr1aqktu9+y8pr169fD2toaixcvzrQsODgYO3bsQEBAAIyNjVGnTh0ULlwYmzdvRq1atXD48OFM5yUpWbIkrl69ioYNG+b4JGzbt2+HkZERQkJCVA4D/vDnDQDFixeHUqnEw4cPVUY3Pj5SI+NIifT0dCmcaELz5s1Rt25dzJgxA97e3siXLx9KliyJgwcPombNmtn6cHJzc4ObmxumT5+ODRs2oEuXLti0aRN69+4t9cn4Zv6hu3fvwsHBAcD7nwPw/nwwH7t9+zYKFSqEfPnyqf36dHR00LBhQzRs2BDz58/HjBkzMG7cOISFhaFRo0Yaea+1ITfeo+zI2F7v3bsnjTIAQFRUFGJiYqT3UV0ZIxOmpqb/uX1v27YNXl5emDdvntSWnJyMmJiYTI+Z1RF5ualNmzbw9vbGmTNnVHZ/f6x48eI4ePAg4uPjVUZvbt++LS3P+FepVOL+/fsqozUf/57k1t8H0oxverfUfxkwYABOnz6NTZs24dq1a2jXrh2aNGmS5YdGduzevRtVq1ZFu3btYG1tDVdXVyxfvlzDVee+pKQkBAcHo0WLFvj5558z3QYMGID4+Hjs3r0bwPsPu59//hl//vkn1q1bh3fv3qnskgLe7zt/9uxZlj+PpKSkTLtHsqKrqwuFQiHN9wHeHxb98e7GjPkIS5YsUWlfuHBhpsf76aefsH379iz/YL948eI/a/qU0aNH49WrV9Lrbd++PdLT0zF16tRMfd+9eyd9iLx58ybTN9uMb90f7/bYuXMnnj17Jt0/d+4czp49KwX6woULw8XFBWvWrFH5kLpx4wYOHDiAZs2aqf26Xr9+nant4/o08V5rQ268R9mR8T74+/urtGeMfDVv3lztxwSAKlWqoGTJkpg7dy4SEhIyLf9w+9bV1c30mhYuXKjyuwYAP/30E65evYodO3Zkejx1R2SyK3/+/Fi6dCkmTZqEli1bfrJfs2bNkJ6ejkWLFqm0+/n5QaFQSL8XGf/+/vvvKv0+/vnn5t8H+nzf9MjNv3n8+DFWr16Nx48fo0iRIgCAESNGYP/+/Tk+LfqDBw+wdOlSDBs2DGPHjsX58+cxaNAgGBgYfHJYMy/avXs34uPj0apVqyyXu7m5wcrKCuvXr5dCTIcOHbBw4UJMnDgRFStWVPkGCgDdunXDli1b0K9fP4SFhaFmzZpIT0/H7du3sWXLFum8Hf+mefPmmD9/Ppo0aYLOnTsjOjoaixcvhpOTE65duyb1q1KlCn766Sf4+/vj1atXcHNzw9GjR3H37l0AqsP/M2fORFhYGKpVq4Y+ffqgXLlyeP36NS5duoSDBw9m+WGeHU2bNkWFChUwf/58+Pj4oG7duvD29oavry+uXLmCxo0bQ19fH/fu3cPWrVuxYMEC/Pzzz1izZg2WLFmCNm3aoGTJkoiPj8fy5cthamqaKYw4OTmhVq1a6N+/P1JSUuDv74+CBQti1KhRUp85c+agadOmqF69Onr16oWkpCQsXLgQZmZmObqGzpQpU3Ds2DE0b94cxYsXR3R0NJYsWQI7OzvpTLKaeK+1ITfeo+yoVKkSvLy8EBgYiJiYGNStWxfnzp3DmjVr4OHhIU3oV5eOjg5WrFiBpk2bonz58ujRoweKFi2KZ8+eISwsDKampvjzzz8BAC1atMC6detgZmaGcuXK4fTp0zh48CAKFiyo8pgjR47Etm3b0K5dO/Ts2RNVqlTB69evsXv3bgQEBKjsitak7Pz9bNmyJerXr49x48bh0aNHqFSpEg4cOIBdu3ZhyJAh0kiWi4sLOnXqhCVLliA2NhY1atTAoUOHsjwHT279fSAN0MoxWnkQALFjxw7p/p49ewQAkS9fPpWbnp6eaN++vRBCiFu3bgkA/3obPXq09Jj6+vqievXqKs87cOBA4ebm9kVeo6a0bNlSGBkZqZzf4mPdu3cX+vr60iGSSqVS2NvbZ3koZobU1FQxa9YsUb58eWFoaCgsLCxElSpVxOTJk0VsbKzUDx8dRv2hlStXilKlSglDQ0NRpkwZsXr1aukw5g8lJiYKHx8fYWlpKfLnzy88PDzEnTt3BAAxc+ZMlb5RUVHCx8dH2NvbC319fWFraysaNmwoAgMD//NnldV5bjIEBQVlOjw0MDBQVKlSRRgbG4sCBQqIihUrilGjRol//vlHCCHEpUuXRKdOnUSxYsWEoaGhsLa2Fi1atBAXLlyQHiPjsNM5c+aIefPmCXt7e2FoaChq164trl69mqmOgwcPipo1awpjY2NhamoqWrZsKf7++2+VPhk/w48P8c44LDfjnDiHDh0SrVu3FkWKFBEGBgaiSJEiolOnTuLu3bsq62X3vf4vOTkU/ONDtD/12rI6LFsIzbxHn/Kp50xLSxOTJ08Wjo6OQl9fX9jb24sxY8aI5OTkTK/5U9vbp1y+fFm0bdtWFCxYUBgaGorixYuL9u3bi0OHDkl93rx5I3r06CEKFSok8ufPL9zd3cXt27cz/YyFEOLVq1diwIABomjRosLAwEDY2dkJLy8v6W9BxqHgW7duVVkvu4dLf+p9/FhWP4v4+HgxdOhQUaRIEaGvry9KlSol5syZo3IovBBCJCUliUGDBomCBQuKfPnyiZYtW4onT55kOhRciOz9feCh4F+eQohcGiv8yigUCuzYsQMeHh4AgM2bN6NLly64efNmpolv+fPnh62tLVJTU//zsMWCBQtKEw2LFy+OH3/8EStWrJCWL126FNOmTVPZfUDaceXKFbi6uuKPP/5Aly5dtF1Ojj169AiOjo6YM2eOyhmgiYi+Fdwt9Qmurq5IT09HdHS0dP6FjxkYGKBMmTLZfsyaNWtmmpR29+7dHE8IpJxLSkrKNCnU398fOjo6qFOnjpaqIiIiTfimw01CQoLKftSHDx/iypUrsLS0xHfffYcuXbrA09MT8+bNg6urK168eIFDhw7B2dk5R5P4hg4diho1amDGjBlo3749zp07h8DAQAQGBmryZVE2zJ49GxcvXkT9+vWhp6eHffv2Yd++fejbt2+uH4pMRES5TNv7xbQpY9/vx7eMfcipqaliwoQJwsHBQejr64vChQuLNm3aiGvXruX4Of/8809RoUIFaU5IduZtkOYdOHBA1KxZU1hYWAh9fX1RsmRJMWnSJJGWlqbt0j7bh3NuiIi+RZxzQ0RERLLC89wQERGRrDDcEBERkax8cxOKlUol/vnnHxQoUOCrOvU7ERHRt0wIgfj4eBQpUgQ6Ov8+NvPNhZt//vmHR8MQERF9pZ48eQI7O7t/7fPNhZuMC6Y9efIEpqamWq6GiIiIsiMuLg729vYqFz79lG8u3GTsijI1NWW4ISIi+spkZ0oJJxQTERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs6Gm7ACLSLIdf92q7BNKyRzOba/X5uQ2StrdBjtwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrGg13CxduhTOzs4wNTWFqakpqlevjn379v3rOlu3bkWZMmVgZGSEihUr4q+//vpC1RIREdHXQKvhxs7ODjNnzsTFixdx4cIFNGjQAK1bt8bNmzez7H/q1Cl06tQJvXr1wuXLl+Hh4QEPDw/cuHHjC1dOREREeZVCCCG0XcSHLC0tMWfOHPTq1SvTsg4dOiAxMRF79uyR2tzc3ODi4oKAgIBsPX5cXBzMzMwQGxsLU1NTjdVNlFfwooWk7YsWchuk3NgG1fn8zjNzbtLT07Fp0yYkJiaievXqWfY5ffo0GjVqpNLm7u6O06dPf/JxU1JSEBcXp3IjIiIi+dJ6uLl+/Try588PQ0ND9OvXDzt27EC5cuWy7BsZGQkbGxuVNhsbG0RGRn7y8X19fWFmZibd7O3tNVo/ERER5S1aDzelS5fGlStXcPbsWfTv3x9eXl74+++/Nfb4Y8aMQWxsrHR78uSJxh6biIiI8h49bRdgYGAAJycnAECVKlVw/vx5LFiwAMuWLcvU19bWFlFRUSptUVFRsLW1/eTjGxoawtDQULNFExERUZ6l9ZGbjymVSqSkpGS5rHr16jh06JBKW2ho6Cfn6BAREdG3R6sjN2PGjEHTpk1RrFgxxMfHY8OGDThy5AhCQkIAAJ6enihatCh8fX0BAIMHD0bdunUxb948NG/eHJs2bcKFCxcQGBiozZdBREREeYhWw010dDQ8PT3x/PlzmJmZwdnZGSEhIfjxxx8BAI8fP4aOzv8Gl2rUqIENGzZg/PjxGDt2LEqVKoWdO3eiQoUK2noJRERElMdoNdysXLnyX5cfOXIkU1u7du3Qrl27XKqIiIiIvnZ5bs4NERER0edguCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZYbghIiIiWdFquPH19cX333+PAgUKwNraGh4eHrhz586/rhMUFASFQqFyMzIy+kIVExERUV6n1XBz9OhR+Pj44MyZMwgNDUVaWhoaN26MxMTEf13P1NQUz58/l24RERFfqGIiIiLK6/S0+eT79+9XuR8UFARra2tcvHgRderU+eR6CoUCtra2uV0eERERfYXy1Jyb2NhYAIClpeW/9ktISEDx4sVhb2+P1q1b4+bNm5/sm5KSgri4OJUbERERyVeeCTdKpRJDhgxBzZo1UaFChU/2K126NFatWoVdu3bhjz/+gFKpRI0aNfD06dMs+/v6+sLMzEy62dvb59ZLICIiojwgz4QbHx8f3LhxA5s2bfrXftWrV4enpydcXFxQt25dBAcHw8rKCsuWLcuy/5gxYxAbGyvdnjx5khvlExERUR6h1Tk3GQYMGIA9e/bg2LFjsLOzU2tdfX19uLq6Ijw8PMvlhoaGMDQ01ESZRERE9BXQ6siNEAIDBgzAjh07cPjwYTg6Oqr9GOnp6bh+/ToKFy6cCxUSERHR10arIzc+Pj7YsGEDdu3ahQIFCiAyMhIAYGZmBmNjYwCAp6cnihYtCl9fXwDAlClT4ObmBicnJ8TExGDOnDmIiIhA7969tfY6iIiIKO/QarhZunQpAKBevXoq7atXr0b37t0BAI8fP4aOzv8GmN68eYM+ffogMjISFhYWqFKlCk6dOoVy5cp9qbKJiIgoD9NquBFC/GefI0eOqNz38/ODn59fLlVEREREX7s8MaFYThx+3avtEkjLHs1sru0SiIi+aXnmUHAiIiIiTWC4ISIiIllhuCEiIiJZYbghIiIiWWG4ISIiIllhuCEiIiJZyVG4uX//PsaPH49OnTohOjoaALBv3z7cvHlTo8URERERqUvtcHP06FFUrFgRZ8+eRXBwMBISEgAAV69excSJEzVeIBEREZE61A43v/76K6ZNm4bQ0FAYGBhI7Q0aNMCZM2c0WhwRERGRutQON9evX0ebNm0ytVtbW+Ply5caKYqIiIgop9QON+bm5nj+/Hmm9suXL6No0aIaKYqIiIgop9QONx07dsTo0aMRGRkJhUIBpVKJkydPYsSIEfD09MyNGomIiIiyTe1wM2PGDJQpUwb29vZISEhAuXLlUKdOHdSoUQPjx4/PjRqJiIiIsk3tq4IbGBhg+fLl+O2333Djxg0kJCTA1dUVpUqVyo36iIiIiNSidrjJUKxYMRQrVkyTtRARERF9NrXDjRAC27ZtQ1hYGKKjo6FUKlWWBwcHa6w4IiIiInWpHW6GDBmCZcuWoX79+rCxsYFCociNuoiIiIhyRO1ws27dOgQHB6NZs2a5UQ8RERHRZ1H7aCkzMzOUKFEiN2ohIiIi+mxqh5tJkyZh8uTJSEpKyo16iIiIiD6L2rul2rdvj40bN8La2hoODg7Q19dXWX7p0iWNFUdERESkLrXDjZeXFy5evIiuXbtyQjERERHlOWqHm7179yIkJAS1atXKjXqIiIiIPovac27s7e1hamqaG7UQERERfTa1w828efMwatQoPHr0KBfKISIiIvo8au+W6tq1K96+fYuSJUvCxMQk04Ti169fa6w4IiIiInWpHW78/f1zoQwiIiIizcjR0VJEREREeVW2wk1cXJw0iTguLu5f+3KyMREREWlTtsKNhYUFnj9/Dmtra5ibm2d5bhshBBQKBdLT0zVeJBEREVF2ZSvcHD58GJaWlgCAsLCwXC2IiIiI6HNkK9zUrVsXJUqUwPnz51G3bt3cromIiIgox7J9nptHjx5xlxMRERHleWqfxI+IiIgoL1PrUPCQkBCYmZn9a59WrVp9VkFEREREn0OtcPNf57jh0VJERESkbWrtloqMjIRSqfzkjcGGiIiItC3b4Sarc9sQERER5TXZDjdCiNysg4iIiEgjsh1uvLy8YGxsnJu1EBEREX22bE8oXr16dW7WQURERKQRPM8NERERyQrDDREREckKww0RERHJSo7DTXh4OEJCQpCUlAQgZ0dT+fr64vvvv0eBAgVgbW0NDw8P3Llz5z/X27p1K8qUKQMjIyNUrFgRf/31l9rPTURERPKkdrh59eoVGjVqhO+++w7NmjXD8+fPAQC9evXC8OHD1Xqso0ePwsfHB2fOnEFoaCjS0tLQuHFjJCYmfnKdU6dOoVOnTujVqxcuX74MDw8PeHh44MaNG+q+FCIiIpIhtcPN0KFDoaenh8ePH8PExERq79ChA/bv36/WY+3fvx/du3dH+fLlUalSJQQFBeHx48e4ePHiJ9dZsGABmjRpgpEjR6Js2bKYOnUqKleujEWLFqn7UoiIiEiG1Lq2FAAcOHAAISEhsLOzU2kvVaoUIiIiPquY2NhYAIClpeUn+5w+fRrDhg1TaXN3d8fOnTuz7J+SkoKUlBTpflxc3GfVSERERHmb2iM3iYmJKiM2GV6/fg1DQ8McF6JUKjFkyBDUrFkTFSpU+GS/yMhI2NjYqLTZ2NggMjIyy/6+vr4wMzOTbvb29jmukYiIiPI+tcNN7dq1sXbtWum+QqGAUqnE7NmzUb9+/RwX4uPjgxs3bmDTpk05foysjBkzBrGxsdLtyZMnGn18IiIiylvU3i01e/ZsNGzYEBcuXEBqaipGjRqFmzdv4vXr1zh58mSOihgwYAD27NmDY8eOZdrd9TFbW1tERUWptEVFRcHW1jbL/oaGhp81okRERERfF7VHbipUqIC7d++iVq1aaN26NRITE9G2bVtcvnwZJUuWVOuxhBAYMGAAduzYgcOHD8PR0fE/16levToOHTqk0hYaGorq1aur9dxEREQkT2qP3ACAmZkZxo0b99lP7uPjgw0bNmDXrl0oUKCANG/GzMxMukinp6cnihYtCl9fXwDA4MGDUbduXcybNw/NmzfHpk2bcOHCBQQGBn52PURERPT1U3vkZv/+/Thx4oR0f/HixXBxcUHnzp3x5s0btR5r6dKliI2NRb169VC4cGHptnnzZqnP48ePpXPpAECNGjWwYcMGBAYGolKlSti2bRt27tz5r5OQiYiI6Nuh9sjNyJEjMWvWLADA9evXMWzYMAwfPhxhYWEYNmyYWlcPz85ZjY8cOZKprV27dmjXrl22n4eIiIi+HWqHm4cPH6JcuXIAgO3bt6Nly5aYMWMGLl26hGbNmmm8QCIiIiJ1qL1bysDAAG/fvgUAHDx4EI0bNwbw/sR7PEEeERERaZvaIze1atXCsGHDULNmTZw7d06aH3P37t3/PIybiIiIKLepPXKzaNEi6OnpYdu2bVi6dCmKFi0KANi3bx+aNGmi8QKJiIiI1KH2yE2xYsWwZ8+eTO1+fn4aKYiIiIjoc+ToPDdKpRLh4eGIjo6GUqlUWVanTh2NFEZERESUE2qHmzNnzqBz586IiIjIdCi3QqFAenq6xoojIiIiUpfa4aZfv36oWrUq9u7di8KFC0OhUORGXUREREQ5ona4uXfvHrZt2wYnJ6fcqIeIiIjos6h9tFS1atUQHh6eG7UQERERfTa1R24GDhyI4cOHIzIyEhUrVoS+vr7KcmdnZ40VR0RERKQutcPNTz/9BADo2bOn1KZQKCCE4IRiIiIi0rocXVuKiIiIKK9SO9wUL148N+ogIiIi0ogcncTv/v378Pf3x61btwAA5cqVw+DBg1GyZEmNFkdERESkLrWPlgoJCUG5cuVw7tw5ODs7w9nZGWfPnkX58uURGhqaGzUSERERZZvaIze//vorhg4dipkzZ2ZqHz16NH788UeNFUdERESkLrVHbm7duoVevXplau/Zsyf+/vtvjRRFRERElFNqhxsrKytcuXIlU/uVK1dgbW2tiZqIiIiIckzt3VJ9+vRB37598eDBA9SoUQMAcPLkScyaNQvDhg3TeIFERERE6lA73Pz2228oUKAA5s2bhzFjxgAAihQpgkmTJmHQoEEaL5CIiIhIHWqHG4VCgaFDh2Lo0KGIj48HABQoUEDjhRERERHlRI7OcwMA0dHRuHPnDgCgTJkysLKy0lhRRERERDml9oTi+Ph4dOvWDUWKFEHdunVRt25dFClSBF27dkVsbGxu1EhERESUbWqHm969e+Ps2bPYu3cvYmJiEBMTgz179uDChQvw9vbOjRqJiIiIsk3t3VJ79uxBSEgIatWqJbW5u7tj+fLlaNKkiUaLIyIiIlKX2iM3BQsWhJmZWaZ2MzMzWFhYaKQoIiIiopxSO9yMHz8ew4YNQ2RkpNQWGRmJkSNH4rffftNocURERETqUnu31NKlSxEeHo5ixYqhWLFiAIDHjx/D0NAQL168wLJly6S+ly5d0lylRERERNmgdrjx8PDIhTKIiIiINEPtcDNx4sTcqIOIiIhII9Sec/PkyRM8ffpUun/u3DkMGTIEgYGBGi2MiIiIKCfUDjedO3dGWFgYgPcTiRs1aoRz585h3LhxmDJlisYLJCIiIlKH2uHmxo0b+OGHHwAAW7ZsQcWKFXHq1CmsX78eQUFBmq6PiIiISC1qh5u0tDQYGhoCAA4ePIhWrVoBeH99qefPn2u2OiIiIiI1qR1uypcvj4CAABw/fhyhoaHSWYn/+ecfFCxYUOMFEhEREalD7XAza9YsLFu2DPXq1UOnTp1QqVIlAMDu3bul3VVERERE2qL2oeD16tXDy5cvERcXp3K5hb59+8LExESjxRERERGpS+2RGwAQQuDixYtYtmwZ4uPjAQAGBgYMN0RERKR1ao/cREREoEmTJnj8+DFSUlLw448/okCBApg1axZSUlIQEBCQG3USERERZYvaIzeDBw9G1apV8ebNGxgbG0vtbdq0waFDhzRaHBEREZG61B65OX78OE6dOgUDAwOVdgcHBzx79kxjhRERERHlhNojN0qlEunp6Znanz59igIFCmikKCIiIqKcUjvcNG7cGP7+/tJ9hUKBhIQETJw4Ec2aNdNkbURERERqU3u31Lx58+Du7o5y5cohOTkZnTt3xr1791CoUCFs3LgxN2okIiIiyja1R27s7Oxw9epVjBs3DkOHDoWrqytmzpyJy5cvw9raWq3HOnbsGFq2bIkiRYpAoVBg586d/9r/yJEjUCgUmW6RkZHqvgwiIiKSKbVHbgBAT08PXbp0QZcuXaS258+fY+TIkVi0aFG2HycxMRGVKlVCz5490bZt22yvd+fOHZiamkr31Q1VREREJF9qhZubN28iLCwMBgYGaN++PczNzfHy5UtMnz4dAQEBKFGihFpP3rRpUzRt2lStdYD3Ycbc3Fzt9YiIiEj+sr1bavfu3XB1dcWgQYPQr18/VK1aFWFhYShbtixu3bqFHTt24ObNm7lZq8TFxQWFCxfGjz/+iJMnT/5r35SUFMTFxanciIiISL6yHW6mTZsGHx8fxMXFYf78+Xjw4AEGDRqEv/76C/v375euDp6bChcujICAAGzfvh3bt2+Hvb096tWrh0uXLn1yHV9fX5iZmUk3e3v7XK+TiIiItCfb4ebOnTvw8fFB/vz5MXDgQOjo6MDPzw/ff/99btanonTp0vD29kaVKlVQo0YNrFq1CjVq1ICfn98n1xkzZgxiY2Ol25MnT75YvURERPTlZXvOTXx8vDSJV1dXF8bGxmrPsckNP/zwA06cOPHJ5YaGhjA0NPyCFREREZE2qTWhOCQkBGZmZgDen6n40KFDuHHjhkqfVq1aaa66bLhy5QoKFy78RZ+TiIiI8i61wo2Xl5fKfW9vb5X7CoUiy0szfEpCQgLCw8Ol+w8fPsSVK1dgaWmJYsWKYcyYMXj27BnWrl0LAPD394ejoyPKly+P5ORkrFixAocPH8aBAwfUeRlEREQkY9kON0qlUuNPfuHCBdSvX1+6P2zYMADvQ1RQUBCeP3+Ox48fS8tTU1MxfPhwPHv2DCYmJnB2dsbBgwdVHoOIiIi+bTk6iZ+m1KtXD0KITy4PCgpSuT9q1CiMGjUql6siIiKir5nal18gIiIiyssYboiIiEhWGG6IiIhIVhhuiIiISFZyFG5iYmKwYsUKjBkzBq9fvwYAXLp0Cc+ePdNocURERETqUvtoqWvXrqFRo0YwMzPDo0eP0KdPH1haWiI4OBiPHz+WzklDREREpA1qj9wMGzYM3bt3x71792BkZCS1N2vWDMeOHdNocURERETqUjvcnD9/PtOZiQGgaNGiiIyM1EhRRERERDmldrgxNDREXFxcpva7d+/CyspKI0URERER5ZTa4aZVq1aYMmUK0tLSALy/ntTjx48xevRo/PTTTxovkIiIiEgdaoebefPmISEhAdbW1khKSkLdunXh5OSEAgUKYPr06blRIxEREVG2qX20lJmZGUJDQ3HixAlcu3YNCQkJqFy5Mho1apQb9RERERGpJccXzqxVqxZq1aqlyVqIiIiIPpva4eb333/Psl2hUMDIyAhOTk6oU6cOdHV1P7s4IiIiInWpHW78/Pzw4sULvH37FhYWFgCAN2/ewMTEBPnz50d0dDRKlCiBsLAw2Nvba7xgIiIion+j9oTiGTNm4Pvvv8e9e/fw6tUrvHr1Cnfv3kW1atWwYMECPH78GLa2thg6dGhu1EtERET0r9QeuRk/fjy2b9+OkiVLSm1OTk6YO3cufvrpJzx48ACzZ8/mYeFERESkFWqP3Dx//hzv3r3L1P7u3TvpDMVFihRBfHz851dHREREpCa1w039+vXh7e2Ny5cvS22XL19G//790aBBAwDA9evX4ejoqLkqiYiIiLJJ7XCzcuVKWFpaokqVKjA0NIShoSGqVq0KS0tLrFy5EgCQP39+zJs3T+PFEhEREf0Xtefc2NraIjQ0FLdv38bdu3cBAKVLl0bp0qWlPvXr19dchURERERqyPFJ/MqUKYMyZcposhYiIiKiz5ajcPP06VPs3r0bjx8/Rmpqqsqy+fPna6QwIiIiopxQO9wcOnQIrVq1QokSJXD79m1UqFABjx49ghAClStXzo0aiYiIiLJN7QnFY8aMwYgRI3D9+nUYGRlh+/btePLkCerWrYt27drlRo1ERERE2aZ2uLl16xY8PT0BAHp6ekhKSkL+/PkxZcoUzJo1S+MFEhEREalD7XCTL18+aZ5N4cKFcf/+fWnZy5cvNVcZERERUQ6oPefGzc0NJ06cQNmyZdGsWTMMHz4c169fR3BwMNzc3HKjRiIiIqJsUzvczJ8/HwkJCQCAyZMnIyEhAZs3b0apUqV4pBQRERFpnVrhJj09HU+fPoWzszOA97uoAgICcqUwIiIiopxQa86Nrq4uGjdujDdv3uRWPURERESfRe0JxRUqVMCDBw9yoxYiIiKiz6Z2uJk2bRpGjBiBPXv24Pnz54iLi1O5EREREWmT2hOKmzVrBgBo1aoVFAqF1C6EgEKhQHp6uuaqIyIiIlKT2uEmLCwsN+ogIiIi0gi1w03dunVzow4iIiIijVB7zg0AHD9+HF27dkWNGjXw7NkzAMC6detw4sQJjRZHREREpC61w8327dvh7u4OY2NjXLp0CSkpKQCA2NhYzJgxQ+MFEhEREakjR0dLBQQEYPny5dDX15faa9asiUuXLmm0OCIiIiJ1qR1u7ty5gzp16mRqNzMzQ0xMjCZqIiIiIsoxtcONra0twsPDM7WfOHECJUqU0EhRRERERDmldrjp06cPBg8ejLNnz0KhUOCff/7B+vXrMWLECPTv3z83aiQiIiLKNrUPBf/111+hVCrRsGFDvH37FnXq1IGhoSFGjBiBgQMH5kaNRERERNmmdrhRKBQYN24cRo4cifDwcCQkJKBcuXLInz9/btRHREREpBa1d0v98ccfePv2LQwMDFCuXDn88MMPDDZERESUZ6gdboYOHQpra2t07twZf/3112ddS+rYsWNo2bIlihQpAoVCgZ07d/7nOkeOHEHlypVhaGgIJycnBAUF5fj5iYiISH7UDjfPnz/Hpk2boFAo0L59exQuXBg+Pj44deqU2k+emJiISpUqYfHixdnq//DhQzRv3hz169fHlStXMGTIEPTu3RshISFqPzcRERHJk9pzbvT09NCiRQu0aNECb9++xY4dO7BhwwbUr18fdnZ2uH//frYfq2nTpmjatGm2+wcEBMDR0RHz5s0DAJQtWxYnTpyAn58f3N3d1X0pREREJENqh5sPmZiYwN3dHW/evEFERARu3bqlqbqydPr0aTRq1Eilzd3dHUOGDPnkOikpKdIlIgAgLi4ut8ojIiKiPCBHF858+/Yt1q9fj2bNmqFo0aLw9/dHmzZtcPPmTU3XpyIyMhI2NjYqbTY2NoiLi0NSUlKW6/j6+sLMzEy62dvb52qNREREpF1qh5uOHTvC2toaQ4cORYkSJXDkyBGEh4dj6tSpKFOmTG7U+FnGjBmD2NhY6fbkyRNtl0RERES5SO3dUrq6utiyZQvc3d2hq6ursuzGjRuoUKGCxor7mK2tLaKiolTaoqKiYGpqCmNj4yzXMTQ0hKGhYa7VRERERHmL2uFm/fr1Kvfj4+OxceNGrFixAhcvXvysQ8P/S/Xq1fHXX3+ptIWGhqJ69eq59pxERET0dcnRnBvg/TlqvLy8ULhwYcydOxcNGjTAmTNn1HqMhIQEXLlyBVeuXAHw/lDvK1eu4PHjxwDe71Ly9PSU+vfr1w8PHjzAqFGjcPv2bSxZsgRbtmzB0KFDc/oyiIiISGbUGrmJjIxEUFAQVq5cibi4OLRv3x4pKSnYuXMnypUrp/aTX7hwAfXr15fuDxs2DADg5eWFoKAgPH/+XAo6AODo6Ii9e/di6NChWLBgAezs7LBixQoeBk5ERESSbIebli1b4tixY2jevDn8/f3RpEkT6OrqIiAgIMdPXq9ePQghPrk8q7MP16tXD5cvX87xcxIREZG8ZTvc7Nu3D4MGDUL//v1RqlSp3KyJiIiIKMeyPefmxIkTiI+PR5UqVVCtWjUsWrQIL1++zM3aiIiIiNSW7XDj5uaG5cuX4/nz5/D29samTZtQpEgRKJVKhIaGIj4+PjfrJCIiIsoWtY+WypcvH3r27IkTJ07g+vXrGD58OGbOnAlra2u0atUqN2okIiIiyrYcHwoOAKVLl8bs2bPx9OlTbNy4UVM1EREREeXYZ4WbDLq6uvDw8MDu3bs18XBEREREOaaRcENERESUVzDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs5Ilws3jxYjg4OMDIyAjVqlXDuXPnPtk3KCgICoVC5WZkZPQFqyUiIqK8TOvhZvPmzRg2bBgmTpyIS5cuoVKlSnB3d0d0dPQn1zE1NcXz58+lW0RExBesmIiIiPIyrYeb+fPno0+fPujRowfKlSuHgIAAmJiYYNWqVZ9cR6FQwNbWVrrZ2Nh8wYqJiIgoL9NquElNTcXFixfRqFEjqU1HRweNGjXC6dOnP7leQkICihcvDnt7e7Ru3Ro3b978ZN+UlBTExcWp3IiIiEi+tBpuXr58ifT09EwjLzY2NoiMjMxyndKlS2PVqlXYtWsX/vjjDyiVStSoUQNPnz7Nsr+vry/MzMykm729vcZfBxEREeUdWt8tpa7q1avD09MTLi4uqFu3LoKDg2FlZYVly5Zl2X/MmDGIjY2Vbk+ePPnCFRMREdGXpKfNJy9UqBB0dXURFRWl0h4VFQVbW9tsPYa+vj5cXV0RHh6e5XJDQ0MYGhp+dq1ERET0ddDqyI2BgQGqVKmCQ4cOSW1KpRKHDh1C9erVs/UY6enpuH79OgoXLpxbZRIREdFXRKsjNwAwbNgweHl5oWrVqvjhhx/g7++PxMRE9OjRAwDg6emJokWLwtfXFwAwZcoUuLm5wcnJCTExMZgzZw4iIiLQu3dvbb4MIiIiyiO0Hm46dOiAFy9eYMKECYiMjISLiwv2798vTTJ+/PgxdHT+N8D05s0b9OnTB5GRkbCwsECVKlVw6tQplCtXTlsvgYiIiPIQrYcbABgwYAAGDBiQ5bIjR46o3Pfz84Ofn98XqIqIiIi+Rl/d0VJERERE/4bhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkheGGiIiIZIXhhoiIiGSF4YaIiIhkJU+Em8WLF8PBwQFGRkaoVq0azp0796/9t27dijJlysDIyAgVK1bEX3/99YUqJSIiorxO6+Fm8+bNGDZsGCZOnIhLly6hUqVKcHd3R3R0dJb9T506hU6dOqFXr164fPkyPDw84OHhgRs3bnzhyomIiCgv0nq4mT9/Pvr06YMePXqgXLlyCAgIgImJCVatWpVl/wULFqBJkyYYOXIkypYti6lTp6Jy5cpYtGjRF66ciIiI8iKthpvU1FRcvHgRjRo1ktp0dHTQqFEjnD59Ost1Tp8+rdIfANzd3T/Zn4iIiL4tetp88pcvXyI9PR02NjYq7TY2Nrh9+3aW60RGRmbZPzIyMsv+KSkpSElJke7HxsYCAOLi4j6n9E9SprzNlcelr0dubVvZxW2QuA2StuXGNpjxmEKI/+yr1XDzJfj6+mLy5MmZ2u3t7bVQDX0LzPy1XQF967gNkrbl5jYYHx8PMzOzf+2j1XBTqFAh6OrqIioqSqU9KioKtra2Wa5ja2urVv8xY8Zg2LBh0n2lUonXr1+jYMGCUCgUn/kK6ENxcXGwt7fHkydPYGpqqu1y6BvEbZC0jdtg7hFCID4+HkWKFPnPvloNNwYGBqhSpQoOHToEDw8PAO/Dx6FDhzBgwIAs16levToOHTqEIUOGSG2hoaGoXr16lv0NDQ1haGio0mZubq6J8ukTTE1N+UtNWsVtkLSN22Du+K8Rmwxa3y01bNgweHl5oWrVqvjhhx/g7++PxMRE9OjRAwDg6emJokWLwtfXFwAwePBg1K1bF/PmzUPz5s2xadMmXLhwAYGBgdp8GURERJRHaD3cdOjQAS9evMCECRMQGRkJFxcX7N+/X5o0/PjxY+jo/O+grho1amDDhg0YP348xo4di1KlSmHnzp2oUKGCtl4CERER5SEKkZ1px0TZkJKSAl9fX4wZMybTrkCiL4HbIGkbt8G8geGGiIiIZEXrZygmIiIi0iSGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYbIiIikhWGGyIiIpIVhhsiIiKSFYYb+mYolUptl0BERF8Aww19MzIuwPry5UsAAK88Ql/axwGb2yBpw8fboRy/+DHc0DdlwYIF8PDwwP3796FQKLRdDn1jdHR0EBsbi5CQEADgNkhaoaOjg5iYGMyZMwdv3ryRvvjJifxeEdEHPv5mrK+vD2NjYxgYGGipIvqWKZVKzJs3D97e3tizZ4+2y6Fv2IEDBzB//nwsWrRI26XkCl4VnL4JcXFxMDU1BQDExsbCzMxMyxXRt0KpVKp8M7516xZWrlyJWbNmQVdXV4uV0bckPT1dZXtLS0vD5s2b0alTJ1luhww3JHtDhw5Feno6xowZg8KFC2u7HPoGxcTEICYmBvb29iofJB9/4BB9jo+D9MdevXqFkydPokaNGihUqJDULsftkLulSHY+zut2dnZYu3at7H556esghMCvv/6KatWq4dGjRyrLuE3S53j+/Dn++ecfvHjxAsD7uTT/Nl6xZcsWeHh44OjRoyrtctwOOXJDX7WMbxxCCCgUik9+c3nz5g0sLCy0UCHJzX99O86qT0REBMaPH4+goCBZfpDQl7d69WosXrwYT548QcmSJVGrVi3Mnj1bpU9WIzL+/v4YMGAA9PT0vmS5XxzDDX01MgIM8P6XVggBPT09PHv2DDt27ECPHj2QL18+AO93RVlYWGDChAmZ1iXKqQ9Dy+HDh/H48WM4OTmhRIkSKFKkiEqf2NhYKJXKTKFajrsA6Mvas2cP2rdvjyVLlsDExAQPHjzA7NmzUaNGDaxZswYFCxaU/ua9fPkS4eHhcHNzU3mMd+/eyTrgcLcU5VkZuTsuLg5JSUlQKBQ4cOAAwsPDoaurCz09PURERMDV1RX//POPFGwSExOhr68PPz8/vH79msGGNEIIIQWbX3/9Fd27d8fcuXPRt29fjBgxAufPnwfwftdASkoKJkyYgMqVK+PVq1cqj8NgQ5/r/PnzaN68Obp374727dtj1KhRCAkJwbVr19ClSxcA708zkJaWhnXr1qFGjRo4ceKEymPIOdgADDeUx0VGRqJixYo4evQoNmzYgCZNmuDvv/8G8H5XU/ny5dGmTRtMnz5dWidfvnwYNWoU7t27B0tLSwYb0oiM7Wju3Ln4448/sHHjRty4cQNt27bFn3/+ifHjx+P06dMAAAMDA7i6uqJhw4YwNzfXYtUkRw8fPsTz589V2r7//nvs3r0bFy9eRJ8+fQC8P/VFixYtMH369EwjN7IniPK4Hj16CFNTU6GjoyOWL18utaemporNmzeL9PR0qU2pVGqjRPpGREVFibZt24pVq1YJIYTYvXu3MDU1Ff369ROurq6iYcOG4syZM0II1W3x3bt3WqmX5CkkJETY2NiITZs2SW0Z29v69euFk5OTOH/+fKb10tLSvliN2saRG8qzMk4J7uPjg/j4eBgYGMDW1hbJyckA3n8rad++vcrETY7SUG6ytrbGqFGj0KRJE1y+fBk+Pj6YNm0ali5dip9++glnzpyBj48PLl68qLItclcUaVLZsmVRr149rFu3DocOHQLwv799Li4uiI6Oli4z8yG574r6EMMN5VkZocXe3h4nTpyAl5cXOnbsiF27diEpKSlTfzleH4W051Pbk6urKwoXLox9+/bB2dkZffv2BQBYWlrCzc0NLVu2hKur65cslb4x9vb26NevH2JiYuDn54fdu3dLywoXLgxHR0ctVpc3fDsxjr4a4v8nAD9//hxpaWkoVqwYrK2tUaNGDSQnJ6NXr14ICgpCixYtYGRkhICAADRq1AhOTk7aLp1kQnwweXjFihWIjo6GgYEBRowYIV26IyUlBc+ePcOjR49QunRpHDhwAK1atcLAgQP/9bQERJ8j42i7evXqYcmSJRg7dixGjx6NkJAQODs7Y8uWLVAoFPjxxx+1XapW8VBwypOCg4MxadIkREVFoXnz5mjTpg1atmwJAOjRowd27NiB4cOHIyoqCkuXLsX169dRrlw5LVdNcjNx4kT4+/vj+++/x7lz51CtWjWsW7cOtra2+PPPPzFt2jS8efMG+vr6EELg2rVr0NPT4xF6lCsytqvg4GAsWbIEBw4cwO3btxEWFoZFixbB3t4e5ubmWL9+PfT19b/p0w4w3FCec/PmTbi7u2Po0KEwMTHBxo0bYWhoCC8vL3Tt2hUAMHjwYFy6dAkpKSkIDAyEi4uLdosmWfhwtOXdu3fw8vLCwIED4erqikePHqF58+awtbXFjh07YGVlhb179yI8PBwJCQkYPXo09PT0vukPFNKMjBAjPjq3l66uLoKDg+Hp6Yn58+dLu0SB99urjo6Oyvb7Lc2x+RjDDeUpt2/fxtatW5GUlIQZM2YAAK5fv44JEyYgLi4OPXr0kAJOZGQk8uXLhwIFCmizZJKJD4PNrVu3EBcXh2XLlmHChAlwcHAA8P4Q3B9//BE2NjbYuXMnrKysVB6DwYY+14fb4cuXL6FQKFCwYEEA7//mVa5cGRMmTEC/fv2kdT4eKeTIIcMN5RFCCLx58wYtWrTA33//jZYtW2LdunXS8mvXrmHChAlISkpCx44d0aNHDy1WS3I2cuRIaVg/KioKwcHBaNq0qfRh8fDhQzRt2hRCCJw8eVLlAoREn+PDUDJ16lTs3LkTcXFxKFSoEKZPn44GDRrg2bNnKFq0qJYrzfs4243yBIVCAUtLS/j6+qJ8+fK4dOkSQkNDpeXOzs6YOnUq0tLSpF94Ik348KioPXv2YP/+/fj999+xZMkSODo6Yty4cbh69ap0xmxHR0fs2bMHLi4uvF4ZaVRGsJkyZQoWLFggnWqgUKFC6NKlC9asWZNptJCyxpEb0ppPDZ0ePXoUY8eOha2tLXx8fNCgQQNp2c2bN2FmZgY7O7svWSp9A4KDg3Hq1CkULFgQY8aMAQAkJCSgcuXKMDU1xYoVK1CpUqVM2yx3RZEmvXr1Co0bN4aPjw969uwptfft2xd//vknwsLCUKZMGe56+g8cuSGtyPjFPHXqFObPn4/ffvsNJ0+eRFpaGurWrYspU6YgMjISixYtwpEjR6T1ypcvz2BDGpeUlITffvsN8+fPx82bN6X2/Pnz49KlS4iPj4e3t7d0/agPMdiQJr179w4vX76URgUzTloaGBiIIkWKwM/PDwBPWPpfGG7oi/vwcMamTZvi5MmT2L17N8aOHYvp06cjNTUVDRs2xJQpU/Dq1StMnToVx48f13bZJGPGxsY4fvw4GjVqhIsXL2L37t1IT08H8L+Ac/v2bSxbtkzLlZKcZLXjxMbGBra2tli1ahUAwMjICKmpqQAAJycnhppsYrihLy5jxGbQoEGYP38+tm/fjq1bt+LixYvYvHkzxo8fLwWcX3/9Ffr6+jzjJmnMh3NshBDSB4ylpSU2bNgACwsLzJkzByEhIdKyfPnyITIyEoGBgVqpmeRHqVRKQeWff/5BdHQ03r59CwCYNGkSbt++LR0RlXHiyKdPn/JCrNnEOTf0xWT8MisUCixZsgRXrlxBYGAgHj58iEaNGqFWrVowNTXF1q1b4e3tjbFjx8LQ0BBv376FiYmJtssnGfjwMNuFCxfi6tWrePDgAYYMGYLKlSvDzs4OL168QOvWraGrq4uxY8fC3d1d5UzDnGNDn2P9+vVwc3NDyZIlAQBjxoxBSEgIIiIi0KhRI7Rq1QpdunTB8uXLMXXqVBQsWBAVKlTA/fv3ERMTI50okv4dww3lmowPkg/DyZUrV+Di4oK4uDg8efIETk5OaNKkCRwdHbFq1SrExsZKZxru3r07pk+fzolz9Nk+3obGjBmDlStXom/fvnj69ClOnz6N1q1bo2/fvnBycsKLFy/Qtm1bvHjxAkFBQXBzc9Ni9SQX+/btQ4sWLTB69GgMGTIE+/btw6hRo+Dv749Xr17h0qVLCAkJwW+//YZ+/frh+vXr8Pf3h46ODiwsLDBjxgyeKDK7cvWa4/TNe/DggejUqZP4+++/xZYtW4RCoRDnzp0TSqVSCCHE9evXRZkyZcTZs2eFEELcv39ftGjRQowdO1Y8fvxYm6WTzKSnpwshhFi3bp1wdHQUFy9eFEIIcfz4caFQKESpUqXE4MGDxYMHD4QQQjx//lz07dtXvHv3Tms1k/wsWrRI2NnZialTp4oBAwaI5cuXS8uePHkipkyZIhwcHMT+/fuzXD8tLe1LlfpV49gW5ark5GQcP34c3bt3x5UrV7B69Wp8//330i4qIQTevXuH06dPo3z58li7di0AYMSIETyHCH22bt26wcrKCvPnz4eOjg7S0tJgYGCAfv36oXLlyti5cyd69OiBFStWIDIyEtOmTYOOjg769OmDsmXLShOI+U2ZPldqaioMDAzg4+MDExMTjBkzBvHx8Zg2bZrUx87ODp6enjhw4AAuXLgAd3f3TBdg5S6pbNJ2uiL5yvimHBAQIHR0dESlSpXE5cuXVfrExsaK7t27i5IlSwoHBwdhZWUlfaMm+hyxsbFi8uTJwtLSUkyaNElqf/bsmYiKihLPnz8XVatWFfPmzZP6FylSRBQuXFgsWLBACCGkEUYiTfH19RXR0dFi/fr1wsTERDRr1kzcvXtXpU+HDh1E27ZttVShPPBoKcoVQgjo6OhACIEiRYpg3rx5ePfuHcaPH48TJ05I/UxNTTF37lwsWbIEEydOxNmzZ1G5cmUtVk5yEB8fD1NTU/Tv3x/jx4+Hv78/Jk6cCAAoUqQIrK2t8fz5c7x580aaT/Ps2TM0btwYEyZMgI+PDwCeS4Q+n/hgWuuaNWswdepU3Lt3D507d4afnx8uXbqEgIAA3LlzBwAQFxeHhw8folixYtoqWRY4vkUaJ/5/8ubhw4dx9OhRDBkyBC1btkSjRo3Qvn17zJw5E2PHjkWNGjUAvL8wZuPGjbVcNcnFqFGjsGzZMty/fx9WVlbo2rUrhBCYOnUqAGDy5MkA3gcgXV1dnDx5EkIIzJw5EyYmJtLht9wVRZqQEZAPHTqEy5cvIzAwUPrb17dvX6SlpWHy5MnYv38/KleujMTERKSmpmL27NnaLPvrp81hI5KfjGH8bdu2CTMzMzFmzBhx/vx5afm1a9dEuXLlRIsWLcQff/whJk2aJBQKhXjy5Al3AZBGXL16VdSpU0eULl1avHjxQgghRHR0tJg3b54wNzcXEyZMkPoOGDBAlCxZUtjZ2Qk3NzeRmpoqhODuKNKsI0eOiIoVK4qCBQuKnTt3CiGESElJkZavXLlS5M+fX1SuXFmsXbtWmsTOycM5x0PBSePOnTuHJk2aYNasWejTp4/UHhcXB1NTU9y6dQt9+vRBUlISYmNjsWXLFu6KIo04ffo0Xrx4gXLlyqFDhw5ISEiQrtz94sULrFu3DlOnTpUuSAi8Pz2BQqFAxYoVoaOjg3fv3nHSJn0W8dGpBxISEjBnzhwEBgaiWrVq2LhxI4yNjZGWlgZ9fX0AwPz583Hq1Cls3boVCoWCI4efieGGNG7RokXYsWMHDh06hNjYWBw+fBh//PEHbt26hREjRqBnz56Ijo5GbGwszMzMYG1tre2SSSY8PT3xzz//4ODBg3j06BF+/vlnxMfHZwo406ZNw4ABAzBlyhSV9fmBQpq0ePFi2NnZoXXr1khKSsLcuXOxY8cO1KtXDzNmzICRkZFKwMkIRR+HI1IfJxSTxtna2uLixYvw9fXFzz//jNWrV8PIyAjNmzdH7969cffuXVhbW6NUqVIMNqRRixcvxtOnT7Fo0SI4ODhg48aNMDMzQ82aNfHy5UtYWVmhW7dumDBhAqZNm4aVK1eqrM9gQ5ry4sULHD58GL/88gv2798PY2NjDBs2DC1atMCpU6cwbtw4JCcnQ19fH+/evQMABhsN4sgNfZaMX8SEhATkz58fABAVFYWFCxdiy5YtaNCgAbp3744ffvgBUVFRaNWqFYKCglC+fHktV05ykzHq8vvvv+Py5cuYP38+LCwscPv2bXh6eiI2NlYawYmMjMTRo0fx008/cRcUacTH56MBgKtXr+L333/HwYMHERAQgKZNmyIxMRGzZ8/GwYMHUbZsWSxZskS6dhRpDkdu6LMoFArs3bsXnTp1Qr169RAUFAQ9PT1MmzYNZ8+eRUBAANzc3KCjo4OFCxciMTGRozWUKzJGXerVq4djx45h7969AIDSpUtj3bp1sLCwQJ06dRAVFQVbW1t06NABenp60rdmos+REWwiIyOltkqVKmHw4MGoX78++vXrh/379yNfvnwYNWoUfvjhB+jo6Ei7pEjDtDSRmWTi5MmTwsjISIwcOVI0adJEODs7C29vbxEeHi71CQsLE3379hWWlpaZTuJHlFMZJ4nMSkBAgPjuu+/EnTt3pLY7d+4IBwcH0bFjxy9RHn0jPtwON23aJEqUKKFyhKgQQly5ckW0bt1aFCtWTBw5ckQIIURSUpJ0VN6/bcuUMxy5oRyLiIhAaGgopk+fjtmzZ2Pfvn3o27cvrl27Bl9fXzx48ACJiYk4ffo0oqOjcfToUbi4uGi7bJKBD3cBnDt3DqdOncLRo0el5a1atUK1atUQFhYmtX333Xc4duwY/vjjjy9eL8lTSkqKtB2mpqaiZMmSKFOmDHx8fHDx4kWpX6VKleDh4YEnT56gcePGOHXqFIyMjKQ5Nh/vzqLPx58oZcuiRYvw119/Sffv3LmDDh06YNWqVTAyMpLafXx80KVLF9y8eROzZ89GTEwMRo4ciTVr1qBChQraKJ1k5sMPg7Fjx6J79+7o2bMnvLy80KFDB8TFxaFw4cLSfIa0tDRpXXt7e+jq6iI9PV1b5ZNM7Nu3D+vWrQMA9OnTBw0aNEDVqlUxfPhw2NrawtvbGxcuXJD6FytWDB07dsS8efNQrVo1qZ2Th3OJtoeOKO97+PCh6Ny5s7h3755K+6+//iqsra1F27ZtpZOlZVi6dKkoXbq0GDRoEE9ERbli7ty5omDBguLs2bMiPT1dzJgxQygUCnHixAmpT82aNYW3t7cWqyS56tSpk3BwcBDu7u6iUKFC4urVq9Kyw4cPCw8PD1GhQgWxb98+8fDhQ+Hh4SGGDx8u9eHV5nMXww1lS2JiohBCiDNnzoht27ZJ7RMmTBAVK1YU48ePF1FRUSrrLF++XDx8+PBLlknfCKVSKby8vERgYKAQQojt27cLc3NzERAQIIQQIj4+XgghxL59+0SrVq3EtWvXtFYryZeLi4tQKBQqF2bNcPz4cdGtWzehUCjEd999J5ydnaUvejwDdu7jMZCULcbGxoiJiYGvry+ePXsGXV1deHh4YPLkyUhLS8PevXshhMDgwYNhZWUFAOjdu7eWqya5Sk5OxtmzZ1GvXj0cOXIEXl5emDNnDry9vfHu3TvMnj0b1atXh5ubG6ZMmYJz586hYsWK2i6bZCI1NRXJyclwcnJCsWLFsHnzZhQtWhQdO3aUTolRq1YtVKtWDX369EFaWhrq1q0LXV1dngH7C+GcG8oWhUIBc3NzDB8+HI6OjvD390dwcDAAYMaMGWjSpAlCQ0MxY8YMvHz5UsvVkpxcu3YNT58+BQAMHToUR48ehbGxMTp37ow//vgDzZo1g5+fn3TByzdv3uDChQu4c+cOLCwssG7dOhQvXlybL4FkxsDAAKampti6dSt27dqF77//HrNnz8amTZsQHx8v9UtOTkbt2rXRoEEDaa4Xg82XwXBD2SLe78JE7dq1MXToUFhYWOD3339XCThubm64fPkyBM8LSRoghMDdu3dRv359rFq1Cv369cOCBQtgYWEBAHBzc0NERASqVauG6tWrAwD++ecfdO/eHTExMRgwYAAAoGTJkmjUqJHWXgfJjxACSqVSur9mzRrUqFEDfn5+WLt2LR4/fowGDRqgXbt2Un+AZ8D+kniGYsqWjLO/xsbGwsTEBNeuXcP06dPx5s0bDB48GB4eHgDen3I8Y7cUkSYsX74co0aNQnJyMnbt2oXGjRtLZ8bevHkzpkyZAiEE9PT0YGxsDKVSiVOnTkFfX5/XiqLP9vr1a1haWqq0ZWx/W7duRWhoKAIDAwEAffv2xZEjR5Ceng5LS0ucPHmSZx/WEo7c0H969+4ddHV18ejRI9SrVw8HDhxAlSpVMGLECFhZWWHy5MnYs2cPADDYkMZkfDO2t7eHoaEhTE1NcebMGTx69Eg6fLZDhw5Yu3YtpkyZgvbt22P06NE4c+aMdL0eBhv6HAsWLMD333+vsqsJgBRsunfvjkqVKkntgYGBWLZsGRYuXIgzZ87AwMCAZ8DWFu3MY6a86lOz+MPDw4WNjY3o3bu3yiGMR44cEd26dROPHj36UiWSzH28DaampoqkpCSxdOlSUbRoUTF27Nj/3N54mC19rmXLlglDQ0OxYcOGTMseP34sKlasKBYtWiS1ZbXNcTvUHu6WIon4/6HW06dP49atWwgPD4enpycKFy6MNWvW4MKFC1izZk2mK9cmJyernMiPKKc+PPPw69evER8frzIZ2N/fH3PnzkWvXr3Qo0cPODg4oGXLlhg3bhzc3Ny0VTbJzPLlyzFw4ECsW7cO7dq1Q0xMDBITE5GcnAxra2sUKFAA9+7dQ6lSpbRdKn0Cww2p2L59O/r27StdYPDFixfo0KEDRo8ejQIFCmi7PJKxD4PNlClTcODAAdy4cQPt27dHmzZt0LRpUwDvA46/vz8qVKiAV69e4fHjx3j06BEvQEga8eDBAzg5OaF9+/bYtGkTbty4gV9++QUvXrxAREQE6tevj/79+6NFixbaLpX+BY9JI8mNGzcwdOhQzJs3D927d0dcXBzMzc1hbGzMYEO5LiPYTJgwAYGBgZgzZw4cHBzQr18/3Lt3DzExMejUqROGDBmCQoUK4erVq0hOTsbx48elq3vzMFv6XFZWVpg1axYmTJiAESNG4MCBA6hduzZat26NuLg4bNu2DePHj0ehQoU4WpiXaXOfGGnP4cOHxf379zO1Va9eXQghxK1bt0Tx4sVF7969peX379/nPmTKVYcPHxbly5cXx44dE0IIcerUKWFgYCDKlSsnqlWrJrZu3Sr1/fCyHrzEB2lScnKymDt3rtDR0RE9e/YUqamp0rILFy6I0qVLi8WLF2uxQvovPFrqGyOEwOXLl9G0aVMsXboUERER0rJnz55BCIGEhAQ0adIEjRs3xrJlywAAoaGhWLp0Kd68eaOt0kmGxEd7xYsWLYr+/fujdu3aOHDgAFq0aIHAwECEhobi/v37+P3337Fy5UoAUBml4YgNaZKhoSH69euH7du3o3fv3tDX15e21SpVqsDIyAhPnjzRcpX0bxhuvjEKhQKurq6YN28etmzZgqVLl+LBgwcAgObNmyMqKgqmpqZo3rw5AgMDpV0FISEhuHbtGg+tJY1RKpXSpPQHDx4gMTERpUqVQqdOnZCcnIwFCxZg0KBB6NatG4oUKYLy5csjPDwct27d0nLl9C3Ily8fmjZtKp0gMmNbjY6OhrGxMcqXL6/N8ug/8OvONyZjXoKPjw8AYM6cOdDV1UXv3r3h6OiI3377DTNmzMC7d+/w9u1bhIeHY+PGjVixYgVOnDghnR2W6HN8OHl4woQJOH36NEaOHIn69evD0tISiYmJeP78OUxMTKCjo4OUlBQ4ODhg1KhRaNKkiZarJzkSHxwBmsHQ0FD6f3p6Ol6+fIk+ffpAoVCgU6dOX7pEUgPDzTcmY+TlwIED0NHRQVpaGvz9/ZGcnIzRo0ejffv2SEpKwowZM7Bt2zbY2NjAwMAAYWFhqFChgparJ7n4MNgsW7YMgYGBcHV1lY54SklJgaWlJU6cOCFNGn716hVWrVoFHR0dlXBElBMRERF4/fo1ChYsCFtb2389k3BaWhrWrVuHjRs34vXr1zhz5ox0rSiOZudNPBT8GxQSEiJdbDBfvny4d+8efv/9d/zyyy8YPXo0rKysEB8fj6NHj8LBwQHW1tawtrbWdtn0lfs4kNy9exceHh6YNWsWWrZsmanf+fPnMX78eCQkJMDS0hLBwcHQ19dnsKHPtnbtWsybNw/R0dEoVKgQBg4cKI3IZPh4OwsNDcXNmzcxYMAAHp33FWC4+cYolUp06dIFCoUCGzZskNoXLlyIUaNGwcfHB7/88gtKlCihxSpJbtq2bYuxY8eiatWqUtuVK1fQpEkTHD16FKVLl87yxJDJyckQQsDIyAgKhYIfKPTZ1q5dCx8fH+nSCjNmzMCDBw9w8uRJadvKCDYxMTE4cOAA2rdvr/IYHLHJ+/j15xuT8U0kY/g/NTUVADBw4EB4e3tj9erV+P3331WOoiL6XGZmZnB2dlZpMzIywps3b3Djxg2pLeN6UqdPn8b27duho6MDY2NjKBQKKJVKBhv6LBcuXMDUqVOxaNEi9OzZExUrVsTQoUPh5OSEU6dO4ebNm4iLi5N22a9Zswa//PIL/vjjD5XHYbDJ+xhuvhH//POP9P/SpUvjzz//RHR0NAwMDJCWlgYAsLOzg4mJCcLCwmBsbKytUklGnj17BgBYvXo1DAwM8Pvvv+PAgQNITU2Fk5MTOnTogDlz5uDgwYNQKBTQ0dFBeno6pk+fjrCwMJV5ENwVRZ8rJSUFQ4YMQfPmzaW2SZMm4dChQ+jUqRM8PT3RsWNHvH79Gvr6+mjWrBlGjBjBycNfIe6W+gZcvXoVAwYMQOfOndG/f3+kpqaiQYMGePnyJY4cOQJbW1sAwOjRo1G+fHm0aNEClpaWWq6avnZ9+vQBAIwZM0bazens7IyXL19i06ZNqFOnDo4fPw4/Pz9cv34dXbp0gYGBAQ4dOoQXL17g0qVLHKkhjVIqlXjx4gVsbGwAAJ6enjh48CB2794Ne3t7HD16FNOmTcPo0aPRuXNnlTk43BX1deFXoW+AiYkJzM3NsW3bNgQFBcHAwADLli2DlZUVypYtCw8PDzRu3BgLFixA1apVGWxII5ydnbF//34sXboU4eHhAIBr166hdOnS6NKlC44dO4batWtjypQp8PT0xLp163D48GEUK1YMFy9elCZtEmmKjo6OFGwAYMSIETh79iyqVq0KGxsbNG3aFK9fv0ZUVFSmw8IZbL4uHLn5RoSHh2Ps2LGIjIxEnz590K1bN6Snp2Pu3LmIiIiAEAIDBw5EuXLltF0qyciqVaswYcIEdOzYEX369EHp0qUBAHXq1MHDhw+xfv161KlTBwDw9u1bmJiYSOty8jB9aU+fPkXXrl0xYsQIXhjzK8dwI1OXLl3C8+fPVfYth4eHY/z48Xj06BEGDhyILl26aLFCkrMPD6NduXIlJkyYgE6dOmUKOBEREVi7di2qV6+uMr8mqxOqEanjw20o4/8Z/7548QJWVlYq/RMTE9GpUyfExsbi8OHDHKn5yjHcyFB8fDyaN28OXV1djBo1Ck2bNpWWPXr0CE2aNIGJiQl69+6NX375RYuVktx86hw0y5cvx+TJk9GhQwf07dtXCjgNGjTAyZMncebMGbi6un7pckmmstoOM9qCg4OxceNGLFiwAEWKFEFSUhJ27dqFdevW4dmzZzh//jz09fU5x+Yrxzk3MpKRUwsUKIDZs2dDT08PixYtwt69e6U+Dg4OqF+/PiIjI3Ho0CHExMRoqVqSmw8/UE6dOoWwsDBcvXoVwPvJxb/99hs2bdqEwMBA3LlzBwBw+PBh9O7dO9Nh4kQ5deLECemilsOGDcPMmTMBvJ9vs3nzZnh6eqJRo0YoUqQIgPcXXX348CFKlCiBCxcuQF9fH+/evWOw+cpx5EYGMoZaM75pZHzInD17Fr/++ivy5cuH/v37S7uohg8fjhIlSqBt27YoXLiwlqsnOfhwF8CwYcOwefNmJCQkwM7ODsWKFcO+ffsAAMuWLcO0adPQsWNHeHl5qVzSg9+U6XMIIRAbGwtra2s0bdoUhQoVQnBwMI4fP44KFSogJiYGbm5u8PHxwcCBA6V1PvzbCXA7lAuGm69cxi9nWFgYdu/ejdevX6NWrVpo164dzM3NcebMGfz2229ISUlBiRIlYGJigs2bN+Pq1auws7PTdvkkAx8GmwMHDmDIkCEIDAyEubk5/v77b0ycOBH58uXDhQsXALyfg+Pt7Q1/f38MGDBAm6WTDEVHR6NEiRJIT0/H9u3b0axZM2lZVnNtspqbQ18/7pb6yikUCuzYsQMtW7bE27dv8fbtW6xbtw79+/fH69ev4ebmhrlz56Ju3boIDw/HgwcPcPjwYQYb0piMD4Pdu3dj06ZNaNSoEWrVqoUKFSrg559/xtq1a5GQkID+/fsDAHr16oVdu3ZJ94k0JSUlBZGRkTAxMYGuri5WrVolnYYAAAoVKiT9P+Ns2B+GGQYb+eDIzVfuwoUL6NixI3799Vf07t0bERERqFy5MoyNjeHi4oK1a9fC0tJSulbPx4fbEmnC69ev0aJFC1y9ehX169fHnj17VJaPHTsWJ0+exF9//YV8+fJJ7dwFQJ/rU5PYHz16BGdnZ9SvXx/z589HyZIltVAdaQtHbr4ivr6+GDdunPSNA3h/ens3Nzf07t0bjx49QsOGDeHh4YHx48fj/Pnz+OWXX/D69WsYGRkBAIMNacSH2yAAWFpaYs2aNfjxxx9x+fJlrF69WmV5qVKl8OrVKyQlJam0M9jQ5/gw2Bw5cgQbNmzA1atX8ezZMzg4OODkyZMICwvDqFGjpEnsbdq0wcKFC7VZNn0BHLn5iixcuBCDBw/GjBkzMGrUKOmX+tatWyhdujRat24tfcgolUq4uLggPDwczZs3x+bNm3ltHtKIDz9Q7t+/D4VCARMTE9ja2uLhw4fw8fFBYmIi2rVrB29vb0RFRcHLywtGRkbYs2cPh/5J40aMGIE1a9ZAT08P+fPnh62tLfz8/FC1alVcv34d9evXh4ODA1JTU/Hu3TtcvXpVungwyZSgr4JSqRRCCLF8+XKho6Mjpk6dKtLS0qTlT548EWXLlhV79uwRQgjx+vVr0alTJ7Fw4ULx9OlTrdRM8pOxHQohxMSJE0XFihVFmTJlROHChUVgYKAQQojw8HDRrFkzYWRkJEqXLi3atGkj3N3dRVJSkhBCiPT0dK3UTvLx4XYYGhoqKlWqJI4fPy5ev34tdu3aJdq0aSOcnJzEpUuXhBBC3Lt3T0yZMkVMnz5d+rv54d9Pkh+Gm6+AUqmUfpmVSqX4448/hI6Ojpg2bZr0QREdHS1cXFyEt7e3ePTokRg7dqz4/vvvRVRUlDZLJ5maMmWKsLKyEiEhISIhIUG0adNGmJubi5s3bwohhHjw4IFo3ry5cHFxEX5+ftJ6ycnJWqqY5GjNmjViwIABom/fvirt58+fF02aNBFeXl4iISFBCKEaiBhs5I/7Kb4SCoUCBw8exPDhw1GlShXpmj0zZ86EEAIWFhbo0qULjh49Cjc3N6xduxYBAQGwtrbWdukkAx/OsVEqlTh37hz8/PzQuHFjhIaG4siRI5gxYwbKlSuHtLQ0ODo6Yt68ebCxscHevXsRHBwMADA0NNTWSyAZEB/Noti5cycWL16MK1euICUlRWqvWrUqateujRMnTiA9PR2A6pFQvGbZN0Db6YqyZ/v27cLY2FhMnTpVnD9/XgghRGBgoLSLSgghUlJSxM2bN0VoaKh48uSJNsslmZowYYKYOXOmKFq0qLhz544ICwsT+fPnF0uXLhVCCPH27Vsxbtw48ejRIyGEEHfv3hUtWrQQVatWFcHBwdosnb5yH468rF+/Xqxdu1YIIcSAAQOEubm5WLx4sYiNjZX6hISEiDJlykjbIn1bGG6+Anfu3BGOjo5iyZIlmZYtW7ZM2kVFpGkfzo/ZtGmTsLe3Fzdu3BBdu3YV7u7uwsTERKxcuVLq8+zZM1G7dm2xdu1aad1bt26Jn3/+WURERHzx+kkePtwOb9y4IVxdXUWlSpXErl27hBBCeHl5iVKlSonp06eL8PBwER4eLho2bCjq1q2rEoro28Gxua/A48ePoa+vr3KmzYwjVvr27Yt8+fKhW7duMDQ0xIgRI7RYKclNxlFRR48exZEjRzB8+HCUL19eOjlkw4YN0bNnTwDvL9jau3dv6OrqonPnztDR0YFSqUSZMmWwYcMGHp1COZaxHY4cORIPHz6EsbExbt++jaFDh+Ldu3cICgpCz549MX78eCxcuBA1a9ZE/vz5sXnzZigUik+eC4fki+HmK5CQkKByfhClUintPz5y5AiqVKmCzZs3q1ynh0hTIiMj0atXL0RHR2Ps2LEAgH79+uH+/fs4fPgwXF1dUapUKTx+/BjJyck4f/48dHV1VU7QxzkO9LmCgoKwYsUKHDp0CI6OjkhJSYGXlxd8fX2ho6ODVatWwcTEBFu2bEGTJk3QsWNHGBoaIjU1FQYGBtoun74wRtmvQKVKlfDy5UsEBgYCeP8tJiPc7Nq1Cxs2bEDbtm1RtmxZbZZJMmVra4vg4GDY2Njgzz//xMWLF6Grq4s5c+ZgypQpaNCgAWxtbdGhQ4dPXlWZ57ahzxUeHo4KFSrAxcUFZmZmsLW1xapVq6Crq4uhQ4dix44dWLRoERo1aoT58+dj9+7diI+PZ7D5RvHr1FfA0dERixYtQr9+/ZCWlgZPT0/o6uoiKCgIQUFBOH36NM/0SrnK2dkZ27dvh5eXFwICAjBw4EA4OzujVatWaNWqlUrf9PR0jtSQxoj/v5iloaEhkpOTkZqaCiMjI6SlpaFo0aLw9fVFixYt4O/vD2NjY2zYsAGdO3fGiBEjoKenh/bt22v7JZAW8AzFXwmlUont27fD29sb+fLlg5GREXR1dbFx40a4urpquzz6Rly+fBm9e/dGlSpVMHjwYJQvX17bJdE34vr163B1dcVvv/2GiRMnSu0hISFYvnw53rx5g/T0dBw5cgQA0KNHD/z2228oUaKEliombWK4+cr8888/iIiIgEKhgKOjI2xsbLRdEn1jLl++DG9vbxQvXhyzZ8+Go6Ojtkuib0RQUBD69u2LIUOGoEOHDrCwsMCgQYNQo0YNtGnTBuXLl8fevXvRtGlTbZdKWsZwQ0RqO3fuHAICArBixQoehUJf1Pbt2/HLL7/AwMAAQghYW1vj1KlTiIqKwo8//oht27bB2dlZ22WSljHcEFGOZMyF4GG29KU9e/YMT548QVpaGmrWrAkdHR2MGTMGO3fuRFhYGGxtbbVdImkZww0R5VhGwCHSlps3b2LWrFn466+/cPDgQbi4uGi7JMoDeEgDEeUYgw1p07t375Camgpra2scPXqUE9xJwpEbIiL6qqWlpfEM2KSC4YaIiIhkhbMAiYiISFYYboiIiEhWGG6IiIhIVhhuiIiISFYYbohI9o4cOQKFQoGYmJhsr+Pg4AB/f/9cq4mIcg/DDRFpXffu3aFQKNCvX79My3x8fKBQKNC9e/cvXxgRfZUYbogoT7C3t8emTZuQlJQktSUnJ2PDhg0oVqyYFisjoq8Nww0R5QmVK1eGvb09goODpbbg4GAUK1YMrq6uUltKSgoGDRoEa2trGBkZoVatWjh//rzKY/3111/47rvvYGxsjPr16+PRo0eZnu/EiROoXbs2jI2NYW9vj0GDBiExMTHXXh8RfTkMN0SUZ/Ts2ROrV6+W7q9atQo9evRQ6TNq1Chs374da9aswaVLl+Dk5AR3d3e8fv0aAPDkyRO0bdsWLVu2xJUrV9C7d2/8+uuvKo9x//59NGnSBD/99BOuXbuGzZs348SJExgwYEDuv0giynUMN0SUZ3Tt2hUnTpxAREQEIiIicPLkSXTt2lVanpiYiKVLl2LOnDlo2rQpypUrh+XLl8PY2BgrV64EACxduhQlS5bEvHnzULp0aXTp0iXTfB1fX1906dIFQ4YMQalSpVCjRg38/vvvWLt2LZKTk7/kSyaiXMALZxJRnmFlZYXmzZsjKCgIQgg0b94chQoVkpbfv38faWlpqFmzptSmr6+PH374Abdu3QIA3Lp1C9WqVVN53OrVq6vcv3r1Kq5du4b169dLbUIIKJVKPHz4EGXLls2Nl0dEXwjDDRHlKT179pR2Dy1evDhXniMhIQHe3t4YNGhQpmWcvEz09WO4IaI8pUmTJkhNTYVCoYC7u7vKspIlS8LAwAAnT55E8eLFAby/IvT58+cxZMgQAEDZsmWxe/dulfXOnDmjcr9y5cr4+++/4eTklHsvhIi0hnNuiChP0dXVxa1bt/D3339DV1dXZVm+fPnQv39/jBw5Evv378fff/+NPn364O3bt+jVqxcAoF+/frh37x5GjhyJO3fuYMOGDQgKClJ5nNGjR+PUqVMYMGAArly5gnv37mHXrl2cUEwkEww3RJTnmJqawtTUNMtlM2fOxE8//YRu3bqhcuXKCA8PR0hICCwsLAC83620fft27Ny5E5UqVUJAQABmzJih8hjOzs44evQo7t69i9q1a8PV1RUTJkxAkSJFcv21EVHuUwghhLaLICIiItIUjtwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGsMNwQERGRrDDcEBERkaww3BAREZGs/B+XLE52CERTBAAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "## calculate avg response time\n", - "unique_models = set(unique_result[\"response\"]['model'] for unique_result in result[0][\"results\"])\n", - "model_dict = {model: {\"response_time\": []} for model in unique_models}\n", - "for iteration in result:\n", - " for completion_result in iteration[\"results\"]:\n", - " model_dict[completion_result[\"response\"][\"model\"]][\"response_time\"].append(completion_result[\"response_time\"])\n", - "\n", - "avg_response_time = {}\n", - "for model, data in model_dict.items():\n", - " avg_response_time[model] = sum(data[\"response_time\"]) / len(data[\"response_time\"])\n", - "\n", - "models = list(avg_response_time.keys())\n", - "response_times = list(avg_response_time.values())\n", - "\n", - "plt.bar(models, response_times)\n", - "plt.xlabel('Model', fontsize=10)\n", - "plt.ylabel('Average Response Time')\n", - "plt.title('Average Response Times for each Model')\n", - "\n", - "plt.xticks(models, [model[:15]+'...' if len(model) > 15 else model for model in models], rotation=45)\n", - "plt.show()" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/cookbook/litellm_model_fallback.ipynb b/cookbook/litellm_model_fallback.ipynb deleted file mode 100644 index 2e7987b9693..00000000000 --- a/cookbook/litellm_model_fallback.ipynb +++ /dev/null @@ -1,51 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "j6yJsCGeaq8G" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "u129iWNPaf72" - }, - "outputs": [], - "source": [ - "from litellm import completion\n", - "\n", - "model_fallback_list = [\"claude-instant-1\", \"gpt-3.5-turbo\", \"chatgpt-test\"]\n", - "\n", - "user_message = \"Hello, how are you?\"\n", - "messages = [{ \"content\": user_message,\"role\": \"user\"}]\n", - "\n", - "for model in model_fallback_list:\n", - " try:\n", - " response = completion(model=model, messages=messages)\n", - " except Exception:\n", - " print(f\"error occurred: {traceback.format_exc()}\")" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/litellm_proxy_server/batch_api/bedrock/README.md b/cookbook/litellm_proxy_server/batch_api/bedrock/README.md deleted file mode 100644 index 1410d76658e..00000000000 --- a/cookbook/litellm_proxy_server/batch_api/bedrock/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Bedrock batch endpoints - -Set `aws_bedrock_control_plane_endpoint` on a deployment when batch create, retrieve, and cancel requests must use a custom Bedrock control-plane endpoint. The proxy signs these requests for the configured URL. Without this setting, batch requests use the standard regional Bedrock endpoint - -```yaml -model_list: - - model_name: bedrock-batch - litellm_params: - model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0 - aws_region_name: us-east-1 - aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID - aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY - aws_batch_role_arn: os.environ/AWS_BATCH_ROLE_ARN - s3_output_bucket_name: os.environ/AWS_S3_OUTPUT_BUCKET_NAME - aws_bedrock_control_plane_endpoint: os.environ/BEDROCK_CONTROL_PLANE_ENDPOINT -``` - -Set `BEDROCK_CONTROL_PLANE_ENDPOINT` to your Bedrock control-plane VPC endpoint or compatible gateway base URL, without `/model-invocation-job`. The proxy adds the operation path. The endpoint must be reachable from the proxy, and the configured AWS region must match the endpoint's signing region - -`api_base` and `aws_bedrock_runtime_endpoint` configure runtime inference separately. They do not override the batch control-plane endpoint. `s3_endpoint_url` configures S3 file access separately - -The control-plane setting belongs in deployment configuration. Sending `aws_bedrock_control_plane_endpoint` in an API request body is rejected. There is no automatic `AWS_BEDROCK_CONTROL_PLANE_ENDPOINT` environment fallback, so reference the desired environment variable explicitly as shown above - -Start the proxy with this configuration and set `LITELLM_API_BASE` to its URL. Set `BEDROCK_BATCH_INPUT_URI` to a valid input JSONL file in S3 that the batch service role can read - -```bash -curl -sS "$LITELLM_API_BASE/v1/batches" \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ - -H 'Content-Type: application/json' \ - -d "{\"model\":\"bedrock-batch\",\"input_file_id\":\"$BEDROCK_BATCH_INPUT_URI\",\"endpoint\":\"/v1/chat/completions\",\"completion_window\":\"24h\"}" -``` - -Use the returned `id` as `BATCH_ID` to retrieve or cancel the same job through the deployment's configured endpoint - -```bash -curl -sS "$LITELLM_API_BASE/v1/batches/$BATCH_ID?model=bedrock-batch" \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" - -curl -sS -X POST "$LITELLM_API_BASE/v1/batches/$BATCH_ID/cancel?model=bedrock-batch" \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" -``` diff --git a/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py b/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py deleted file mode 100644 index 5b5f0978eb3..00000000000 --- a/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -from openai import OpenAI - -client = OpenAI( - base_url="http://0.0.0.0:4000", - api_key=os.environ["LITELLM_MASTER_KEY"], -) - -BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0" - -# Upload file -batch_input_file = client.files.create( - file=open("./bedrock_batch_completions.jsonl", "rb"), - purpose="batch", - extra_body={"target_model_names": BEDROCK_BATCH_MODEL}, -) -print(batch_input_file) - -# Create batch -batch = client.batches.create( - input_file_id=batch_input_file.id, - endpoint="/v1/chat/completions", - completion_window="24h", - metadata={"description": "Test batch job"}, -) -print(batch) diff --git a/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl b/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl deleted file mode 100644 index adef9ac2dd5..00000000000 --- a/cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl +++ /dev/null @@ -1,128 +0,0 @@ -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} diff --git a/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md b/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md deleted file mode 100644 index 1bf52d922c6..00000000000 --- a/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md +++ /dev/null @@ -1,279 +0,0 @@ -# Braintrust Prompt Wrapper for LiteLLM - -This directory contains a wrapper server that enables LiteLLM to use prompts from [Braintrust](https://www.braintrust.dev/) through the generic prompt management API. - -## Architecture - -``` -┌─────────────┐ ┌──────────────────────┐ ┌─────────────┐ -│ LiteLLM │ ──────> │ Wrapper Server │ ──────> │ Braintrust │ -│ Client │ │ (This Server) │ │ API │ -└─────────────┘ └──────────────────────┘ └─────────────┘ - Uses generic Transforms Stores actual - prompt manager Braintrust format prompt templates - to LiteLLM format -``` - -## Components - -### 1. Generic Prompt Manager (`litellm/integrations/generic_prompt_management/`) - -A generic client that can work with any API implementing the `/beta/litellm_prompt_management` endpoint. - -**Expected API Response Format:** -```json -{ - "prompt_id": "string", - "prompt_template": [ - {"role": "system", "content": "You are a helpful assistant"}, - {"role": "user", "content": "Hello {name}"} - ], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": { - "temperature": 0.7, - "max_tokens": 100 - } -} -``` - -### 2. Braintrust Wrapper Server (`braintrust_prompt_wrapper_server.py`) - -A FastAPI server that: -- Implements the `/beta/litellm_prompt_management` endpoint -- Fetches prompts from Braintrust API -- Transforms Braintrust response format to LiteLLM format - -## Setup - -### Install Dependencies - -```bash -pip install fastapi uvicorn httpx litellm -``` - -### Set Environment Variables - -```bash -export BRAINTRUST_API_KEY="your-braintrust-api-key" -``` - -## Usage - -### Step 1: Start the Wrapper Server - -```bash -python braintrust_prompt_wrapper_server.py -``` - -The server will start on `http://localhost:8080` by default. - -You can customize the port and host: -```bash -export PORT=8000 -export HOST=0.0.0.0 -python braintrust_prompt_wrapper_server.py -``` - -### Step 2: Use with LiteLLM - -```python -import litellm -from litellm.integrations.generic_prompt_management import GenericPromptManager - -# Configure the generic prompt manager to use your wrapper server -generic_config = { - "api_base": "http://localhost:8080", - "api_key": "your-braintrust-api-key", # Will be passed to Braintrust - "timeout": 30, -} - -# Create the prompt manager -prompt_manager = GenericPromptManager(**generic_config) - -# Use with completion -response = litellm.completion( - model="generic_prompt/gpt-4", - prompt_id="your-braintrust-prompt-id", - prompt_variables={"name": "World"}, # Variables to substitute - messages=[{"role": "user", "content": "Additional message"}] -) - -print(response) -``` - -### Step 3: Direct API Testing - -You can also test the wrapper API directly: - -```bash -# Test with curl -curl -H "Authorization: Bearer YOUR_BRAINTRUST_TOKEN" \ - "http://localhost:8080/beta/litellm_prompt_management?prompt_id=YOUR_PROMPT_ID" - -# Health check -curl http://localhost:8080/health - -# Service info -curl http://localhost:8080/ -``` - -## API Documentation - -Once the server is running, visit: -- Swagger UI: `http://localhost:8080/docs` -- ReDoc: `http://localhost:8080/redoc` - -## Braintrust Format Transformation - -The wrapper automatically transforms Braintrust's response format: - -**Braintrust API Response:** -```json -{ - "id": "prompt-123", - "prompt_data": { - "prompt": { - "type": "chat", - "messages": [ - { - "role": "system", - "content": "You are a helpful assistant" - } - ] - }, - "options": { - "model": "gpt-4", - "params": { - "temperature": 0.7, - "max_tokens": 100 - } - } - } -} -``` - -**Transformed to LiteLLM Format:** -```json -{ - "prompt_id": "prompt-123", - "prompt_template": [ - { - "role": "system", - "content": "You are a helpful assistant" - } - ], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": { - "temperature": 0.7, - "max_tokens": 100 - } -} -``` - -## Supported Parameters - -The wrapper automatically maps these Braintrust parameters to LiteLLM: - -- `temperature` -- `max_tokens` / `max_completion_tokens` -- `top_p` -- `frequency_penalty` -- `presence_penalty` -- `n` -- `stop` -- `response_format` -- `tool_choice` -- `function_call` -- `tools` - -## Variable Substitution - -The generic prompt manager supports simple variable substitution: - -```python -# In your Braintrust prompt: -# "Hello {name}, welcome to {place}!" - -# In your code: -prompt_variables = { - "name": "Alice", - "place": "Wonderland" -} - -# Result: -# "Hello Alice, welcome to Wonderland!" -``` - -Supports both `{variable}` and `{{variable}}` syntax. - -## Error Handling - -The wrapper provides detailed error messages: - -- **401**: Missing or invalid Braintrust API token -- **404**: Prompt not found in Braintrust -- **502**: Failed to connect to Braintrust API -- **500**: Error transforming response - -## Production Deployment - -For production use: - -1. **Use HTTPS**: Deploy behind a reverse proxy with SSL -2. **Authentication**: Add authentication to the wrapper endpoint if needed -3. **Rate Limiting**: Implement rate limiting to prevent abuse -4. **Caching**: Consider caching prompt responses -5. **Monitoring**: Add logging and monitoring - -Example with Docker: - -```dockerfile -FROM python:3.11-slim - -WORKDIR /app - -RUN pip install fastapi uvicorn httpx - -COPY braintrust_prompt_wrapper_server.py . - -ENV PORT=8080 -ENV HOST=0.0.0.0 - -EXPOSE 8080 - -CMD ["python", "braintrust_prompt_wrapper_server.py"] -``` - -## Extending to Other Providers - -This pattern can be used with any prompt management provider: - -1. Create a wrapper server that implements `/beta/litellm_prompt_management` -2. Transform the provider's response to LiteLLM format -3. Use the generic prompt manager to connect - -Example providers: -- Langsmith -- PromptLayer -- Humanloop -- Custom internal systems - -## Troubleshooting - -### "No Braintrust API token provided" -- Set `BRAINTRUST_API_KEY` environment variable -- Or pass token in `Authorization: Bearer TOKEN` header - -### "Failed to connect to Braintrust API" -- Check your internet connection -- Verify Braintrust API is accessible -- Check firewall settings - -### "Prompt not found" -- Verify the prompt ID exists in Braintrust -- Check that your API token has access to the prompt - -## License - -This wrapper is part of the LiteLLM project and follows the same license. - diff --git a/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py b/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py deleted file mode 100644 index 6379314c5b6..00000000000 --- a/cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py +++ /dev/null @@ -1,274 +0,0 @@ -""" -Mock server that implements the /beta/litellm_prompt_management endpoint -and acts as a wrapper for calling the Braintrust API. - -This server transforms Braintrust's prompt API response into the format -expected by LiteLLM's generic prompt management client. - -Usage: - python braintrust_prompt_wrapper_server.py - - # Then test with: - curl -H "Authorization: Bearer YOUR_BRAINTRUST_TOKEN" \ - "http://localhost:8080/beta/litellm_prompt_management?prompt_id=YOUR_PROMPT_ID" -""" - -import json -import os -from typing import Any, Dict, List, Optional - -import httpx -from fastapi import FastAPI, HTTPException, Header, Query -from fastapi.responses import JSONResponse -import uvicorn - - -app = FastAPI( - title="Braintrust Prompt Wrapper", - description="Wrapper server for Braintrust prompts to work with LiteLLM", - version="1.0.0", -) - - -def transform_braintrust_message(message: Dict[str, Any]) -> Dict[str, str]: - """ - Transform a Braintrust message to LiteLLM format. - - Braintrust message format: - { - "role": "system", - "content": "...", - "name": "..." (optional) - } - - LiteLLM format: - { - "role": "system", - "content": "..." - } - """ - result = { - "role": message.get("role", "user"), - "content": message.get("content", ""), - } - - # Include name if present - if "name" in message: - result["name"] = message["name"] - - return result - - -def transform_braintrust_response( - braintrust_response: Dict[str, Any], -) -> Dict[str, Any]: - """ - Transform Braintrust API response to LiteLLM prompt management format. - - Braintrust response format: - { - "objects": [{ - "id": "prompt_id", - "prompt_data": { - "prompt": { - "type": "chat", - "messages": [...], - "tools": "..." - }, - "options": { - "model": "gpt-4", - "params": { - "temperature": 0.7, - "max_tokens": 100, - ... - } - } - } - }] - } - - LiteLLM format: - { - "prompt_id": "prompt_id", - "prompt_template": [...], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": {...} - } - """ - # Extract the first object from the objects array if it exists - if "objects" in braintrust_response and len(braintrust_response["objects"]) > 0: - prompt_object = braintrust_response["objects"][0] - else: - prompt_object = braintrust_response - - prompt_data = prompt_object.get("prompt_data", {}) - prompt_info = prompt_data.get("prompt", {}) - options = prompt_data.get("options", {}) - - # Extract messages - messages = prompt_info.get("messages", []) - transformed_messages = [transform_braintrust_message(msg) for msg in messages] - - # Extract model - model = options.get("model") - - # Extract optional parameters - params = options.get("params", {}) - optional_params: Dict[str, Any] = {} - - # Map common parameters - param_mapping = { - "temperature": "temperature", - "max_tokens": "max_tokens", - "max_completion_tokens": "max_tokens", # Alternative name - "top_p": "top_p", - "frequency_penalty": "frequency_penalty", - "presence_penalty": "presence_penalty", - "n": "n", - "stop": "stop", - } - - for braintrust_param, litellm_param in param_mapping.items(): - if braintrust_param in params: - value = params[braintrust_param] - if value is not None: - optional_params[litellm_param] = value - - # Handle response_format - if "response_format" in params: - optional_params["response_format"] = params["response_format"] - - # Handle tool_choice - if "tool_choice" in params: - optional_params["tool_choice"] = params["tool_choice"] - - # Handle function_call - if "function_call" in params: - optional_params["function_call"] = params["function_call"] - - # Add tools if present - if "tools" in prompt_info and prompt_info["tools"]: - optional_params["tools"] = prompt_info["tools"] - - # Handle tool_functions from prompt_data - if "tool_functions" in prompt_data and prompt_data["tool_functions"]: - optional_params["tool_functions"] = prompt_data["tool_functions"] - - return { - "prompt_id": prompt_object.get("id"), - "prompt_template": transformed_messages, - "prompt_template_model": model, - "prompt_template_optional_params": optional_params if optional_params else None, - } - - -@app.get("/beta/litellm_prompt_management") -async def get_prompt( - prompt_id: str = Query(..., description="The Braintrust prompt ID to fetch"), - authorization: Optional[str] = Header( - None, description="Bearer token for Braintrust API" - ), -) -> JSONResponse: - """ - Fetch a prompt from Braintrust and transform it to LiteLLM format. - - Args: - prompt_id: The Braintrust prompt ID - authorization: Bearer token for Braintrust API (from header) - - Returns: - JSONResponse with the transformed prompt data - """ - # Extract token from Authorization header or environment - braintrust_token = None - if authorization and authorization.startswith("Bearer "): - braintrust_token = authorization.replace("Bearer ", "") - else: - braintrust_token = os.getenv("BRAINTRUST_API_KEY") - - if not braintrust_token: - raise HTTPException( - status_code=401, - detail="No Braintrust API token provided. Pass via Authorization header or set BRAINTRUST_API_KEY environment variable.", - ) - - # Call Braintrust API - braintrust_url = f"https://api.braintrust.dev/v1/prompt/{prompt_id}" - headers = { - "Authorization": f"Bearer {braintrust_token}", - "Accept": "application/json", - } - print(f"headers: {headers}") - print(f"braintrust_url: {braintrust_url}") - print(f"braintrust_token: {braintrust_token}") - - try: - async with httpx.AsyncClient(timeout=30.0) as client: - response = await client.get(braintrust_url, headers=headers) - response.raise_for_status() - braintrust_data = response.json() - except httpx.HTTPStatusError as e: - raise HTTPException( - status_code=e.response.status_code, - detail=f"Braintrust API error: {e.response.text}", - ) - except httpx.RequestError as e: - raise HTTPException( - status_code=502, - detail=f"Failed to connect to Braintrust API: {str(e)}", - ) - except json.JSONDecodeError as e: - raise HTTPException( - status_code=502, - detail=f"Failed to parse Braintrust API response: {str(e)}", - ) - - print(f"braintrust_data: {braintrust_data}") - # Transform the response - try: - transformed_data = transform_braintrust_response(braintrust_data) - print(f"transformed_data: {transformed_data}") - return JSONResponse(content=transformed_data) - except Exception as e: - raise HTTPException( - status_code=500, - detail=f"Failed to transform Braintrust response: {str(e)}", - ) - - -@app.get("/health") -async def health_check(): - """Health check endpoint.""" - return {"status": "healthy", "service": "braintrust-prompt-wrapper"} - - -@app.get("/") -async def root(): - """Root endpoint with service information.""" - return { - "service": "Braintrust Prompt Wrapper for LiteLLM", - "version": "1.0.0", - "endpoints": { - "prompt_management": "/beta/litellm_prompt_management?prompt_id=", - "health": "/health", - }, - "documentation": "/docs", - } - - -def main(): - """Run the server.""" - port = int(os.getenv("PORT", "8080")) - host = os.getenv("HOST", "0.0.0.0") - - print(f"🚀 Starting Braintrust Prompt Wrapper Server on {host}:{port}") - print(f"📚 API Documentation available at http://{host}:{port}/docs") - print( - f"🔑 Make sure to set BRAINTRUST_API_KEY environment variable or pass token in Authorization header" - ) - - uvicorn.run(app, host=host, port=port) - - -if __name__ == "__main__": - main() diff --git a/cookbook/litellm_proxy_server/cli_token_usage.py b/cookbook/litellm_proxy_server/cli_token_usage.py deleted file mode 100644 index c9c91e3283b..00000000000 --- a/cookbook/litellm_proxy_server/cli_token_usage.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 -""" -Example: Using CLI token with LiteLLM SDK - -This example shows how to use the CLI authentication token -in your Python scripts after running `lite login`. -""" - -from textwrap import indent -import litellm - -LITELLM_BASE_URL = "http://localhost:4000/" - - -def main(): - """Using CLI token with LiteLLM SDK""" - print("🚀 Using CLI Token with LiteLLM SDK") - print("=" * 40) - # litellm._turn_on_debug() - - # Get the CLI token - api_key = litellm.get_litellm_gateway_api_key() - - if not api_key: - print("❌ No CLI token found. Please run 'lite login' first.") - return - - print("✅ Found CLI token.") - - available_models = litellm.get_valid_models( - check_provider_endpoint=True, - custom_llm_provider="litellm_proxy", - api_key=api_key, - api_base=LITELLM_BASE_URL, - ) - - print("✅ Available models:") - if available_models: - for i, model in enumerate(available_models, 1): - print(f" {i:2d}. {model}") - else: - print(" No models available") - - # Use with LiteLLM - try: - response = litellm.completion( - model="litellm_proxy/gemini/gemini-2.5-flash", - messages=[{"role": "user", "content": "Hello from CLI token!"}], - api_key=api_key, - base_url=LITELLM_BASE_URL, - ) - print(f"✅ LLM Response: {response.model_dump_json(indent=4)}") - except Exception as e: - print(f"❌ Error: {e}") - - -if __name__ == "__main__": - main() - - print("\n💡 Tips:") - print("1. Run 'lite login' to authenticate first") - print("2. Replace 'https://your-proxy.com' with your actual proxy URL") - print("3. The token is stored in your OS keychain, or in ~/.litellm/token.json when there is none") diff --git a/cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py b/cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py deleted file mode 100644 index cc8b44d42b0..00000000000 --- a/cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Use LiteLLM Proxy MCP Gateway to call MCP tools. - -When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers. -""" - -import os - -import openai - -client = openai.OpenAI( - api_key=os.environ["LITELLM_MASTER_KEY"], - base_url="http://localhost:4000", # paste your litellm proxy base url here -) -print("Making API request to Responses API with MCP tools") - -response = client.responses.create( - model="gpt-5", - input=[ - { - "role": "user", - "content": "give me TLDR of what BerriAI/litellm repo is about", - "type": "message", - } - ], - tools=[ - { - "type": "mcp", - "server_label": "litellm", - "server_url": "litellm_proxy", - "require_approval": "never", - } - ], - stream=True, - tool_choice="required", -) - -for chunk in response: - print("response chunk: ", chunk) diff --git a/cookbook/litellm_proxy_server/readme.md b/cookbook/litellm_proxy_server/readme.md deleted file mode 100644 index 2c1eab72c24..00000000000 --- a/cookbook/litellm_proxy_server/readme.md +++ /dev/null @@ -1,177 +0,0 @@ -# liteLLM Proxy Server: 50+ LLM Models, Error Handling, Caching - -### Azure, Llama2, OpenAI, Claude, Hugging Face, Replicate Models - -[![PyPI Version](https://img.shields.io/pypi/v/litellm.svg)](https://pypi.org/project/litellm/) -[![PyPI Version](https://img.shields.io/badge/stable%20version-v0.1.345-blue?color=green&link=https://pypi.org/project/litellm/0.1.1/)](https://pypi.org/project/litellm/0.1.1/) -![Downloads](https://img.shields.io/pypi/dm/litellm) -[![litellm](https://img.shields.io/badge/%20%F0%9F%9A%85%20liteLLM-OpenAI%7CAzure%7CAnthropic%7CPalm%7CCohere%7CReplicate%7CHugging%20Face-blue?color=green)](https://github.com/BerriAI/litellm) - -[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/DYqQAW?referralCode=t3ukrU) - -![4BC6491E-86D0-4833-B061-9F54524B2579](https://github.com/BerriAI/litellm/assets/17561003/f5dd237b-db5e-42e1-b1ac-f05683b1d724) - -## What does liteLLM proxy do - -- Make `/chat/completions` requests for 50+ LLM models **Azure, OpenAI, Replicate, Anthropic, Hugging Face** - - Example: for `model` use `claude-2`, `gpt-3.5`, `gpt-4`, `command-nightly`, `stabilityai/stablecode-completion-alpha-3b-4k` - - ```json - { - "model": "replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1", - "messages": [ - { - "content": "Hello, whats the weather in San Francisco??", - "role": "user" - } - ] - } - ``` - -- **Consistent Input/Output** Format - - Call all models using the OpenAI format - `completion(model, messages)` - - Text responses will always be available at `['choices'][0]['message']['content']` -- **Error Handling** Using Model Fallbacks (if `GPT-4` fails, try `llama2`) -- **Logging** - Log Requests, Responses and Errors to `Supabase`, `Posthog`, `Mixpanel`, `Sentry`, `Lunary`,`Athina`, `Helicone` (Any of the supported providers here: https://litellm.readthedocs.io/en/latest/advanced/ - - **Example: Logs sent to Supabase** - Screenshot 2023-08-11 at 4 02 46 PM - -- **Token Usage & Spend** - Track Input + Completion tokens used + Spend/model -- **Caching** - Implementation of Semantic Caching -- **Streaming & Async Support** - Return generators to stream text responses - -## API Endpoints - -### `/chat/completions` (POST) - -This endpoint is used to generate chat completions for 50+ support LLM API Models. Use llama2, GPT-4, Claude2 etc - -#### Input - -This API endpoint accepts all inputs in raw JSON and expects the following inputs - -- `model` (string, required): ID of the model to use for chat completions. See all supported models [here]: (https://litellm.readthedocs.io/en/latest/supported/): - eg `gpt-3.5-turbo`, `gpt-4`, `claude-2`, `command-nightly`, `stabilityai/stablecode-completion-alpha-3b-4k` -- `messages` (array, required): A list of messages representing the conversation context. Each message should have a `role` (system, user, assistant, or function), `content` (message text), and `name` (for function role). -- Additional Optional parameters: `temperature`, `functions`, `function_call`, `top_p`, `n`, `stream`. See the full list of supported inputs here: https://litellm.readthedocs.io/en/latest/input/ - -#### Example JSON body - -For claude-2 - -```json -{ - "model": "claude-2", - "messages": [ - { - "content": "Hello, whats the weather in San Francisco??", - "role": "user" - } - ] -} -``` - -### Making an API request to the Proxy Server - -```python -import requests -import json - -# TODO: use your URL -url = "http://localhost:5000/chat/completions" - -payload = json.dumps({ - "model": "gpt-3.5-turbo", - "messages": [ - { - "content": "Hello, whats the weather in San Francisco??", - "role": "user" - } - ] -}) -headers = { - 'Content-Type': 'application/json' -} -response = requests.request("POST", url, headers=headers, data=payload) -print(response.text) - -``` - -### Output [Response Format] - -Responses from the server are given in the following format. -All responses from the server are returned in the following format (for all LLM models). More info on output here: https://litellm.readthedocs.io/en/latest/output/ - -```json -{ - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "I'm sorry, but I don't have the capability to provide real-time weather information. However, you can easily check the weather in San Francisco by searching online or using a weather app on your phone.", - "role": "assistant" - } - } - ], - "created": 1691790381, - "id": "chatcmpl-7mUFZlOEgdohHRDx2UpYPRTejirzb", - "model": "gpt-3.5-turbo-0613", - "object": "chat.completion", - "usage": { - "completion_tokens": 41, - "prompt_tokens": 16, - "total_tokens": 57 - } -} -``` - -## Installation & Usage - -### Running Locally - -1. Clone liteLLM repository to your local machine: - ``` - git clone https://github.com/BerriAI/liteLLM-proxy - ``` -2. Install the required dependencies using pip - ``` - pip install -r requirements.txt - ``` -3. Set your LLM API keys - ``` - os.environ['OPENAI_API_KEY]` = "YOUR_API_KEY" - or - set OPENAI_API_KEY in your .env file - ``` -4. Run the server: - ``` - python main.py - ``` - -## Deploying - -1. Quick Start: Deploy on Railway - - [![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/DYqQAW?referralCode=t3ukrU) - -2. `GCP`, `AWS`, `Azure` - This project includes a `Dockerfile` allowing you to build and deploy a Docker Project on your providers - -# Support / Talk with founders - -- [Our calendar 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) -- [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai - -## Roadmap - -- [ ] Support hosted db (e.g. Supabase) -- [ ] Easily send data to places like posthog and sentry. -- [ ] Add a hot-cache for project spend logs - enables fast checks for user + project limitings -- [ ] Implement user-based rate-limiting -- [ ] Spending controls per project - expose key creation endpoint -- [ ] Need to store a keys db -> mapping created keys to their alias (i.e. project name) -- [ ] Easily add new models as backups / as the entry-point (add this to the available model list) diff --git a/cookbook/litellm_proxy_server/secret_manager/custom_secret_manager_config.yaml b/cookbook/litellm_proxy_server/secret_manager/custom_secret_manager_config.yaml deleted file mode 100644 index 3598a9b1b65..00000000000 --- a/cookbook/litellm_proxy_server/secret_manager/custom_secret_manager_config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY - key_management_system: "custom" - key_management_settings: - custom_secret_manager: my_secret_manager.InMemorySecretManager - store_virtual_keys: true - prefix_for_stored_virtual_keys: "litellm/" - access_mode: "read_and_write" - -model_list: - - model_name: gpt-4 - litellm_params: - model: openai/gpt-4 - api_key: os.environ/OPENAI_API_KEY # Read from custom secret manager - - - model_name: claude-3-5-sonnet - litellm_params: - model: anthropic/claude-3-5-sonnet-20241022 - api_key: os.environ/ANTHROPIC_API_KEY # Read from custom secret manager - diff --git a/cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py b/cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py deleted file mode 100644 index 65c7f754b41..00000000000 --- a/cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -Example custom secret manager for LiteLLM Proxy. - -This is a simple in-memory secret manager for testing purposes. -In production, replace this with your actual secret management system. -""" - -from typing import Optional, Union - -import httpx - -from litellm.integrations.custom_secret_manager import CustomSecretManager - - -class InMemorySecretManager(CustomSecretManager): - def __init__(self): - super().__init__(secret_manager_name="in_memory_secrets") - # Store your secrets in memory - print("INITIALIZING CUSTOM SECRET MANAGER IN MEMORY") - self.secrets = {} - print("CUSTOM SECRET MANAGER IN MEMORY INITIALIZED") - - async def async_read_secret( - self, - secret_name: str, - optional_params: Optional[dict] = None, - timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Optional[str]: - """Read secret asynchronously""" - print("READING SECRET ASYNCHRONOUSLY") - print("SECRET NAME: %s", secret_name) - print("SECRET: %s", self.secrets.get(secret_name)) - return self.secrets.get(secret_name) - - def sync_read_secret( - self, - secret_name: str, - optional_params: Optional[dict] = None, - timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> Optional[str]: - """Read secret synchronously""" - from litellm._logging import verbose_proxy_logger - - verbose_proxy_logger.info( - f"CUSTOM SECRET MANAGER: LOOKING FOR SECRET: {secret_name}" - ) - value = self.secrets.get(secret_name) - verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: READ SECRET: {value}") - return value - - async def async_write_secret( - self, - secret_name: str, - secret_value: str, - description: Optional[str] = None, - optional_params: Optional[dict] = None, - timeout: Optional[Union[float, httpx.Timeout]] = None, - tags: Optional[Union[dict, list]] = None, - ) -> dict: - """Write a secret to the in-memory store""" - self.secrets[secret_name] = secret_value - print("ALL SECRETS=%s", self.secrets) - return { - "status": "success", - "secret_name": secret_name, - "description": description, - } - - async def async_delete_secret( - self, - secret_name: str, - recovery_window_in_days: Optional[int] = 7, - optional_params: Optional[dict] = None, - timeout: Optional[Union[float, httpx.Timeout]] = None, - ) -> dict: - """Delete a secret from the in-memory store""" - if secret_name in self.secrets: - del self.secrets[secret_name] - return {"status": "deleted", "secret_name": secret_name} - return {"status": "not_found", "secret_name": secret_name} diff --git a/cookbook/litellm_router/error_log.txt b/cookbook/litellm_router/error_log.txt deleted file mode 100644 index 983b47cbbba..00000000000 --- a/cookbook/litellm_router/error_log.txt +++ /dev/null @@ -1,1004 +0,0 @@ -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: Expecting value: line 1 column 1 (char 0) - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: Expecting value: line 1 column 1 (char 0) - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Exception: 'Response' object has no attribute 'get' - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Exception: 'Response' object has no attribute 'get' - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Exception: 'Response' object has no attribute 'get' - diff --git a/cookbook/litellm_router/load_test_proxy.py b/cookbook/litellm_router/load_test_proxy.py deleted file mode 100644 index 9ae6e764d91..00000000000 --- a/cookbook/litellm_router/load_test_proxy.py +++ /dev/null @@ -1,148 +0,0 @@ -import sys -import os -from dotenv import load_dotenv - -load_dotenv() - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path - -from litellm import Router -import litellm - -litellm.set_verbose = False -os.environ.pop("AZURE_AD_TOKEN") - -model_list = [ - { # list of model deployments - "model_name": "gpt-3.5-turbo", # model alias - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-v-2", # actual model name - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-functioncalling", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { # params for litellm completion/embedding call - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, -] -router = Router(model_list=model_list) - - -file_paths = [ - "test_questions/question1.txt", - "test_questions/question2.txt", - "test_questions/question3.txt", -] -questions = [] - -for file_path in file_paths: - try: - print(file_path) - with open(file_path, "r") as file: - content = file.read() - questions.append(content) - except FileNotFoundError as e: - print(f"File not found: {e}") - except Exception as e: - print(f"An error occurred: {e}") - -# for q in questions: -# print(q) - - -# make X concurrent calls to litellm.completion(model=gpt-35-turbo, messages=[]), pick a random question in questions array. -# Allow me to tune X concurrent calls.. Log question, output/exception, response time somewhere -# show me a summary of requests made, success full calls, failed calls. For failed calls show me the exceptions - -import concurrent.futures -import random -import time - - -# Function to make concurrent calls to OpenAI API -def make_openai_completion(question): - try: - start_time = time.time() - import openai - - client = openai.OpenAI( - api_key=os.environ["OPENAI_API_KEY"], base_url="http://0.0.0.0:8000" - ) # base_url="http://0.0.0.0:8000", - response = client.chat.completions.create( - model="gpt-3.5-turbo", - messages=[ - { - "role": "system", - "content": f"You are a helpful assistant. Answer this question{question}", - } - ], - ) - print(response) - end_time = time.time() - - # Log the request details - with open("request_log.txt", "a") as log_file: - log_file.write( - f"Question: {question[:100]}\nResponse ID:{response.id} Content:{response.choices[0].message.content[:10]}\nTime: {end_time - start_time:.2f} seconds\n\n" - ) - - return response - except Exception as e: - # Log exceptions for failed calls - with open("error_log.txt", "a") as error_log_file: - error_log_file.write(f"Question: {question[:100]}\nException: {str(e)}\n\n") - return None - - -# Number of concurrent calls (you can adjust this) -concurrent_calls = 100 - -# List to store the futures of concurrent calls -futures = [] - -# Make concurrent calls -with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_calls) as executor: - for _ in range(concurrent_calls): - random_question = random.choice(questions) - futures.append(executor.submit(make_openai_completion, random_question)) - -# Wait for all futures to complete -concurrent.futures.wait(futures) - -# Summarize the results -successful_calls = 0 -failed_calls = 0 - -for future in futures: - if future.result() is not None: - successful_calls += 1 - else: - failed_calls += 1 - -print("Load test Summary:") -print(f"Total Requests: {concurrent_calls}") -print(f"Successful Calls: {successful_calls}") -print(f"Failed Calls: {failed_calls}") - -# Display content of the logs -with open("request_log.txt", "r") as log_file: - print("\nRequest Log:\n", log_file.read()) - -with open("error_log.txt", "r") as error_log_file: - print("\nError Log:\n", error_log_file.read()) diff --git a/cookbook/litellm_router/load_test_queuing.py b/cookbook/litellm_router/load_test_queuing.py deleted file mode 100644 index 7d4d44b2528..00000000000 --- a/cookbook/litellm_router/load_test_queuing.py +++ /dev/null @@ -1,164 +0,0 @@ -import sys -import os -from dotenv import load_dotenv - -load_dotenv() - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path - -from litellm import Router -import litellm - -litellm.set_verbose = False -# os.environ.pop("AZURE_AD_TOKEN") - -model_list = [ - { # list of model deployments - "model_name": "gpt-3.5-turbo", # model alias - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-v-2", # actual model name - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-functioncalling", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { # params for litellm completion/embedding call - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, -] -router = Router(model_list=model_list) - - -file_paths = [ - "test_questions/question1.txt", - "test_questions/question2.txt", - "test_questions/question3.txt", -] -questions = [] - -for file_path in file_paths: - try: - print(file_path) - with open(file_path, "r") as file: - content = file.read() - questions.append(content) - except FileNotFoundError as e: - print(f"File not found: {e}") - except Exception as e: - print(f"An error occurred: {e}") - -# for q in questions: -# print(q) - - -# make X concurrent calls to litellm.completion(model=gpt-35-turbo, messages=[]), pick a random question in questions array. -# Allow me to tune X concurrent calls.. Log question, output/exception, response time somewhere -# show me a summary of requests made, success full calls, failed calls. For failed calls show me the exceptions - -import concurrent.futures -import random -import time - - -# Function to make concurrent calls to OpenAI API -def make_openai_completion(question): - try: - start_time = time.time() - import requests - - data = { - "model": "gpt-3.5-turbo", - "messages": [ - { - "role": "system", - "content": f"You are a helpful assistant. Answer this question{question}", - }, - ], - } - response = requests.post("http://0.0.0.0:8000/queue/request", json=data) - response = response.json() - end_time = time.time() - # Log the request details - with open("request_log.txt", "a") as log_file: - log_file.write( - f"Question: {question[:100]}\nResponse ID: {response.get('id', 'N/A')} Url: {response.get('url', 'N/A')}\nTime: {end_time - start_time:.2f} seconds\n\n" - ) - - # polling the url - while True: - try: - url = response["url"] - polling_url = f"http://0.0.0.0:8000{url}" - polling_response = requests.get(polling_url) - polling_response = polling_response.json() - print("\n RESPONSE FROM POLLING JoB", polling_response) - status = polling_response["status"] - if status == "finished": - llm_response = polling_response["result"] - with open("response_log.txt", "a") as log_file: - log_file.write( - f"Response ID: {llm_response.get('id', 'NA')}\nLLM Response: {llm_response}\nTime: {end_time - start_time:.2f} seconds\n\n" - ) - - break - print( - f"POLLING JOB{polling_url}\nSTATUS: {status}, \n Response {polling_response}" - ) - time.sleep(0.5) - except Exception as e: - print("got exception in polling", e) - break - - return response - except Exception as e: - # Log exceptions for failed calls - with open("error_log.txt", "a") as error_log_file: - error_log_file.write(f"Question: {question[:100]}\nException: {str(e)}\n\n") - return None - - -# Number of concurrent calls (you can adjust this) -concurrent_calls = 10 - -# List to store the futures of concurrent calls -futures = [] - -# Make concurrent calls -with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_calls) as executor: - for _ in range(concurrent_calls): - random_question = random.choice(questions) - futures.append(executor.submit(make_openai_completion, random_question)) - -# Wait for all futures to complete -concurrent.futures.wait(futures) - -# Summarize the results -successful_calls = 0 -failed_calls = 0 - -for future in futures: - if future.done(): - if future.result() is not None: - successful_calls += 1 - else: - failed_calls += 1 - -print("Load test Summary:") -print(f"Total Requests: {concurrent_calls}") -print(f"Successful Calls: {successful_calls}") -print(f"Failed Calls: {failed_calls}") diff --git a/cookbook/litellm_router/load_test_router.py b/cookbook/litellm_router/load_test_router.py deleted file mode 100644 index 92533b6c929..00000000000 --- a/cookbook/litellm_router/load_test_router.py +++ /dev/null @@ -1,143 +0,0 @@ -import sys -import os -from dotenv import load_dotenv - -load_dotenv() - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path - -from litellm import Router -import litellm - -litellm.set_verbose = False -os.environ.pop("AZURE_AD_TOKEN") - -model_list = [ - { # list of model deployments - "model_name": "gpt-3.5-turbo", # model alias - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-v-2", # actual model name - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-functioncalling", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { # params for litellm completion/embedding call - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, -] -router = Router(model_list=model_list) - - -file_paths = [ - "test_questions/question1.txt", - "test_questions/question2.txt", - "test_questions/question3.txt", -] -questions = [] - -for file_path in file_paths: - try: - print(file_path) - with open(file_path, "r") as file: - content = file.read() - questions.append(content) - except FileNotFoundError as e: - print(f"File not found: {e}") - except Exception as e: - print(f"An error occurred: {e}") - -# for q in questions: -# print(q) - - -# make X concurrent calls to litellm.completion(model=gpt-35-turbo, messages=[]), pick a random question in questions array. -# Allow me to tune X concurrent calls.. Log question, output/exception, response time somewhere -# show me a summary of requests made, success full calls, failed calls. For failed calls show me the exceptions - -import concurrent.futures -import random -import time - - -# Function to make concurrent calls to OpenAI API -def make_openai_completion(question): - try: - start_time = time.time() - response = router.completion( - model="gpt-3.5-turbo", - messages=[ - { - "role": "system", - "content": f"You are a helpful assistant. Answer this question{question}", - } - ], - ) - print(response) - end_time = time.time() - - # Log the request details - with open("request_log.txt", "a") as log_file: - log_file.write( - f"Question: {question[:100]}\nResponse: {response.choices[0].message.content}\nTime: {end_time - start_time:.2f} seconds\n\n" - ) - - return response - except Exception as e: - # Log exceptions for failed calls - with open("error_log.txt", "a") as error_log_file: - error_log_file.write(f"Question: {question[:100]}\nException: {str(e)}\n\n") - return None - - -# Number of concurrent calls (you can adjust this) -concurrent_calls = 150 - -# List to store the futures of concurrent calls -futures = [] - -# Make concurrent calls -with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_calls) as executor: - for _ in range(concurrent_calls): - random_question = random.choice(questions) - futures.append(executor.submit(make_openai_completion, random_question)) - -# Wait for all futures to complete -concurrent.futures.wait(futures) - -# Summarize the results -successful_calls = 0 -failed_calls = 0 - -for future in futures: - if future.result() is not None: - successful_calls += 1 - else: - failed_calls += 1 - -print("Load test Summary:") -print(f"Total Requests: {concurrent_calls}") -print(f"Successful Calls: {successful_calls}") -print(f"Failed Calls: {failed_calls}") - -# Display content of the logs -with open("request_log.txt", "r") as log_file: - print("\nRequest Log:\n", log_file.read()) - -with open("error_log.txt", "r") as error_log_file: - print("\nError Log:\n", error_log_file.read()) diff --git a/cookbook/litellm_router/request_log.txt b/cookbook/litellm_router/request_log.txt deleted file mode 100644 index 821d87ab56a..00000000000 --- a/cookbook/litellm_router/request_log.txt +++ /dev/null @@ -1,48 +0,0 @@ -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Response ID: 71a47cd4-92d9-4091-9429-8d22af6b56bf Url: /queue/response/71a47cd4-92d9-4091-9429-8d22af6b56bf -Time: 0.77 seconds - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Response ID: a0855c20-59ba-4eed-85c1-e0719eebdeab Url: /queue/response/a0855c20-59ba-4eed-85c1-e0719eebdeab -Time: 1.46 seconds - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Response ID: b131cdcd-0693-495b-ad41-b0cf2afc4833 Url: /queue/response/b131cdcd-0693-495b-ad41-b0cf2afc4833 -Time: 2.13 seconds - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Response ID: a58e5185-90e7-4832-9f28-e5a5ac167a40 Url: /queue/response/a58e5185-90e7-4832-9f28-e5a5ac167a40 -Time: 2.83 seconds - -Question: Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. -Response ID: 52dbbd49-eedb-4c11-8382-3ca7deb1af35 Url: /queue/response/52dbbd49-eedb-4c11-8382-3ca7deb1af35 -Time: 3.50 seconds - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Response ID: eedda05f-61e1-4081-b49d-27f9449bcf69 Url: /queue/response/eedda05f-61e1-4081-b49d-27f9449bcf69 -Time: 4.20 seconds - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Response ID: 8a484722-66ec-4193-b19b-2dfc4265cfd2 Url: /queue/response/8a484722-66ec-4193-b19b-2dfc4265cfd2 -Time: 4.89 seconds - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Response ID: ae1e2b71-d711-456d-8df0-13ce0709eb04 Url: /queue/response/ae1e2b71-d711-456d-8df0-13ce0709eb04 -Time: 5.60 seconds - -Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 10 -Response ID: cfabd174-838e-4252-b82b-648923573db8 Url: /queue/response/cfabd174-838e-4252-b82b-648923573db8 -Time: 6.29 seconds - -Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope -Response ID: 02d5b7d6-5443-41e9-94e4-90d8b00d49fb Url: /queue/response/02d5b7d6-5443-41e9-94e4-90d8b00d49fb -Time: 7.01 seconds - diff --git a/cookbook/litellm_router/test_questions/question1.txt b/cookbook/litellm_router/test_questions/question1.txt deleted file mode 100644 index d633a8ea22e..00000000000 --- a/cookbook/litellm_router/test_questions/question1.txt +++ /dev/null @@ -1,43 +0,0 @@ -Given this context, what is litellm? LiteLLM about: About -Call all LLM APIs using the OpenAI format. Use Bedrock, Azure, OpenAI, Cohere, Anthropic, Ollama, Sagemaker, HuggingFace, Replicate (100+ LLMs). LiteLLM manages - -Translating inputs to the provider's completion and embedding endpoints -Guarantees consistent output, text responses will always be available at ['choices'][0]['message']['content'] -Exception mapping - common exceptions across providers are mapped to the OpenAI exception types. -10/05/2023: LiteLLM is adopting Semantic Versioning for all commits. Learn more -10/16/2023: Self-hosted OpenAI-proxy server Learn more - -Usage (Docs) -Important -LiteLLM v1.0.0 is being launched to require openai>=1.0.0. Track this here - -Open In Colab -pip install litellm -from litellm import completion -import os - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "your-openai-key" -os.environ["COHERE_API_KEY"] = "your-cohere-key" - -messages = [{ "content": "Hello, how are you?","role": "user"}] - -# openai call -response = completion(model="gpt-3.5-turbo", messages=messages) - -# cohere call -response = completion(model="command-nightly", messages=messages) -print(response) -Streaming (Docs) -liteLLM supports streaming the model response back, pass stream=True to get a streaming iterator in response. -Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.) - -from litellm import completion -response = completion(model="gpt-3.5-turbo", messages=messages, stream=True) -for chunk in response: - print(chunk['choices'][0]['delta']) - -# claude 2 -result = completion('claude-2', messages, stream=True) -for chunk in result: - print(chunk['choices'][0]['delta']) \ No newline at end of file diff --git a/cookbook/litellm_router/test_questions/question2.txt b/cookbook/litellm_router/test_questions/question2.txt deleted file mode 100644 index 78188d06668..00000000000 --- a/cookbook/litellm_router/test_questions/question2.txt +++ /dev/null @@ -1,65 +0,0 @@ -Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the OpenAI format. Use Bedrock, Azure, OpenAI, Cohere, Anthropic, Ollama, Sagemaker, HuggingFace, Replicate (100+ LLMs). LiteLLM manages - -Translating inputs to the provider's completion and embedding endpoints -Guarantees consistent output, text responses will always be available at ['choices'][0]['message']['content'] -Exception mapping - common exceptions across providers are mapped to the OpenAI exception types. -10/05/2023: LiteLLM is adopting Semantic Versioning for all commits. Learn more -10/16/2023: Self-hosted OpenAI-proxy server Learn more - -Usage (Docs) -Important -LiteLLM v1.0.0 is being launched to require openai>=1.0.0. Track this here - -Open In Colab -pip install litellm -from litellm import completion -import os - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "your-openai-key" -os.environ["COHERE_API_KEY"] = "your-cohere-key" - -messages = [{ "content": "Hello, how are you?","role": "user"}] - -# openai call -response = completion(model="gpt-3.5-turbo", messages=messages) - -# cohere call -response = completion(model="command-nightly", messages=messages) -print(response) -Streaming (Docs) -liteLLM supports streaming the model response back, pass stream=True to get a streaming iterator in response. -Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.) - -from litellm import completion -response = completion(model="gpt-3.5-turbo", messages=messages, stream=True) -for chunk in response: - print(chunk['choices'][0]['delta']) - -# claude 2 -result = completion('claude-2', messages, stream=True) -for chunk in result: - print(chunk['choices'][0]['delta']) Supported LiteLLM providers Supported Provider (Docs) -Provider Completion Streaming Async Completion Async Streaming -openai ✅ ✅ ✅ ✅ -azure ✅ ✅ ✅ ✅ -aws - sagemaker ✅ ✅ ✅ ✅ -aws - bedrock ✅ ✅ ✅ ✅ -cohere ✅ ✅ ✅ ✅ -anthropic ✅ ✅ ✅ ✅ -huggingface ✅ ✅ ✅ ✅ -replicate ✅ ✅ ✅ ✅ -together_ai ✅ ✅ ✅ ✅ -openrouter ✅ ✅ ✅ ✅ -google - vertex_ai ✅ ✅ ✅ ✅ -google - palm ✅ ✅ ✅ ✅ -ai21 ✅ ✅ ✅ ✅ -baseten ✅ ✅ ✅ ✅ -vllm ✅ ✅ ✅ ✅ -nlp_cloud ✅ ✅ ✅ ✅ -aleph alpha ✅ ✅ ✅ ✅ -petals ✅ ✅ ✅ ✅ -ollama ✅ ✅ ✅ ✅ -deepinfra ✅ ✅ ✅ ✅ -perplexity-ai ✅ ✅ ✅ ✅ -anyscale ✅ ✅ ✅ ✅ \ No newline at end of file diff --git a/cookbook/litellm_router/test_questions/question3.txt b/cookbook/litellm_router/test_questions/question3.txt deleted file mode 100644 index d6006f9c73c..00000000000 --- a/cookbook/litellm_router/test_questions/question3.txt +++ /dev/null @@ -1,50 +0,0 @@ -What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server -LiteLLM Server manages: - -Calling 100+ LLMs Huggingface/Bedrock/TogetherAI/etc. in the OpenAI ChatCompletions & Completions format -Set custom prompt templates + model-specific configs (temperature, max_tokens, etc.) -Quick Start -View all the supported args for the Proxy CLI here - -$ litellm --model huggingface/bigcode/starcoder - -#INFO: Proxy running on http://0.0.0.0:8000 - -Test -In a new shell, run, this will make an openai.ChatCompletion request - -litellm --test - -This will now automatically route any requests for gpt-3.5-turbo to bigcode starcoder, hosted on huggingface inference endpoints. - -Replace openai base -import openai - -openai.api_base = "http://0.0.0.0:8000" - -print(openai.chat.completions.create(model="test", messages=[{"role":"user", "content":"Hey!"}])) - -Supported LLMs -Bedrock -Huggingface (TGI) -Anthropic -VLLM -OpenAI Compatible Server -TogetherAI -Replicate -Petals -Palm -Azure OpenAI -AI21 -Cohere -$ export AWS_ACCESS_KEY_ID="" -$ export AWS_REGION_NAME="" # e.g. us-west-2 -$ export AWS_SECRET_ACCESS_KEY="" - -$ litellm --model bedrock/anthropic.claude-v2 - -Server Endpoints -POST /chat/completions - chat completions endpoint to call 100+ LLMs -POST /completions - completions endpoint -POST /embeddings - embedding endpoint for Azure, OpenAI, Huggingface endpoints -GET /models - available models on server \ No newline at end of file diff --git a/cookbook/litellm_router_load_test/memory_usage/router_endpoint.py b/cookbook/litellm_router_load_test/memory_usage/router_endpoint.py deleted file mode 100644 index 1dc2d914857..00000000000 --- a/cookbook/litellm_router_load_test/memory_usage/router_endpoint.py +++ /dev/null @@ -1,65 +0,0 @@ -from fastapi import FastAPI -import uvicorn -from memory_profiler import profile -import os -import litellm -from litellm import Router -from dotenv import load_dotenv -from litellm._uuid import uuid - -load_dotenv() - -model_list = [ - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "azure/chatgpt-v-2", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - "tpm": 240000, - "rpm": 1800, - }, - { - "model_name": "text-embedding-ada-002", - "litellm_params": { - "model": "azure/azure-embedding-model", - "api_key": os.getenv("AZURE_API_KEY"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - "tpm": 100000, - "rpm": 10000, - }, -] - -litellm.set_verbose = True -litellm.cache = litellm.Cache( - type="s3", s3_bucket_name="litellm-my-test-bucket-2", s3_region_name="us-east-1" -) -router = Router(model_list=model_list, set_verbose=True) - -app = FastAPI() - - -@app.get("/") -async def read_root(): - return {"message": "Welcome to the FastAPI endpoint!"} - - -@profile -@app.post("/router_acompletion") -async def router_acompletion(): - question = f"This is a test: {uuid.uuid4()}" * 100 - resp = await router.aembedding(model="text-embedding-ada-002", input=question) - print("embedding-resp", resp) - - response = await router.acompletion( - model="gpt-3.5-turbo", messages=[{"role": "user", "content": question}] - ) - print("completion-resp", response) - return response - - -if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py b/cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py deleted file mode 100644 index 76d5d3913f5..00000000000 --- a/cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py +++ /dev/null @@ -1,91 +0,0 @@ -#### What this tests #### - -from memory_profiler import profile -import sys -import os -import time -import asyncio - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import litellm -from litellm import Router -from dotenv import load_dotenv -from litellm._uuid import uuid - -load_dotenv() - - -model_list = [ - { - "model_name": "gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-v-2", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - "tpm": 240000, - "rpm": 1800, - }, - { - "model_name": "text-embedding-ada-002", - "litellm_params": { - "model": "azure/azure-embedding-model", - "api_key": os.environ["AZURE_API_KEY"], - "api_base": os.environ["AZURE_API_BASE"], - }, - "tpm": 100000, - "rpm": 10000, - }, -] -litellm.set_verbose = True -litellm.cache = litellm.Cache( - type="s3", s3_bucket_name="litellm-my-test-bucket-2", s3_region_name="us-east-1" -) -router = Router( - model_list=model_list, - set_verbose=True, -) # type: ignore - - -@profile -async def router_acompletion(): - # embedding call - question = f"This is a test: {uuid.uuid4()}" * 100 - resp = await router.aembedding(model="text-embedding-ada-002", input=question) - print("embedding-resp", resp) - - response = await router.acompletion( - model="gpt-3.5-turbo", messages=[{"role": "user", "content": question}] - ) - print("completion-resp", response) - return response - - -async def main(): - for i in range(1): - start = time.time() - n = 50 # Number of concurrent tasks - tasks = [router_acompletion() for _ in range(n)] - - chat_completions = await asyncio.gather(*tasks) - - successful_completions = [c for c in chat_completions if c is not None] - - # Write errors to error_log.txt - with open("error_log.txt", "a") as error_log: - for completion in chat_completions: - if isinstance(completion, str): - error_log.write(completion + "\n") - - print(n, time.time() - start, len(successful_completions)) - time.sleep(10) - - -if __name__ == "__main__": - # Blank out contents of error_log.txt - open("error_log.txt", "w").close() - - asyncio.run(main()) diff --git a/cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py b/cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py deleted file mode 100644 index 76d5d3913f5..00000000000 --- a/cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py +++ /dev/null @@ -1,91 +0,0 @@ -#### What this tests #### - -from memory_profiler import profile -import sys -import os -import time -import asyncio - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import litellm -from litellm import Router -from dotenv import load_dotenv -from litellm._uuid import uuid - -load_dotenv() - - -model_list = [ - { - "model_name": "gpt-3.5-turbo", # openai model name - "litellm_params": { # params for litellm completion/embedding call - "model": "azure/chatgpt-v-2", - "api_key": os.getenv("AZURE_API_KEY"), - "api_version": os.getenv("AZURE_API_VERSION"), - "api_base": os.getenv("AZURE_API_BASE"), - }, - "tpm": 240000, - "rpm": 1800, - }, - { - "model_name": "text-embedding-ada-002", - "litellm_params": { - "model": "azure/azure-embedding-model", - "api_key": os.environ["AZURE_API_KEY"], - "api_base": os.environ["AZURE_API_BASE"], - }, - "tpm": 100000, - "rpm": 10000, - }, -] -litellm.set_verbose = True -litellm.cache = litellm.Cache( - type="s3", s3_bucket_name="litellm-my-test-bucket-2", s3_region_name="us-east-1" -) -router = Router( - model_list=model_list, - set_verbose=True, -) # type: ignore - - -@profile -async def router_acompletion(): - # embedding call - question = f"This is a test: {uuid.uuid4()}" * 100 - resp = await router.aembedding(model="text-embedding-ada-002", input=question) - print("embedding-resp", resp) - - response = await router.acompletion( - model="gpt-3.5-turbo", messages=[{"role": "user", "content": question}] - ) - print("completion-resp", response) - return response - - -async def main(): - for i in range(1): - start = time.time() - n = 50 # Number of concurrent tasks - tasks = [router_acompletion() for _ in range(n)] - - chat_completions = await asyncio.gather(*tasks) - - successful_completions = [c for c in chat_completions if c is not None] - - # Write errors to error_log.txt - with open("error_log.txt", "a") as error_log: - for completion in chat_completions: - if isinstance(completion, str): - error_log.write(completion + "\n") - - print(n, time.time() - start, len(successful_completions)) - time.sleep(10) - - -if __name__ == "__main__": - # Blank out contents of error_log.txt - open("error_log.txt", "w").close() - - asyncio.run(main()) diff --git a/cookbook/litellm_router_load_test/memory_usage/send_request.py b/cookbook/litellm_router_load_test/memory_usage/send_request.py deleted file mode 100644 index 6a3473e230f..00000000000 --- a/cookbook/litellm_router_load_test/memory_usage/send_request.py +++ /dev/null @@ -1,28 +0,0 @@ -import requests -from concurrent.futures import ThreadPoolExecutor - -# Replace the URL with your actual endpoint -url = "http://localhost:8000/router_acompletion" - - -def make_request(session): - headers = {"Content-Type": "application/json"} - data = {} # Replace with your JSON payload if needed - - response = session.post(url, headers=headers, json=data) - print(f"Status code: {response.status_code}") - - -# Number of concurrent requests -num_requests = 20 - -# Create a session to reuse the underlying TCP connection -with requests.Session() as session: - # Use ThreadPoolExecutor for concurrent requests - with ThreadPoolExecutor(max_workers=num_requests) as executor: - # Use list comprehension to submit tasks - futures = [executor.submit(make_request, session) for _ in range(num_requests)] - - # Wait for all futures to complete - for future in futures: - future.result() diff --git a/cookbook/litellm_router_load_test/test_loadtest_openai_client.py b/cookbook/litellm_router_load_test/test_loadtest_openai_client.py deleted file mode 100644 index 8c50825be12..00000000000 --- a/cookbook/litellm_router_load_test/test_loadtest_openai_client.py +++ /dev/null @@ -1,73 +0,0 @@ -import sys -import os -from dotenv import load_dotenv - -load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import asyncio -from litellm import Timeout -import time -import openai - -### Test just calling AsyncAzureOpenAI - -openai_client = openai.AsyncAzureOpenAI( - azure_endpoint=os.getenv("AZURE_API_BASE"), - api_key=os.getenv("AZURE_API_KEY"), -) - - -async def call_acompletion(semaphore, input_data): - async with semaphore: - try: - # Use asyncio.wait_for to set a timeout for the task - response = await openai_client.chat.completions.create(**input_data) - # Handle the response as needed - print(response) - return response - except Timeout: - print(f"Task timed out: {input_data}") - return None # You may choose to return something else or raise an exception - - -async def main(): - # Initialize the Router - - # Create a semaphore with a capacity of 100 - semaphore = asyncio.Semaphore(100) - - # List to hold all task references - tasks = [] - start_time_all_tasks = time.time() - # Launch 1000 tasks - for _ in range(500): - task = asyncio.create_task( - call_acompletion( - semaphore, - { - "model": "chatgpt-v-2", - "messages": [{"role": "user", "content": "Hey, how's it going?"}], - }, - ) - ) - tasks.append(task) - - # Wait for all tasks to complete - responses = await asyncio.gather(*tasks) - # Process responses as needed - # Record the end time for all tasks - end_time_all_tasks = time.time() - # Calculate the total time for all tasks - total_time_all_tasks = end_time_all_tasks - start_time_all_tasks - print(f"Total time for all tasks: {total_time_all_tasks} seconds") - - # Calculate the average time per response - average_time_per_response = total_time_all_tasks / len(responses) - print(f"Average time per response: {average_time_per_response} seconds") - print(f"NUMBER OF COMPLETED TASKS: {len(responses)}") - - -# Run the main function -asyncio.run(main()) diff --git a/cookbook/litellm_router_load_test/test_loadtest_router.py b/cookbook/litellm_router_load_test/test_loadtest_router.py deleted file mode 100644 index 280e495e771..00000000000 --- a/cookbook/litellm_router_load_test/test_loadtest_router.py +++ /dev/null @@ -1,87 +0,0 @@ -import sys -import os -from dotenv import load_dotenv - -load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import asyncio -from litellm import Router, Timeout -import time - -### Test calling router async - - -async def call_acompletion(semaphore, router: Router, input_data): - async with semaphore: - try: - # Use asyncio.wait_for to set a timeout for the task - response = await router.acompletion(**input_data) - # Handle the response as needed - print(response) - return response - except Timeout: - print(f"Task timed out: {input_data}") - return None # You may choose to return something else or raise an exception - - -async def main(): - # Initialize the Router - model_list = [ - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "azure/chatgpt-v-2", - "api_key": os.getenv("AZURE_API_KEY"), - "api_base": os.getenv("AZURE_API_BASE"), - "api_version": os.getenv("AZURE_API_VERSION"), - }, - }, - ] - router = Router(model_list=model_list, num_retries=3, timeout=10) - - # Create a semaphore with a capacity of 100 - semaphore = asyncio.Semaphore(100) - - # List to hold all task references - tasks = [] - start_time_all_tasks = time.time() - # Launch 1000 tasks - for _ in range(500): - task = asyncio.create_task( - call_acompletion( - semaphore, - router, - { - "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": "Hey, how's it going?"}], - }, - ) - ) - tasks.append(task) - - # Wait for all tasks to complete - responses = await asyncio.gather(*tasks) - # Process responses as needed - # Record the end time for all tasks - end_time_all_tasks = time.time() - # Calculate the total time for all tasks - total_time_all_tasks = end_time_all_tasks - start_time_all_tasks - print(f"Total time for all tasks: {total_time_all_tasks} seconds") - - # Calculate the average time per response - average_time_per_response = total_time_all_tasks / len(responses) - print(f"Average time per response: {average_time_per_response} seconds") - print(f"NUMBER OF COMPLETED TASKS: {len(responses)}") - - -# Run the main function -asyncio.run(main()) diff --git a/cookbook/litellm_router_load_test/test_loadtest_router_withs3_cache.py b/cookbook/litellm_router_load_test/test_loadtest_router_withs3_cache.py deleted file mode 100644 index b093489be1b..00000000000 --- a/cookbook/litellm_router_load_test/test_loadtest_router_withs3_cache.py +++ /dev/null @@ -1,93 +0,0 @@ -import sys -import os -from dotenv import load_dotenv - -load_dotenv() -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import asyncio -from litellm import Router, Timeout -import time -from litellm.caching.caching import Cache -import litellm - -litellm.cache = Cache( - type="s3", s3_bucket_name="cache-bucket-litellm", s3_region_name="us-west-2" -) - -### Test calling router with s3 Cache - - -async def call_acompletion(semaphore, router: Router, input_data): - async with semaphore: - try: - # Use asyncio.wait_for to set a timeout for the task - response = await router.acompletion(**input_data) - # Handle the response as needed - print(response) - return response - except Timeout: - print(f"Task timed out: {input_data}") - return None # You may choose to return something else or raise an exception - - -async def main(): - # Initialize the Router - model_list = [ - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "gpt-3.5-turbo", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - { - "model_name": "gpt-3.5-turbo", - "litellm_params": { - "model": "azure/chatgpt-v-2", - "api_key": os.getenv("AZURE_API_KEY"), - "api_base": os.getenv("AZURE_API_BASE"), - "api_version": os.getenv("AZURE_API_VERSION"), - }, - }, - ] - router = Router(model_list=model_list, num_retries=3, timeout=10) - - # Create a semaphore with a capacity of 100 - semaphore = asyncio.Semaphore(100) - - # List to hold all task references - tasks = [] - start_time_all_tasks = time.time() - # Launch 1000 tasks - for _ in range(500): - task = asyncio.create_task( - call_acompletion( - semaphore, - router, - { - "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": "Hey, how's it going?"}], - }, - ) - ) - tasks.append(task) - - # Wait for all tasks to complete - responses = await asyncio.gather(*tasks) - # Process responses as needed - # Record the end time for all tasks - end_time_all_tasks = time.time() - # Calculate the total time for all tasks - total_time_all_tasks = end_time_all_tasks - start_time_all_tasks - print(f"Total time for all tasks: {total_time_all_tasks} seconds") - - # Calculate the average time per response - average_time_per_response = total_time_all_tasks / len(responses) - print(f"Average time per response: {average_time_per_response} seconds") - print(f"NUMBER OF COMPLETED TASKS: {len(responses)}") - - -# Run the main function -asyncio.run(main()) diff --git a/cookbook/litellm_test_multiple_llm_demo.ipynb b/cookbook/litellm_test_multiple_llm_demo.ipynb deleted file mode 100644 index f22448e46b9..00000000000 --- a/cookbook/litellm_test_multiple_llm_demo.ipynb +++ /dev/null @@ -1,55 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "adotBkqZSh5g" - }, - "outputs": [], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "code", - "source": [ - "from litellm import completion\n", - "\n", - "## set ENV variables\n", - "os.environ[\"OPENAI_API_KEY\"] = \"openai key\"\n", - "os.environ[\"COHERE_API_KEY\"] = \"cohere key\"\n", - "os.environ[\"REPLICATE_API_KEY\"] = \"replicate key\"\n", - "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", - "\n", - "# openai call\n", - "response = completion(model=\"gpt-3.5-turbo\", messages=messages)\n", - "\n", - "# cohere call\n", - "response = completion(\"command-nightly\", messages)\n", - "\n", - "# replicate call\n", - "response = completion(\"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\", messages)" - ], - "metadata": { - "id": "LeOqznSgSj-z" - }, - "execution_count": null, - "outputs": [] - } - ] -} diff --git a/cookbook/livekit_agent_sdk/README.md b/cookbook/livekit_agent_sdk/README.md deleted file mode 100644 index 6c794165e96..00000000000 --- a/cookbook/livekit_agent_sdk/README.md +++ /dev/null @@ -1,116 +0,0 @@ -# LiveKit Voice Agent with LiteLLM Gateway - -Simple example showing how to use LiveKit's xAI realtime plugin with LiteLLM as a proxy. This lets you switch between xAI, OpenAI, and Azure realtime APIs without changing your code. - -## Quick Start - -### 1. Install dependencies - -```bash -pip install livekit-agents[xai] websockets -``` - -### 2. Start LiteLLM proxy - -```bash -# With xAI -export XAI_API_KEY="your-xai-key" -export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)" -litellm --config config.yaml --port 4000 -``` - -### 3. Run the voice agent - -```bash -export LITELLM_API_KEY="$LITELLM_MASTER_KEY" -python main.py -``` - -Type your message and get a voice response from Grok! - -## Configuration - -Set these environment variables if needed: - -```bash -export LITELLM_PROXY_URL="http://localhost:4000" -export LITELLM_API_KEY="$LITELLM_MASTER_KEY" -export LITELLM_MODEL="grok-voice-agent" -``` - -Or use the defaults - connects to `http://localhost:4000` by default. - -## Example Config File - -Create a `config.yaml` with your realtime models: - -```yaml -model_list: - - model_name: grok-voice-agent - litellm_params: - model: xai/grok-2-vision-1212 - api_key: os.environ/XAI_API_KEY - model_info: - mode: realtime - - - model_name: openai-voice-agent - litellm_params: - model: gpt-4o-realtime-preview - api_key: os.environ/OPENAI_API_KEY - model_info: - mode: realtime - -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY -``` - -Then start: `litellm --config config.yaml --port 4000` - -## How It Works - -LiveKit's xAI plugin connects through LiteLLM proxy by setting `base_url`: - -```python -from livekit.plugins import xai - -model = xai.realtime.RealtimeModel( - voice="ara", - api_key=os.environ["LITELLM_MASTER_KEY"], - base_url="http://localhost:4000", # Point to LiteLLM -) -``` - -## Switching Providers - -Just change the model in your config - no code changes needed: - -**xAI Grok:** -```yaml -model: xai/grok-2-vision-1212 -``` - -**OpenAI:** -```yaml -model: gpt-4o-realtime-preview -``` - -**Azure OpenAI:** -```yaml -model: azure/gpt-4o-realtime-preview -api_base: https://your-endpoint.openai.azure.com/ -``` - -## Why Use LiteLLM? - -- ✅ **Switch providers** without changing agent code -- ✅ **Cost tracking** across all voice sessions -- ✅ **Rate limiting** and budgets -- ✅ **Load balancing** across multiple API keys -- ✅ **Fallbacks** to backup models - -## Learn More - -- [LiveKit xAI Realtime Tutorial](/docs/tutorials/livekit_xai_realtime) -- [xAI Realtime Docs](/docs/providers/xai_realtime) -- [LiveKit Agents Documentation](https://docs.livekit.io/agents/) -- [LiteLLM Realtime API](/docs/realtime) diff --git a/cookbook/livekit_agent_sdk/config.example.yaml b/cookbook/livekit_agent_sdk/config.example.yaml deleted file mode 100644 index 388aa24f501..00000000000 --- a/cookbook/livekit_agent_sdk/config.example.yaml +++ /dev/null @@ -1,20 +0,0 @@ -model_list: - - model_name: grok-voice-agent - litellm_params: - model: xai/grok-2-vision-1212 - api_key: os.environ/XAI_API_KEY - model_info: - mode: realtime - - - model_name: openai-voice-agent - litellm_params: - model: gpt-4o-realtime-preview - api_key: os.environ/OPENAI_API_KEY - model_info: - mode: realtime - -litellm_settings: - drop_params: True - -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY diff --git a/cookbook/livekit_agent_sdk/main.py b/cookbook/livekit_agent_sdk/main.py deleted file mode 100644 index fdc72e518b4..00000000000 --- a/cookbook/livekit_agent_sdk/main.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -Simple xAI Voice Agent using LiveKit SDK with LiteLLM Gateway - -This example shows how to use LiveKit's xAI realtime plugin through LiteLLM proxy. -LiteLLM acts as a unified interface, allowing you to switch between xAI, OpenAI, -and Azure realtime APIs without changing your agent code. -""" - -import asyncio -import json -import os -import websockets - -# Configuration -PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000") -API_KEY = os.environ["LITELLM_API_KEY"] -MODEL = os.getenv("LITELLM_MODEL", "grok-voice-agent") - - -async def run_voice_agent(): - """ - Simple voice agent that: - 1. Connects to xAI realtime API through LiteLLM proxy - 2. Sends a user message - 3. Streams back the response - """ - - url = f"ws://{PROXY_URL.replace('http://', '').replace('https://', '')}/v1/realtime?model={MODEL}" - headers = {"Authorization": f"Bearer {API_KEY}"} - - print(f"🎙️ Connecting to voice agent...") - print(f" Model: {MODEL}") - print(f" Proxy: {PROXY_URL}") - print() - - async with websockets.connect(url, additional_headers=headers) as ws: - # Receive initial connection event - initial = json.loads(await ws.recv()) - print(f"✅ Connected! Event: {initial['type']}\n") - - # Get user input - user_message = input("💬 Your message: ").strip() - if not user_message: - user_message = "Tell me a fun fact about AI!" - - print(f"\n🤖 Sending to {MODEL}...\n") - - # Send user message - await ws.send( - json.dumps( - { - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": user_message}], - }, - } - ) - ) - - # Request response - await ws.send( - json.dumps( - { - "type": "response.create", - "response": {"modalities": ["text", "audio"]}, - } - ) - ) - - # Stream response - print("🎤 Response: ", end="", flush=True) - transcript = [] - - try: - while True: - msg = await asyncio.wait_for(ws.recv(), timeout=15.0) - event = json.loads(msg) - - # Capture transcript deltas - if event["type"] == "response.output_audio_transcript.delta": - delta = event.get("delta", "") - if delta: - print(delta, end="", flush=True) - transcript.append(delta) - - # Done when response completes - elif event["type"] == "response.done": - break - - except asyncio.TimeoutError: - pass - - print("\n") - - if transcript: - print(f"✅ Complete response: {''.join(transcript)}") - - await ws.close() - - -def main(): - """Run the voice agent""" - print("=" * 70) - print("LiveKit xAI Voice Agent via LiteLLM Proxy") - print("=" * 70) - print() - - try: - asyncio.run(run_voice_agent()) - except KeyboardInterrupt: - print("\n\n👋 Goodbye!") - except Exception as e: - print(f"\n❌ Error: {e}") - print("\nMake sure LiteLLM proxy is running:") - print(f" litellm --config config.yaml --port 4000") - - -if __name__ == "__main__": - main() diff --git a/cookbook/livekit_agent_sdk/requirements.txt b/cookbook/livekit_agent_sdk/requirements.txt deleted file mode 100644 index 9e3542fac27..00000000000 --- a/cookbook/livekit_agent_sdk/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -livekit-agents[xai]>=1.3.12 -websockets>=15.0.1 diff --git a/cookbook/logging_observability/LiteLLM_Arize.ipynb b/cookbook/logging_observability/LiteLLM_Arize.ipynb deleted file mode 100644 index 72a082f874d..00000000000 --- a/cookbook/logging_observability/LiteLLM_Arize.ipynb +++ /dev/null @@ -1,172 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "4FbDOmcj2VkM" - }, - "source": [ - "## Use LiteLLM with Arize\n", - "https://docs.litellm.ai/docs/observability/arize_integration\n", - "\n", - "This method uses the litellm proxy to send the data to Arize. The callback is set in the litellm config below, instead of using OpenInference tracing." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "21W8Woog26Ns" - }, - "source": [ - "## Install Dependencies" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "xrjKLBxhxu2L" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: litellm in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (1.54.1)\n", - "Requirement already satisfied: aiohttp in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (3.11.10)\n", - "Requirement already satisfied: click in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (8.1.7)\n", - "Requirement already satisfied: httpx<0.28.0,>=0.23.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.27.2)\n", - "Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (8.5.0)\n", - "Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (3.1.4)\n", - "Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (4.23.0)\n", - "Requirement already satisfied: openai>=1.55.3 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (1.57.1)\n", - "Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (2.10.3)\n", - "Requirement already satisfied: python-dotenv>=0.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (1.0.1)\n", - "Requirement already satisfied: requests<3.0.0,>=2.31.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (2.32.3)\n", - "Requirement already satisfied: tiktoken>=0.7.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.7.0)\n", - "Requirement already satisfied: tokenizers in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from litellm) (0.21.0)\n", - "Requirement already satisfied: anyio in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (4.7.0)\n", - "Requirement already satisfied: certifi in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (2024.8.30)\n", - "Requirement already satisfied: httpcore==1.* in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (1.0.7)\n", - "Requirement already satisfied: idna in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (3.10)\n", - "Requirement already satisfied: sniffio in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpx<0.28.0,>=0.23.0->litellm) (1.3.1)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from httpcore==1.*->httpx<0.28.0,>=0.23.0->litellm) (0.14.0)\n", - "Requirement already satisfied: zipp>=3.20 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from importlib-metadata>=6.8.0->litellm) (3.21.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jinja2<4.0.0,>=3.1.2->litellm) (3.0.2)\n", - "Requirement already satisfied: attrs>=22.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (24.2.0)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2024.10.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.22.3)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (1.9.0)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (0.6.1)\n", - "Requirement already satisfied: tqdm>4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (4.67.1)\n", - "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from openai>=1.55.3->litellm) (4.12.2)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->litellm) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->litellm) (2.27.1)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from requests<3.0.0,>=2.31.0->litellm) (3.4.0)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from requests<3.0.0,>=2.31.0->litellm) (2.0.7)\n", - "Requirement already satisfied: regex>=2022.1.18 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from tiktoken>=0.7.0->litellm) (2024.11.6)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (2.4.4)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.3.1)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.5.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (6.1.0)\n", - "Requirement already satisfied: propcache>=0.2.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (0.2.1)\n", - "Requirement already satisfied: yarl<2.0,>=1.17.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from aiohttp->litellm) (1.18.3)\n", - "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from tokenizers->litellm) (0.26.5)\n", - "Requirement already satisfied: filelock in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (3.16.1)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (2024.10.0)\n", - "Requirement already satisfied: packaging>=20.9 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (24.2)\n", - "Requirement already satisfied: pyyaml>=5.1 in /Users/ericxiao/Documents/arize/.venv/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers->litellm) (6.0.2)\n" - ] - } - ], - "source": [ - "!pip install litellm" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jHEu-TjZ29PJ" - }, - "source": [ - "## Set Env Variables" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "id": "QWd9rTysxsWO" - }, - "outputs": [], - "source": [ - "import litellm\n", - "import os\n", - "from getpass import getpass\n", - "\n", - "os.environ[\"ARIZE_SPACE_KEY\"] = getpass(\"Enter your Arize space key: \")\n", - "os.environ[\"ARIZE_API_KEY\"] = getpass(\"Enter your Arize API key: \")\n", - "os.environ['OPENAI_API_KEY']= getpass(\"Enter your OpenAI API key: \")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's run a completion call and see the traces in Arize" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Hello! Nice to meet you, OpenAI. How can I assist you today?\n" - ] - } - ], - "source": [ - "# set arize as a callback, litellm will send the data to arize\n", - "litellm.callbacks = [\"arize\"]\n", - " \n", - "# openai call\n", - "response = litellm.completion(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n", - " ]\n", - ")\n", - "print(response.choices[0].message.content)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.6" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/logging_observability/LiteLLM_Langfuse.ipynb b/cookbook/logging_observability/LiteLLM_Langfuse.ipynb deleted file mode 100644 index 2a63666e069..00000000000 --- a/cookbook/logging_observability/LiteLLM_Langfuse.ipynb +++ /dev/null @@ -1,197 +0,0 @@ -{ - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "## Use LiteLLM with Langfuse\n", - "https://docs.litellm.ai/docs/observability/langfuse_integration" - ], - "metadata": { - "id": "4FbDOmcj2VkM" - } - }, - { - "cell_type": "markdown", - "source": [ - "## Install Dependencies" - ], - "metadata": { - "id": "21W8Woog26Ns" - } - }, - { - "cell_type": "code", - "source": [ - "!pip install litellm langfuse" - ], - "metadata": { - "id": "xrjKLBxhxu2L" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Set Env Variables" - ], - "metadata": { - "id": "jHEu-TjZ29PJ" - } - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "id": "QWd9rTysxsWO" - }, - "outputs": [], - "source": [ - "import litellm\n", - "from litellm import completion\n", - "import os\n", - "\n", - "# from https://cloud.langfuse.com/\n", - "os.environ[\"LANGFUSE_PUBLIC_KEY\"] = \"\"\n", - "os.environ[\"LANGFUSE_SECRET_KEY\"] = \"\"\n", - "\n", - "\n", - "# OpenAI and Cohere keys\n", - "# You can use any of the litellm supported providers: https://docs.litellm.ai/docs/providers\n", - "os.environ['OPENAI_API_KEY']=\"\"\n", - "os.environ['COHERE_API_KEY']=\"\"\n" - ] - }, - { - "cell_type": "markdown", - "source": [ - "## Set LangFuse as a callback for sending data\n", - "## OpenAI completion call" - ], - "metadata": { - "id": "NodQl0hp3Lma" - } - }, - { - "cell_type": "code", - "source": [ - "# set langfuse as a callback, litellm will send the data to langfuse\n", - "litellm.success_callback = [\"langfuse\"]\n", - "\n", - "# openai call\n", - "response = completion(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n", - " ]\n", - ")\n", - "\n", - "print(response)" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "vNAuwJY1yp_F", - "outputId": "c3a71e26-13f5-4379-fac9-409290ba79bb" - }, - "execution_count": 8, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "{\n", - " \"id\": \"chatcmpl-85nP4xHdAP3jAcGneIguWATS9qdoO\",\n", - " \"object\": \"chat.completion\",\n", - " \"created\": 1696392238,\n", - " \"model\": \"gpt-3.5-turbo-0613\",\n", - " \"choices\": [\n", - " {\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Hello! How can I assist you today?\"\n", - " },\n", - " \"finish_reason\": \"stop\"\n", - " }\n", - " ],\n", - " \"usage\": {\n", - " \"prompt_tokens\": 15,\n", - " \"completion_tokens\": 9,\n", - " \"total_tokens\": 24\n", - " }\n", - "}\n" - ] - } - ] - }, - { - "cell_type": "code", - "source": [ - "# we set langfuse as a callback in the prev cell\n", - "# cohere call\n", - "response = completion(\n", - " model=\"command-nightly\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"Hi 👋 - i'm cohere\"}\n", - " ]\n", - ")\n", - "\n", - "print(response)" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "2PMSLc_FziJL", - "outputId": "1c37605e-b406-4ffc-aafd-e1983489c6be" - }, - "execution_count": 9, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "{\n", - " \"object\": \"chat.completion\",\n", - " \"choices\": [\n", - " {\n", - " \"finish_reason\": \"stop\",\n", - " \"index\": 0,\n", - " \"message\": {\n", - " \"content\": \" Nice to meet you, Cohere! I'm excited to be meeting new members of the AI community\",\n", - " \"role\": \"assistant\",\n", - " \"logprobs\": null\n", - " }\n", - " }\n", - " ],\n", - " \"id\": \"chatcmpl-a14e903f-4608-4ceb-b996-8ebdf21360ca\",\n", - " \"created\": 1696392247.3313863,\n", - " \"model\": \"command-nightly\",\n", - " \"usage\": {\n", - " \"prompt_tokens\": 8,\n", - " \"completion_tokens\": 20,\n", - " \"total_tokens\": 28\n", - " }\n", - "}\n" - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/cookbook/logging_observability/LiteLLM_Lunary.ipynb b/cookbook/logging_observability/LiteLLM_Lunary.ipynb deleted file mode 100644 index 3b1dc5d5e25..00000000000 --- a/cookbook/logging_observability/LiteLLM_Lunary.ipynb +++ /dev/null @@ -1,348 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "4FbDOmcj2VkM" - }, - "source": [ - "## Use LiteLLM with Langfuse\n", - "https://docs.litellm.ai/docs/observability/langfuse_integration" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "21W8Woog26Ns" - }, - "source": [ - "## Install Dependencies" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "xrjKLBxhxu2L" - }, - "outputs": [], - "source": [ - "%pip install litellm lunary" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jHEu-TjZ29PJ" - }, - "source": [ - "## Set Env Variables" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "id": "QWd9rTysxsWO" - }, - "outputs": [], - "source": [ - "import litellm\n", - "from litellm import completion\n", - "import os\n", - "\n", - "# from https://app.lunary.ai/\n", - "os.environ[\"LUNARY_PUBLIC_KEY\"] = \"\"\n", - "\n", - "\n", - "# LLM provider keys\n", - "# You can use any of the litellm supported providers: https://docs.litellm.ai/docs/providers\n", - "os.environ['OPENAI_API_KEY'] = \"\"\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NodQl0hp3Lma" - }, - "source": [ - "## Set Lunary as a callback for sending data\n", - "## OpenAI completion call" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "vNAuwJY1yp_F", - "outputId": "c3a71e26-13f5-4379-fac9-409290ba79bb" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))]ModelResponse(id='chatcmpl-8xIWykI0GiJSmYtXYuB8Z363kpIBm', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))], created=1709143276, model='gpt-3.5-turbo-0125', object='chat.completion', system_fingerprint='fp_86156a94a0', usage=Usage(completion_tokens=9, prompt_tokens=15, total_tokens=24))\n", - "\n", - "[Lunary] Add event: {\n", - " \"event\": \"start\",\n", - " \"type\": \"llm\",\n", - " \"name\": \"gpt-3.5-turbo\",\n", - " \"runId\": \"a363776a-bd07-4474-bce2-193067f01b2e\",\n", - " \"timestamp\": \"2024-02-28T18:01:15.188153+00:00\",\n", - " \"input\": {\n", - " \"role\": \"user\",\n", - " \"content\": \"Hi \\ud83d\\udc4b - i'm openai\"\n", - " },\n", - " \"extra\": {},\n", - " \"runtime\": \"litellm\",\n", - " \"metadata\": {}\n", - "}\n", - "\n", - "\n", - "[Lunary] Add event: {\n", - " \"event\": \"end\",\n", - " \"type\": \"llm\",\n", - " \"runId\": \"a363776a-bd07-4474-bce2-193067f01b2e\",\n", - " \"timestamp\": \"2024-02-28T18:01:16.846581+00:00\",\n", - " \"output\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Hello! How can I assist you today?\"\n", - " },\n", - " \"runtime\": \"litellm\",\n", - " \"tokensUsage\": {\n", - " \"completion\": 9,\n", - " \"prompt\": 15\n", - " }\n", - "}\n", - "\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "--- Logging error ---\n", - "Traceback (most recent call last):\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 537, in _make_request\n", - " response = conn.getresponse()\n", - " ^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connection.py\", line 466, in getresponse\n", - " httplib_response = super().getresponse()\n", - " ^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 1423, in getresponse\n", - " response.begin()\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 331, in begin\n", - " version, status, reason = self._read_status()\n", - " ^^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 292, in _read_status\n", - " line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/socket.py\", line 707, in readinto\n", - " return self._sock.recv_into(b)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^\n", - "TimeoutError: timed out\n", - "\n", - "The above exception was the direct cause of the following exception:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/adapters.py\", line 486, in send\n", - " resp = conn.urlopen(\n", - " ^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 847, in urlopen\n", - " retries = retries.increment(\n", - " ^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/util/retry.py\", line 470, in increment\n", - " raise reraise(type(error), error, _stacktrace)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/util/util.py\", line 39, in reraise\n", - " raise value\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 793, in urlopen\n", - " response = self._make_request(\n", - " ^^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 539, in _make_request\n", - " self._raise_timeout(err=e, url=url, timeout_value=read_timeout)\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 370, in _raise_timeout\n", - " raise ReadTimeoutError(\n", - "urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 59, in send_batch\n", - " response = requests.post(\n", - " ^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/api.py\", line 115, in post\n", - " return request(\"post\", url, data=data, json=json, **kwargs)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/api.py\", line 59, in request\n", - " return session.request(method=method, url=url, **kwargs)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/sessions.py\", line 589, in request\n", - " resp = self.send(prep, **send_kwargs)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/sessions.py\", line 703, in send\n", - " r = adapter.send(request, **kwargs)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/adapters.py\", line 532, in send\n", - " raise ReadTimeout(e, request=request)\n", - "requests.exceptions.ReadTimeout: HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 1160, in emit\n", - " msg = self.format(record)\n", - " ^^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 999, in format\n", - " return fmt.format(record)\n", - " ^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 703, in format\n", - " record.message = record.getMessage()\n", - " ^^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 392, in getMessage\n", - " msg = msg % self.args\n", - " ~~~~^~~~~~~~~~~\n", - "TypeError: not all arguments converted during string formatting\n", - "Call stack:\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py\", line 1030, in _bootstrap\n", - " self._bootstrap_inner()\n", - " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py\", line 1073, in _bootstrap_inner\n", - " self.run()\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 24, in run\n", - " self.send_batch()\n", - " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 73, in send_batch\n", - " logging.error(\"[Lunary] Error sending events\", e)\n", - "Message: '[Lunary] Error sending events'\n", - "Arguments: (ReadTimeout(ReadTimeoutError(\"HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\")),)\n" - ] - } - ], - "source": [ - "# set langfuse as a callback, litellm will send the data to langfuse\n", - "litellm.success_callback = [\"lunary\"]\n", - "\n", - "# openai call\n", - "response = completion(\n", - " model=\"gpt-3.5-turbo\",\n", - " messages=[\n", - " {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n", - " ]\n", - ")\n", - "\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Using LiteLLM with Lunary Templates\n", - "\n", - "You can use LiteLLM seamlessly with Lunary templates to manage your prompts and completions.\n", - "\n", - "Assuming you have created a template \"test-template\" with a variable \"question\", you can use it like this:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "2PMSLc_FziJL", - "outputId": "1c37605e-b406-4ffc-aafd-e1983489c6be" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))]ModelResponse(id='chatcmpl-8xIXegwpudg4YKnLB6pmpFGXqTHcH', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))], created=1709143318, model='gpt-4-0125-preview', object='chat.completion', system_fingerprint='fp_c8aa5a06d6', usage=Usage(completion_tokens=9, prompt_tokens=21, total_tokens=30))\n", - "\n", - "[Lunary] Add event: {\n", - " \"event\": \"start\",\n", - " \"type\": \"llm\",\n", - " \"name\": \"gpt-4-turbo-preview\",\n", - " \"runId\": \"3a5b698d-cb55-4b3b-ab6d-04d2b99e40cb\",\n", - " \"timestamp\": \"2024-02-28T18:01:56.746249+00:00\",\n", - " \"input\": [\n", - " {\n", - " \"role\": \"system\",\n", - " \"content\": \"You are an helpful assistant.\"\n", - " },\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"Hi! Hello!\"\n", - " }\n", - " ],\n", - " \"extra\": {\n", - " \"temperature\": 1,\n", - " \"max_tokens\": 100\n", - " },\n", - " \"runtime\": \"litellm\",\n", - " \"metadata\": {}\n", - "}\n", - "\n", - "\n", - "[Lunary] Add event: {\n", - " \"event\": \"end\",\n", - " \"type\": \"llm\",\n", - " \"runId\": \"3a5b698d-cb55-4b3b-ab6d-04d2b99e40cb\",\n", - " \"timestamp\": \"2024-02-28T18:01:58.741244+00:00\",\n", - " \"output\": {\n", - " \"role\": \"assistant\",\n", - " \"content\": \"Hello! How can I assist you today?\"\n", - " },\n", - " \"runtime\": \"litellm\",\n", - " \"tokensUsage\": {\n", - " \"completion\": 9,\n", - " \"prompt\": 21\n", - " }\n", - "}\n", - "\n", - "\n" - ] - } - ], - "source": [ - "import lunary\n", - "from litellm import completion\n", - "\n", - "template = lunary.render_template(\"test-template\", {\"question\": \"Hello!\"})\n", - "\n", - "response = completion(**template)\n", - "\n", - "print(response)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb b/cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb deleted file mode 100644 index 0baaab3f49f..00000000000 --- a/cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb +++ /dev/null @@ -1,252 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## LLM Ops Stack - LiteLLM Proxy + Langfuse \n", - "\n", - "This notebook demonstrates how to use LiteLLM Proxy with Langfuse \n", - "- Use LiteLLM Proxy for calling 100+ LLMs in OpenAI format\n", - "- Use Langfuse for viewing request / response traces \n", - "\n", - "\n", - "In this notebook we will setup LiteLLM Proxy to make requests to OpenAI, Anthropic, Bedrock and automatically log traces to Langfuse." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Setup LiteLLM Proxy\n", - "\n", - "### 1.1 Define .env variables \n", - "Define .env variables on the container that litellm proxy is running on.\n", - "```bash\n", - "## LLM API Keys\n", - "OPENAI_API_KEY=sk-proj-1234567890\n", - "ANTHROPIC_API_KEY=sk-ant-api03-1234567890\n", - "AWS_ACCESS_KEY_ID=1234567890\n", - "AWS_SECRET_ACCESS_KEY=1234567890\n", - "\n", - "## Langfuse Logging \n", - "LANGFUSE_PUBLIC_KEY=\"pk-lf-xxxx9\"\n", - "LANGFUSE_SECRET_KEY=\"sk-lf-xxxx9\"\n", - "LANGFUSE_HOST=\"https://us.cloud.langfuse.com\"\n", - "```\n", - "\n", - "\n", - "### 1.1 Setup LiteLLM Proxy Config yaml \n", - "```yaml\n", - "model_list:\n", - " - model_name: gpt-4o\n", - " litellm_params:\n", - " model: openai/gpt-4o\n", - " api_key: os.environ/OPENAI_API_KEY\n", - " - model_name: claude-3-5-sonnet-20241022\n", - " litellm_params:\n", - " model: anthropic/claude-3-5-sonnet-20241022\n", - " api_key: os.environ/ANTHROPIC_API_KEY\n", - " - model_name: us.amazon.nova-micro-v1:0\n", - " litellm_params:\n", - " model: bedrock/us.amazon.nova-micro-v1:0\n", - " aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID\n", - " aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY\n", - "\n", - "litellm_settings:\n", - " callbacks: [\"langfuse\"]\n", - "\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Make LLM Requests to LiteLLM Proxy\n", - "\n", - "Now we will make our first LLM request to LiteLLM Proxy" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.1 Setup Client Side Variables to point to LiteLLM Proxy\n", - "Set `LITELLM_PROXY_BASE_URL` to the base url of the LiteLLM Proxy and `LITELLM_VIRTUAL_KEY` to the virtual key you want to use for Authentication to LiteLLM Proxy. (Note: In this initial setup you can)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "LITELLM_PROXY_BASE_URL=\"http://0.0.0.0:4000\"\n", - "LITELLM_VIRTUAL_KEY=\"sk-oXXRa1xxxxxxxxxxx\"" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "ChatCompletion(id='chatcmpl-B0sq6QkOKNMJ0dwP3x7OoMqk1jZcI', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content='Langfuse is a platform designed to monitor, observe, and troubleshoot AI and large language model (LLM) applications. It provides features that help developers gain insights into how their AI systems are performing, make debugging easier, and optimize the deployment of models. Langfuse allows for tracking of model interactions, collecting telemetry, and visualizing data, which is crucial for understanding the behavior of AI models in production environments. This kind of tool is particularly useful for developers working with language models who need to ensure reliability and efficiency in their applications.', refusal=None, role='assistant', audio=None, function_call=None, tool_calls=None))], created=1739550502, model='gpt-4o-2024-08-06', object='chat.completion', service_tier='default', system_fingerprint='fp_523b9b6e5f', usage=CompletionUsage(completion_tokens=109, prompt_tokens=13, total_tokens=122, completion_tokens_details=CompletionTokensDetails(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0), prompt_tokens_details=PromptTokensDetails(audio_tokens=0, cached_tokens=0)))" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import openai\n", - "client = openai.OpenAI(\n", - " api_key=LITELLM_VIRTUAL_KEY,\n", - " base_url=LITELLM_PROXY_BASE_URL\n", - ")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"gpt-4o\",\n", - " messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"what is Langfuse?\"\n", - " }\n", - " ],\n", - ")\n", - "\n", - "response" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.3 View Traces on Langfuse\n", - "LiteLLM will send the request / response, model, tokens (input + output), cost to Langfuse.\n", - "\n", - "![image_description](litellm_proxy_langfuse.png)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.4 Call Anthropic, Bedrock models \n", - "\n", - "Now we can call `us.amazon.nova-micro-v1:0` and `claude-3-5-sonnet-20241022` models defined on your config.yaml both in the OpenAI request / response format." - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "ChatCompletion(id='chatcmpl-7756e509-e61f-4f5e-b5ae-b7a41013522a', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content=\"Langfuse is an observability tool designed specifically for machine learning models and applications built with natural language processing (NLP) and large language models (LLMs). It focuses on providing detailed insights into how these models perform in real-world scenarios. Here are some key features and purposes of Langfuse:\\n\\n1. **Real-time Monitoring**: Langfuse allows developers to monitor the performance of their NLP and LLM applications in real time. This includes tracking the inputs and outputs of the models, as well as any errors or issues that arise during operation.\\n\\n2. **Error Tracking**: It helps in identifying and tracking errors in the models' outputs. By analyzing incorrect or unexpected responses, developers can pinpoint where and why errors occur, facilitating more effective debugging and improvement.\\n\\n3. **Performance Metrics**: Langfuse provides various performance metrics, such as latency, throughput, and error rates. These metrics help developers understand how well their models are performing under different conditions and workloads.\\n\\n4. **Traceability**: It offers detailed traceability of requests and responses, allowing developers to follow the path of a request through the system and see how it is processed by the model at each step.\\n\\n5. **User Feedback Integration**: Langfuse can integrate user feedback to provide context for model outputs. This helps in understanding how real users are interacting with the model and how its outputs align with user expectations.\\n\\n6. **Customizable Dashboards**: Users can create custom dashboards to visualize the data collected by Langfuse. These dashboards can be tailored to highlight the most important metrics and insights for a specific application or team.\\n\\n7. **Alerting and Notifications**: It can set up alerts for specific conditions or errors, notifying developers when something goes wrong or when performance metrics fall outside of acceptable ranges.\\n\\nBy providing comprehensive observability for NLP and LLM applications, Langfuse helps developers to build more reliable, accurate, and user-friendly models and services.\", refusal=None, role='assistant', audio=None, function_call=None, tool_calls=None))], created=1739554005, model='us.amazon.nova-micro-v1:0', object='chat.completion', service_tier=None, system_fingerprint=None, usage=CompletionUsage(completion_tokens=380, prompt_tokens=5, total_tokens=385, completion_tokens_details=None, prompt_tokens_details=None))" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import openai\n", - "client = openai.OpenAI(\n", - " api_key=LITELLM_VIRTUAL_KEY,\n", - " base_url=LITELLM_PROXY_BASE_URL\n", - ")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"us.amazon.nova-micro-v1:0\",\n", - " messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"what is Langfuse?\"\n", - " }\n", - " ],\n", - ")\n", - "\n", - "response" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Advanced - Set Langfuse Trace ID, Tags, Metadata \n", - "\n", - "Here is an example of how you can set Langfuse specific params on your client side request. See full list of supported langfuse params [here](https://docs.litellm.ai/docs/observability/langfuse_integration)\n", - "\n", - "You can view the logged trace of this request [here](https://us.cloud.langfuse.com/project/clvlhdfat0007vwb74m9lvfvi/traces/567890?timestamp=2025-02-14T17%3A30%3A26.709Z)" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "ChatCompletion(id='chatcmpl-789babd5-c064-4939-9093-46e4cd2e208a', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content=\"Langfuse is an observability platform designed specifically for monitoring and improving the performance of natural language processing (NLP) models and applications. It provides developers with tools to track, analyze, and optimize how their language models interact with users and handle natural language inputs.\\n\\nHere are some key features and benefits of Langfuse:\\n\\n1. **Real-Time Monitoring**: Langfuse allows developers to monitor their NLP applications in real time. This includes tracking user interactions, model responses, and overall performance metrics.\\n\\n2. **Error Tracking**: It helps in identifying and tracking errors in the model's responses. This can include incorrect, irrelevant, or unsafe outputs.\\n\\n3. **User Feedback Integration**: Langfuse enables the collection of user feedback directly within the platform. This feedback can be used to identify areas for improvement in the model's performance.\\n\\n4. **Performance Metrics**: The platform provides detailed metrics and analytics on model performance, including latency, throughput, and accuracy.\\n\\n5. **Alerts and Notifications**: Developers can set up alerts to notify them of any significant issues or anomalies in model performance.\\n\\n6. **Debugging Tools**: Langfuse offers tools to help developers debug and refine their models by providing insights into how the model processes different types of inputs.\\n\\n7. **Integration with Development Workflows**: It integrates seamlessly with various development environments and CI/CD pipelines, making it easier to incorporate observability into the development process.\\n\\n8. **Customizable Dashboards**: Users can create custom dashboards to visualize the data in a way that best suits their needs.\\n\\nLangfuse aims to help developers build more reliable, accurate, and user-friendly NLP applications by providing them with the tools to observe and improve how their models perform in real-world scenarios.\", refusal=None, role='assistant', audio=None, function_call=None, tool_calls=None))], created=1739554281, model='us.amazon.nova-micro-v1:0', object='chat.completion', service_tier=None, system_fingerprint=None, usage=CompletionUsage(completion_tokens=346, prompt_tokens=5, total_tokens=351, completion_tokens_details=None, prompt_tokens_details=None))" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import openai\n", - "client = openai.OpenAI(\n", - " api_key=LITELLM_VIRTUAL_KEY,\n", - " base_url=LITELLM_PROXY_BASE_URL\n", - ")\n", - "\n", - "response = client.chat.completions.create(\n", - " model=\"us.amazon.nova-micro-v1:0\",\n", - " messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"what is Langfuse?\"\n", - " }\n", - " ],\n", - " extra_body={\n", - " \"metadata\": {\n", - " \"generation_id\": \"1234567890\",\n", - " \"trace_id\": \"567890\",\n", - " \"trace_user_id\": \"user_1234567890\",\n", - " \"tags\": [\"tag1\", \"tag2\"]\n", - " }\n", - " }\n", - ")\n", - "\n", - "response" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## " - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/cookbook/logging_observability/litellm_proxy_langfuse.png b/cookbook/logging_observability/litellm_proxy_langfuse.png deleted file mode 100644 index 6b0691e6a5f..00000000000 Binary files a/cookbook/logging_observability/litellm_proxy_langfuse.png and /dev/null differ diff --git a/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md b/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md deleted file mode 100644 index 4a6fa9367fc..00000000000 --- a/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md +++ /dev/null @@ -1,543 +0,0 @@ -# LiteLLM Release Notes Generation Instructions - -This document provides comprehensive instructions for AI agents to generate release notes for LiteLLM following the established format and style. - -## Required Inputs - -1. **Release Version** (e.g., `v1.77.3-stable`) -2. **PR Diff/Changelog** - List of PRs with titles and contributors -3. **Previous Version Commit Hash** - To compare model pricing changes -4. **Reference Release Notes** - Use recent stable releases (v1.76.3-stable, v1.77.2-stable) as templates for consistent formatting - -### Resolving Staging PRs - -The GitHub release page (e.g. `https://github.com/BerriAI/litellm/releases/tag/v1.83.3-stable`) does **not** list the real changelog directly. The "What's Changed" section contains **staging PRs** that each bundle many individual commits/PRs. For example: - -- `Litellm oss staging 03 14 2026 by @RheagalFire in #23686` -- `Litellm ryan march 16 by @ryan-crabbe in #23822` - -To get the real changelog, you MUST click into each staging PR (e.g. `#23686`, `#23822`), open its **Commits** tab, and extract every underlying commit/PR (look for the `(#NNNNN)` suffix on commit titles). Those underlying PRs — not the staging PRs — are what get categorized in the release notes. Never treat a staging PR title as a single changelog entry. - -**IMPORTANT — staging PRs are not the complete source.** Some PRs land on the release branch *before* the staging PRs and are therefore not reachable via `gh api /pulls//commits`. GitHub's auto-generated "What's Changed" on the release page also misses these. To catch every PR in the release, you MUST additionally walk the full git log range between the previous release's commit and this release's commit: - -```bash -git fetch origin --tags -git log .. --oneline | grep -oE '#[0-9]+' | sort -u -``` - -Union the PR set from the staging-PR walk with the PR set from `git log`. Any PR in `git log` but missing from your staging-expanded set is almost certainly a content PR that merged directly to the release branch — fetch its title/body with `gh pr view ` and categorize it. Do not trust the GH release body or the staging PRs alone as the authoritative list. - -**Sanity check for new contributors.** The GH release body's "New Contributors" list is a *floor*, not authoritative. For every PR author who appears in the release (including underlying PRs from staging and PRs found only via `git log`), verify whether they are a first-time contributor by running: - -```bash -gh api "search/issues?q=is:pr+author:+repo:BerriAI/litellm+is:merged&sort=created&order=asc" --jq '.items[0] | {n:.number, merged:.closed_at}' -``` - -If the author's earliest merged PR number matches a PR in this release window, they are a new contributor. If their earliest merged PR predates the previous release tag, they are not. Do not copy the GH release body's list blindly — it can both miss contributors (PRs that merged via an older dev branch) and falsely include contributors whose "first" PR in this window was not actually their first ever. - -## Step-by-Step Process - -### 1. Initial Setup and Analysis - -```bash -# Check git diff for model pricing changes -git diff HEAD -- model_prices_and_context_window.json -``` - -**Key Analysis Points:** -- New models added (look for new entries) -- Deprecated models removed (look for deleted entries) -- Pricing updates (look for cost changes) -- Feature support changes (tool calling, reasoning, etc.) - -### 2. Release Notes Structure - -Follow this exact structure based on recent stable releases (v1.76.3-stable, v1.77.2-stable, v1.77.5-stable): - -```markdown ---- -title: "v1.77.X-stable - [Key Theme]" -slug: "v1-77-X" -date: YYYY-MM-DDTHH:mm:ss -authors: [standard author block] -hide_table_of_contents: false ---- - -## Deploy this version -[Docker and pip installation tabs] - -## Key Highlights -[3-5 bullet points of major features - prioritize MCP OAuth 2.0, scheduled key rotations, and major model updates] - -## New Providers and Endpoints - -### New Providers -[Table with Provider, Supported Endpoints, Description columns] - -### New LLM API Endpoints -[Optional table for new endpoint additions with Endpoint, Method, Description, Documentation columns] - -## New Models / Updated Models -#### New Model Support -[Model pricing table] - -#### Features -[Provider-specific features organized by provider] - -### Bug Fixes -[Provider-specific bug fixes organized by provider] - -## LLM API Endpoints -#### Features -[API-specific features organized by API type] - -#### Bugs -[General bug fixes] - -## Management Endpoints / UI -#### Features -[UI and management features - group by functionality like Proxy CLI Auth, Virtual Keys, Models + Endpoints] - -#### Bugs -[Management-related bug fixes] - -## AI Integrations - -### Logging -[Logging integrations organized by provider with proper doc links, includes General subsection] - -### Guardrails -[Guardrail-specific features and fixes] - -### Prompt Management -[Prompt management integrations like BitBucket] - -### Secret Managers -[Secret manager integrations - AWS, HashiCorp Vault, CyberArk, etc.] - -## Spend Tracking, Budgets and Rate Limiting -[Cost tracking, service tier pricing, rate limiting improvements] - -## MCP Gateway -[MCP-specific features, OAuth 2.0, configuration improvements] - -## Performance / Loadbalancing / Reliability improvements -[Infrastructure improvements, memory fixes, performance optimizations] - -## Documentation Updates -[Documentation improvements, guides, corrections - separate section for visibility] - -## New Contributors -[List of first-time contributors] - -## Full Changelog -[Link to GitHub comparison] -``` - -### 3. Categorization Rules - -**Performance Improvements:** -- RPS improvements -- Memory optimizations -- CPU usage optimizations -- Timeout controls -- Worker configuration -- Memory leak fixes -- Cache performance improvements -- Database connection management -- Dependency management (fastuuid, etc.) -- Configuration management - -**New Models/Updated Models:** -- Extract from model_prices_and_context_window.json diff -- Create tables with: Provider, Model, Context Window, Input Cost, Output Cost, Features -- **Structure:** - - `#### New Model Support` - pricing table - - `#### Features` - organized by provider with documentation links - - `### Bug Fixes` - provider-specific bug fixes - - `#### New Provider Support` - major new provider integrations -- Group by provider with proper doc links: `**[Provider Name](../../docs/providers/[provider])**` -- Use bullet points under each provider for multiple features -- Separate features from bug fixes clearly - -**LLM API Endpoints:** -- **Structure:** - - `#### Features` - organized by API type (Responses API, Batch API, etc.) - - `#### Bugs` - general bug fixes under **General** category -- **API Categories:** - - Responses API - - Batch API - - CountTokens API - - Images API - - Video Generation (if applicable) - - General (miscellaneous improvements) -- Use proper documentation links for each API type - -**UI/Management:** -- Authentication changes -- Dashboard improvements -- Team management -- Key management -- Proxy CLI authentication and improvements -- Virtual key management and scheduled rotations -- SSO configuration fixes -- Admin settings updates -- Management routes and endpoints - -**AI Integrations:** -- **Structure:** - - `### Logging` - organized by integration provider with proper doc links, includes **General** subsection - - `### Guardrails` - guardrail-specific features and fixes - - `### Prompt Management` - prompt management integrations - - `### Secret Managers` - secret manager integrations -- **Logging Categories:** - - **[DataDog](../../docs/proxy/logging#datadog)** - group all DataDog-related changes - - **[Langfuse](../../docs/proxy/logging#langfuse)** - Langfuse-specific features - - **[Prometheus](../../docs/proxy/logging#prometheus)** - monitoring improvements - - **[PostHog](../../docs/observability/posthog)** - observability integration - - **[SQS](../../docs/proxy/logging#sqs)** - SQS logging features - - **[Opik](../../docs/proxy/logging#opik)** - Opik integration improvements - - **[Arize Phoenix](../../docs/observability/arize_phoenix)** - Arize Phoenix integration - - **General** - miscellaneous logging features like callback controls, sensitive data masking - - Other logging providers with proper doc links -- **Guardrail Categories:** - - LakeraAI, Presidio, Noma, Grayswan, IBM Guardrails, and other guardrail providers -- **Prompt Management:** - - BitBucket, GitHub, and other prompt management integrations - - Prompt versioning, testing, and UI features -- **Secret Managers:** - - **[AWS Secrets Manager](../../docs/secret_managers)** - AWS secret manager features - - **[HashiCorp Vault](../../docs/secret_managers)** - Vault integrations - - **[CyberArk](../../docs/secret_managers)** - CyberArk integrations - - **General** - cross-secret-manager features -- Use bullet points under each provider for multiple features -- Separate logging, guardrails, prompt management, and secret managers clearly - -### 4. Documentation Linking Strategy - -**Link to docs when:** -- New provider support added -- Significant feature additions -- API endpoint changes -- Integration additions - -**Link format:** `../../docs/[category]/[specific_doc]` - -**Common doc paths:** -- `../../docs/providers/[provider]` - Provider-specific docs -- `../../docs/image_generation` - Image generation -- `../../docs/video_generation` - Video generation (if exists) -- `../../docs/response_api` - Responses API -- `../../docs/proxy/logging` - Logging integrations -- `../../docs/proxy/guardrails` - Guardrails -- `../../docs/pass_through/[provider]` - Passthrough endpoints - -### 5. Model Table Generation - -From git diff analysis, create tables like: - -```markdown -| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features | -| -------- | ----- | -------------- | ------------------- | -------------------- | -------- | -| OpenRouter | `openrouter/openai/gpt-4.1` | 1M | $2.00 | $8.00 | Chat completions with vision | -``` - -**Extract from JSON:** -- `max_input_tokens` → Context Window -- `input_cost_per_token` × 1,000,000 → Input cost -- `output_cost_per_token` × 1,000,000 → Output cost -- `supports_*` fields → Features -- Special pricing fields (per image, per second) for generation models - -### 6. PR Categorization Logic - -**By Keywords in PR Title:** -- `[Perf]`, `Performance`, `RPS` → Performance Improvements -- `[Bug]`, `[Bug Fix]`, `Fix` → Bug Fixes section -- `[Feat]`, `[Feature]`, `Add support` → Features section -- `[Docs]` → Documentation Updates section -- Provider names (Gemini, OpenAI, etc.) → Group under provider -- `MCP`, `oauth`, `Model Context Protocol` → MCP Gateway -- `service_tier`, `priority`, `cost tracking` → Spend Tracking, Budgets and Rate Limiting - -**By PR Content Analysis:** -- New model additions → New Models section -- UI changes → Management Endpoints/UI -- Logging/observability → Logging/Guardrail/Prompt Management Integrations -- Rate limiting/budgets → Spend Tracking, Budgets and Rate Limiting -- Authentication → Management Endpoints/UI -- MCP-related changes → MCP Gateway -- Documentation updates → Documentation Updates -- Performance/memory fixes → Performance/Loadbalancing/Reliability improvements - -**Special Categorization Rules:** -- **Service tier pricing** (OpenAI priority/flex) → Spend Tracking section (NOT provider features) -- **Cost breakdown in logging** → Spend Tracking section -- **MCP configuration/OAuth** → MCP Gateway (NOT General Proxy Improvements) -- **All documentation PRs** → Documentation Updates section for visibility -- **Callback controls/logging features** → AI Integrations > Logging > General -- **Secret manager features** → AI Integrations > Secret Managers -- **Video generation tag-based routing** → LLM API Endpoints > Video Generation API - -### 7. Writing Style Guidelines - -**Tone:** -- Professional but accessible -- Focus on user impact -- Highlight breaking changes clearly -- Use active voice - -**Formatting:** -- Use consistent markdown formatting -- Include PR links: `[PR #XXXXX](https://github.com/BerriAI/litellm/pull/XXXXX)` -- Use code blocks for configuration examples -- Bold important terms and section headers - -**Warnings/Notes:** -- Add warning boxes for breaking changes -- Include migration instructions when needed -- Provide override options for default changes - -### 8. Quality Checks - -**Before finalizing:** -- Verify all PR links work -- Check documentation links are valid -- Ensure model pricing is accurate -- Confirm provider names are consistent -- Review for typos and formatting issues -- **Count PRs by section** - Provide final count like: - ``` - ## MM/DD/YYYY - * New Models / Updated Models: XX - * LLM API Endpoints: XX - * Management Endpoints / UI: XX - * Logging / Guardrail / Prompt Management Integrations: XX - * Spend Tracking, Budgets and Rate Limiting: XX - * MCP Gateway: XX - * Performance / Loadbalancing / Reliability improvements: XX - * Documentation Updates: XX - ``` - -### 9. Common Patterns to Follow - -**Performance Changes:** -```markdown -- **+400 RPS Performance Boost** - Description - [PR #XXXXX](link) -``` - -**New Models:** -Always include pricing table and feature highlights - -**Breaking Changes:** -```markdown -:::warning -This release has a known issue... -::: -``` - -**Provider Features (New Models / Updated Models section):** -```markdown -#### Features - -- **[Provider Name](../../docs/providers/provider)** - - Feature description - [PR #XXXXX](link) - - Another feature description - [PR #YYYYY](link) -``` - -**API Features (LLM API Endpoints section):** -```markdown -#### Features - -- **[API Name](../../docs/api_path)** - - Feature description - [PR #XXXXX](link) - - Another feature - [PR #YYYYY](link) -- **General** - - Miscellaneous improvements - [PR #ZZZZZ](link) -``` - -**Integration Features (Logging / Guardrail Integrations section):** -```markdown -#### Features - -- **[Integration Name](../../docs/proxy/logging#integration)** - - Feature description - [PR #XXXXX](link) - - Bug fix description - [PR #YYYYY](link) -``` - -**Bug Fixes Pattern:** -```markdown -### Bug Fixes - -- **[Provider/Component Name](../../docs/providers/provider)** - - Bug fix description - [PR #XXXXX](link) -``` - -### 10. Missing Documentation Check - -**Review for missing docs:** -- New providers without documentation -- New API endpoints without examples -- Complex features without guides -- Integration setup instructions - -**Flag for documentation needs:** -- New provider integrations -- Significant API changes -- Complex configuration options -- Migration requirements - -### 11. New Sections and Categories (Added in v1.77.5) - -**MCP Gateway Section:** -- All MCP-related changes go here (not in General Proxy Improvements) -- OAuth 2.0 flow improvements -- MCP configuration and tools -- Server management features - -**Spend Tracking, Budgets and Rate Limiting Section:** -- Service tier pricing (OpenAI priority/flex pricing) -- Cost tracking and breakdown features -- Rate limiting improvements (Parallel Request Limiter v3) -- Priority reservation fixes -- Metadata handling for rate limiting - -**Documentation Updates Section:** -- Create separate section for all documentation improvements -- Include provider documentation fixes -- Model reference updates -- New guides and tutorials -- Documentation corrections and clarifications -- This gives documentation changes proper visibility - -**Management Endpoints / UI Grouping:** -- Group related features under sub-categories: - - **Proxy CLI Auth** - CLI authentication improvements - - **Virtual Keys** - Key rotation and management - - **Models + Endpoints** - Provider and endpoint management - -**AI Integrations Section Expansion:** -- Renamed from "Logging / Guardrail / Prompt Management Integrations" to "AI Integrations" -- Structure with four main subsections: - - **Logging** - with **General** subsection for miscellaneous logging features - - **Guardrails** - separate from logging features - - **Prompt Management** - BitBucket, GitHub integrations, versioning features - - **Secret Managers** - AWS, HashiCorp Vault, CyberArk, etc. - -**New Providers and Endpoints Section:** -- Add section after Key Highlights and before New Models / Updated Models -- Include tables for: - - **New Providers** - Provider name, supported endpoints, description - - **New LLM API Endpoints** (optional) - Endpoint, method, description, documentation link -- Only include major new provider integrations, not minor provider updates -- **IMPORTANT**: When adding new providers, also update `provider_endpoints_support.json` in the repository root (see Section 13) - -### 12. Section Header Counts - -**Always include counts in section headers for:** -- **New Providers** - Add count in parentheses: `### New Providers (X new providers)` -- **New LLM API Endpoints** - Add count in parentheses: `### New LLM API Endpoints (X new endpoints)` -- **New Model Support** - Add count in parentheses: `#### New Model Support (X new models)` - -**Format:** -```markdown -### New Providers (4 new providers) - -| Provider | Supported LiteLLM Endpoints | Description | -| -------- | --------------------------- | ----------- | -... - -### New LLM API Endpoints (2 new endpoints) - -| Endpoint | Method | Description | Documentation | -| -------- | ------ | ----------- | ------------- | -... - -#### New Model Support (32 new models) - -| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features | -| -------- | ----- | -------------- | ------------------- | -------------------- | -------- | -... -``` - -**Counting Rules:** -- Count each row in the table (excluding the header row) -- For models, count each model entry in the pricing table -- For providers, count each new provider added -- For endpoints, count each new API endpoint added - -### 13. Update provider_endpoints_support.json - -**When adding new providers or endpoints, you MUST also update `provider_endpoints_support.json` in the repository root.** - -This file tracks which endpoints are supported by each LiteLLM provider and is used to generate documentation. - -**Required Steps:** -1. For each new provider added to the release notes, add a corresponding entry to `provider_endpoints_support.json` -2. For each new endpoint type added, update the schema comment and add the endpoint to relevant providers - -**Provider Entry Format:** -```json -"provider_slug": { - "display_name": "Provider Name (`provider_slug`)", - "url": "https://docs.litellm.ai/docs/providers/provider_slug", - "endpoints": { - "chat_completions": true, - "messages": true, - "responses": true, - "embeddings": false, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, - "batches": false, - "rerank": false, - "a2a": true - } -} -``` - -**Available Endpoint Types:** -- `chat_completions` - `/chat/completions` endpoint -- `messages` - `/messages` endpoint (Anthropic format) -- `responses` - `/responses` endpoint (OpenAI/Anthropic unified) -- `embeddings` - `/embeddings` endpoint -- `image_generations` - `/image/generations` endpoint -- `audio_transcriptions` - `/audio/transcriptions` endpoint -- `audio_speech` - `/audio/speech` endpoint -- `moderations` - `/moderations` endpoint -- `batches` - `/batches` endpoint -- `rerank` - `/rerank` endpoint -- `ocr` - `/ocr` endpoint -- `search` - `/search` endpoint -- `vector_stores` - `/vector_stores` endpoint -- `a2a` - `/a2a/{agent}/message/send` endpoint (A2A Protocol) - -**Checklist:** -- [ ] All new providers from release notes are added to `provider_endpoints_support.json` -- [ ] Endpoint support flags accurately reflect provider capabilities -- [ ] Documentation URL points to correct provider docs page - -## Example Command Workflow - -```bash -# 1. Get model changes -git diff HEAD -- model_prices_and_context_window.json - -# 2. Analyze PR list for categorization -# 3. Create release notes following template -# 4. Link to appropriate documentation -# 5. Review for missing documentation needs -``` - -## Output Requirements - -- Follow exact markdown structure from reference -- Include all PR links and contributors -- Provide accurate model pricing tables -- Link to relevant documentation -- Highlight breaking changes with warnings -- Include deployment instructions -- End with full changelog link - -This process ensures consistent, comprehensive release notes that help users understand changes and upgrade smoothly. diff --git a/cookbook/misc/add_new_models.py b/cookbook/misc/add_new_models.py deleted file mode 100644 index 3cd0bfb2fcc..00000000000 --- a/cookbook/misc/add_new_models.py +++ /dev/null @@ -1,71 +0,0 @@ -import requests - - -def get_initial_config(): - proxy_base_url = input("Enter your proxy base URL (e.g., http://localhost:4000): ") - master_key = input("Enter your LITELLM_MASTER_KEY ") - return proxy_base_url, master_key - - -def get_user_input(): - model_name = input( - "Enter model_name (this is the 'model' passed in /chat/completions requests):" - ) - model = input("litellm_params: Enter model eg. 'azure/': ") - tpm = int(input("litellm_params: Enter tpm (tokens per minute): ")) - rpm = int(input("litellm_params: Enter rpm (requests per minute): ")) - api_key = input("litellm_params: Enter api_key: ") - api_base = input("litellm_params: Enter api_base: ") - api_version = input("litellm_params: Enter api_version: ") - timeout = int(input("litellm_params: Enter timeout (0 for default): ")) - stream_timeout = int( - input("litellm_params: Enter stream_timeout (0 for default): ") - ) - max_retries = int(input("litellm_params: Enter max_retries (0 for default): ")) - - return { - "model_name": model_name, - "litellm_params": { - "model": model, - "tpm": tpm, - "rpm": rpm, - "api_key": api_key, - "api_base": api_base, - "api_version": api_version, - "timeout": timeout, - "stream_timeout": stream_timeout, - "max_retries": max_retries, - }, - } - - -def make_request(proxy_base_url, master_key, data): - url = f"{proxy_base_url}/model/new" - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {master_key}", - } - - response = requests.post(url, headers=headers, json=data) - - print(f"Status Code: {response.status_code}") - print(f"Response from adding model: {response.text}") - - -def main(): - proxy_base_url, master_key = get_initial_config() - - while True: - print("Adding new Model to your proxy server...") - data = get_user_input() - make_request(proxy_base_url, master_key, data) - - add_another = input("Do you want to add another model? (yes/no): ").lower() - if add_another != "yes": - break - - print("Script finished.") - - -if __name__ == "__main__": - main() diff --git a/cookbook/misc/config.yaml b/cookbook/misc/config.yaml deleted file mode 100644 index 8ad1d764af5..00000000000 --- a/cookbook/misc/config.yaml +++ /dev/null @@ -1,72 +0,0 @@ -model_list: - - model_name: gpt-3.5-turbo - litellm_params: - model: azure/chatgpt-v-2 - api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ - api_version: "2023-05-15" - api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - - model_name: gpt-3.5-turbo-large - litellm_params: - model: "gpt-3.5-turbo-1106" - api_key: os.environ/OPENAI_API_KEY - rpm: 480 - timeout: 300 - stream_timeout: 60 - - model_name: gpt-4 - litellm_params: - model: azure/chatgpt-v-2 - api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ - api_version: "2023-05-15" - api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - rpm: 480 - timeout: 300 - stream_timeout: 60 - - model_name: sagemaker-completion-model - litellm_params: - model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4 - cost_per_second: 0.000420 - - model_name: text-embedding-ada-002 - litellm_params: - model: azure/azure-embedding-model - api_key: os.environ/AZURE_API_KEY - api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ - api_version: "2023-05-15" - model_info: - mode: embedding - base_model: text-embedding-ada-002 - - model_name: dall-e-2 - litellm_params: - model: azure/ - api_version: 2023-06-01-preview - api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ - api_key: os.environ/AZURE_API_KEY - - model_name: openai-dall-e-3 - litellm_params: - model: dall-e-3 - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - -litellm_settings: - drop_params: True - # max_budget: 100 - # budget_duration: 30d - num_retries: 5 - request_timeout: 600 - context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}] - -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY - store_model_in_db: True - proxy_budget_rescheduler_min_time: 60 - proxy_budget_rescheduler_max_time: 64 - proxy_batch_write_at: 1 - # database_url: "postgresql://:@:/" # [OPTIONAL] use for token-based auth to proxy - -# environment_variables: - # settings for using redis caching - # REDIS_HOST: redis-16337.c322.us-east-1-2.ec2.cloud.redislabs.com - # REDIS_PORT: "16337" - # REDIS_PASSWORD: diff --git a/cookbook/misc/dev_release.txt b/cookbook/misc/dev_release.txt deleted file mode 100644 index bd40f89e6f2..00000000000 --- a/cookbook/misc/dev_release.txt +++ /dev/null @@ -1,11 +0,0 @@ -python3 -m build -twine upload --verbose dist/litellm-1.18.13.dev4.tar.gz -u __token__ - - - -Note: You might need to make a MANIFEST.ini file on root for build process incase it fails - -Place this in MANIFEST.ini -recursive-exclude venv * -recursive-exclude myenv * -recursive-exclude py313_env * -recursive-exclude **/.venv * diff --git a/cookbook/misc/migrate_proxy_config.py b/cookbook/misc/migrate_proxy_config.py deleted file mode 100644 index b15f9663c64..00000000000 --- a/cookbook/misc/migrate_proxy_config.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -LiteLLM Migration Script! - -Takes a config.yaml and calls /model/new - -Inputs: - - File path to config.yaml - - Proxy base url to your hosted proxy - -Step 1: Reads your config.yaml -Step 2: reads `model_list` and loops through all models -Step 3: calls `/model/new` for each model -""" - -import os - -import yaml -import requests - -_in_memory_os_variables = {} - - -def migrate_models(config_file, proxy_base_url): - # Step 1: Read the config.yaml file - with open(config_file, "r") as f: - config = yaml.safe_load(f) - - # Step 2: Read the model_list and loop through all models - model_list = config.get("model_list", []) - print("model_list: ", model_list) - for model in model_list: - - model_name = model.get("model_name") - print("\nAdding model: ", model_name) - litellm_params = model.get("litellm_params", {}) - api_base = litellm_params.get("api_base", "") - print("api_base on config.yaml: ", api_base) - - litellm_model_name = litellm_params.get("model", "") or "" - if "vertex_ai/" in litellm_model_name: - print("\033[91m\nSkipping Vertex AI model\033[0m", model) - continue - - for param, value in litellm_params.items(): - if isinstance(value, str) and value.startswith("os.environ/"): - # check if value is in _in_memory_os_variables - if value in _in_memory_os_variables: - new_value = _in_memory_os_variables[value] - print( - "\033[92mAlready entered value for \033[0m", - value, - "\033[92musing \033[0m", - new_value, - ) - else: - new_value = input(f"Enter value for {value}: ") - _in_memory_os_variables[value] = new_value - litellm_params[param] = new_value - if "api_key" not in litellm_params: - new_value = input(f"Enter api key for {model_name}: ") - litellm_params["api_key"] = new_value - - print("\nlitellm_params: ", litellm_params) - # Confirm before sending POST request - confirm = input( - "\033[92mDo you want to send the POST request with the above parameters? (y/n): \033[0m" - ) - if confirm.lower() != "y": - print("Aborting POST request.") - exit() - - # Step 3: Call /model/new for each model - url = f"{proxy_base_url}/model/new" - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {master_key}", - } - data = {"model_name": model_name, "litellm_params": litellm_params} - print("POSTING data to proxy url", url) - response = requests.post(url, headers=headers, json=data) - if response.status_code != 200: - print(f"Error: {response.status_code} - {response.text}") - raise Exception(f"Error: {response.status_code} - {response.text}") - - # Print the response for each model - print( - f"Response for model '{model_name}': Status Code:{response.status_code} - {response.text}" - ) - - -# Usage -config_file = "config.yaml" -proxy_base_url = "http://0.0.0.0:4000" -master_key = os.environ["LITELLM_MASTER_KEY"] -print(f"config_file: {config_file}") -print(f"proxy_base_url: {proxy_base_url}") -migrate_models(config_file, proxy_base_url) diff --git a/cookbook/misc/openai_timeouts.py b/cookbook/misc/openai_timeouts.py deleted file mode 100644 index fe3e6d426d2..00000000000 --- a/cookbook/misc/openai_timeouts.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -from openai import OpenAI -from dotenv import load_dotenv -import concurrent.futures - -load_dotenv() - -client = OpenAI( - # This is the default and can be omitted - api_key=os.environ.get("OPENAI_API_KEY"), -) - - -def create_chat_completion(): - return client.chat.completions.create( - messages=[ - { - "role": "user", - "content": "Say this is a test. Respond in 20 lines", - } - ], - model="gpt-3.5-turbo", - ) - - -with concurrent.futures.ThreadPoolExecutor() as executor: - # Set a timeout of 10 seconds - future = executor.submit(create_chat_completion) - try: - chat_completion = future.result(timeout=0.00001) - print(chat_completion) - except concurrent.futures.TimeoutError: - print("Operation timed out.") diff --git a/cookbook/misc/sagmaker_streaming.py b/cookbook/misc/sagmaker_streaming.py deleted file mode 100644 index 1a6cc2e32ce..00000000000 --- a/cookbook/misc/sagmaker_streaming.py +++ /dev/null @@ -1,55 +0,0 @@ -# Notes - on how to do sagemaker streaming using boto3 -import json -import boto3 - -import sys -import os -from dotenv import load_dotenv - -load_dotenv() -import io - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path - - -class TokenIterator: - def __init__(self, stream): - self.byte_iterator = iter(stream) - self.buffer = io.BytesIO() - self.read_pos = 0 - - def __iter__(self): - return self - - def __next__(self): - while True: - self.buffer.seek(self.read_pos) - line = self.buffer.readline() - if line and line[-1] == ord("\n"): - self.read_pos += len(line) + 1 - full_line = line[:-1].decode("utf-8") - line_data = json.loads(full_line.lstrip("data:").rstrip("/n")) - return line_data["token"]["text"] - chunk = next(self.byte_iterator) - self.buffer.seek(0, io.SEEK_END) - self.buffer.write(chunk["PayloadPart"]["Bytes"]) - - -payload = { - "inputs": "How do I build a website?", - "parameters": {"max_new_tokens": 256}, - "stream": True, -} - - -client = boto3.client("sagemaker-runtime", region_name="us-west-2") -response = client.invoke_endpoint_with_response_stream( - EndpointName="berri-benchmarking-Llama-2-70b-chat-hf-4", - Body=json.dumps(payload), - ContentType="application/json", -) - -# for token in TokenIterator(response["Body"]): -# print(token) diff --git a/cookbook/misc/test_responses_api.py b/cookbook/misc/test_responses_api.py deleted file mode 100644 index 7abf3170fbc..00000000000 --- a/cookbook/misc/test_responses_api.py +++ /dev/null @@ -1,51 +0,0 @@ -import base64 -import os - -from openai import OpenAI -import time - -client = OpenAI(base_url="http://0.0.0.0:4001", api_key=os.environ["LITELLM_MASTER_KEY"]) - - -# Function to encode the image -def encode_image(image_path): - with open(image_path, "rb") as image_file: - return base64.b64encode(image_file.read()).decode("utf-8") - - -# Path to your image -image_path = "litellm/proxy/logo.png" - -# Getting the Base64 string -base64_image = encode_image(image_path) - - -response = client.responses.create( - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - input=[ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "what color is the image"}, - { - "type": "input_image", - "image_url": f"data:image/png;base64,{base64_image}", - }, - ], - } - ], -) - - -print(response.output_text) -print("response1 id===", response.id) -print("sleeping for 20 seconds...") -time.sleep(20) -print("making follow up request for existing id") -response2 = client.responses.create( - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", - previous_response_id=response.id, - input="ok, and what objects are in the image?", -) - -print(response2.output_text) diff --git a/cookbook/misc/update_json_caching.py b/cookbook/misc/update_json_caching.py deleted file mode 100644 index 8202d7033fd..00000000000 --- a/cookbook/misc/update_json_caching.py +++ /dev/null @@ -1,54 +0,0 @@ -import json - -# List of models to update -models_to_update = [ - "gpt-4o-mini", - "gpt-4o-mini-2024-07-18", - "gpt-4o", - "gpt-4o-2024-11-20", - "gpt-4o-2024-08-06", - "gpt-4o-2024-05-13", - "text-embedding-3-small", - "text-embedding-3-large", - "text-embedding-ada-002-v2", - "ft:gpt-4o-2024-08-06", - "ft:gpt-4o-mini-2024-07-18", - "ft:gpt-3.5-turbo", - "ft:davinci-002", - "ft:babbage-002", -] - - -def update_model_prices(file_path): - # Read the JSON file as text first to preserve number formatting - with open(file_path, "r") as file: - original_text = file.read() - data = json.loads(original_text) - - # Update specified models - for model_name in models_to_update: - print("finding model", model_name) - if model_name in data: - print("found model") - model = data[model_name] - if "input_cost_per_token" in model: - # Format new values to match original style - model["input_cost_per_token_batches"] = float( - "{:.12f}".format(model["input_cost_per_token"] / 2) - ) - if "output_cost_per_token" in model: - model["output_cost_per_token_batches"] = float( - "{:.12f}".format(model["output_cost_per_token"] / 2) - ) - print("new pricing for model=") - # Convert all float values to full decimal format before printing - formatted_model = { - k: "{:.9f}".format(v) if isinstance(v, float) else v - for k, v in data[model_name].items() - } - print(json.dumps(formatted_model, indent=4)) - - -# Run the update -file_path = "model_prices_and_context_window.json" -update_model_prices(file_path) diff --git a/cookbook/mlflow_langchain_tracing_litellm_proxy.ipynb b/cookbook/mlflow_langchain_tracing_litellm_proxy.ipynb deleted file mode 100644 index 818a25e4503..00000000000 --- a/cookbook/mlflow_langchain_tracing_litellm_proxy.ipynb +++ /dev/null @@ -1,315 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Databricks Notebook with MLFlow AutoLogging for LiteLLM Proxy calls\n", - "\n", - "Set `LITELLM_PROXY_API_KEY` in the notebook environment to a key accepted by the proxy before running the model cell\n" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "5e2812ed-8000-4793-b090-49a31464d810", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "%pip install -U -qqqq databricks-agents mlflow langchain==0.3.1 langchain-core==0.3.6 " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "52530b37-1860-4bba-a6c1-723de83bc58f", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "%pip install \"langchain-openai<=0.3.1\"" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "43c6f4b1-e2d5-431c-b1a2-b97df7707d59", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Before logging this chain using the driver notebook, you must comment out this line.\n", - "dbutils.library.restartPython() " - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "88eb8dd7-16b1-480b-aa70-cd429ef87159", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "import mlflow\n", - "from operator import itemgetter\n", - "from langchain_core.output_parsers import StrOutputParser\n", - "from langchain_core.prompts import PromptTemplate\n", - "from langchain_core.runnables import RunnableLambda\n", - "from langchain_databricks import ChatDatabricks\n", - "from langchain_openai import ChatOpenAI" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "f0fdca8f-6f6f-407c-ad4a-0d5a2778728e", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "mlflow.langchain.autolog()" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "2ef67315-e468-4d60-a318-98c2cac75bc4", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# These helper functions parse the `messages` array.\n", - "\n", - "# Return the string contents of the most recent message from the user\n", - "def extract_user_query_string(chat_messages_array):\n", - " return chat_messages_array[-1][\"content\"]\n", - "\n", - "\n", - "# Return the chat history, which is is everything before the last question\n", - "def extract_chat_history(chat_messages_array):\n", - " return chat_messages_array[:-1]" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "17708467-1976-48bd-94a0-8c7895cfae3b", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "model = ChatOpenAI(\n", - " openai_api_base=\"LITELLM_PROXY_BASE_URL\", # e.g.: http://0.0.0.0:4000\n", - " model = \"gpt-3.5-turbo\", # LITELLM 'model_name'\n", - " temperature=0.1, \n", - " api_key=os.environ[\"LITELLM_PROXY_API_KEY\"]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "a5f2c2af-82f7-470d-b559-47b67fb00cda", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "############\n", - "# Prompt Template for generation\n", - "############\n", - "prompt = PromptTemplate(\n", - " template=\"You are a hello world bot. Respond with a reply to the user's question that is fun and interesting to the user. User's question: {question}\",\n", - " input_variables=[\"question\"],\n", - ")\n", - "\n", - "############\n", - "# FM for generation\n", - "# ChatDatabricks accepts any /llm/v1/chat model serving endpoint\n", - "############\n", - "model = ChatDatabricks(\n", - " endpoint=\"databricks-dbrx-instruct\",\n", - " extra_params={\"temperature\": 0.01, \"max_tokens\": 500},\n", - ")\n", - "\n", - "\n", - "############\n", - "# Simple chain\n", - "############\n", - "# The framework requires the chain to return a string value.\n", - "chain = (\n", - " {\n", - " \"question\": itemgetter(\"messages\")\n", - " | RunnableLambda(extract_user_query_string),\n", - " \"chat_history\": itemgetter(\"messages\") | RunnableLambda(extract_chat_history),\n", - " }\n", - " | prompt\n", - " | model\n", - " | StrOutputParser()\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "366edd90-62a1-4d6f-8a65-0211fb24ca02", - "showTitle": false, - "title": "" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'Hello there! I\\'m here to help with your questions. Regarding your query about \"rag,\" it\\'s not something typically associated with a \"hello world\" bot, but I\\'m happy to explain!\\n\\nRAG, or Remote Angular GUI, is a tool that allows you to create and manage Angular applications remotely. It\\'s a way to develop and test Angular components and applications without needing to set up a local development environment. This can be particularly useful for teams working on distributed systems or for developers who prefer to work in a cloud-based environment.\\n\\nI hope this explanation of RAG has been helpful and interesting! If you have any other questions or need further clarification, feel free to ask.'" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "application/databricks.mlflow.trace": "\"tr-ea2226413395413ba2cf52cffc523502\"", - "text/plain": [ - "Trace(request_id=tr-ea2226413395413ba2cf52cffc523502)" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# This is the same input your chain's REST API will accept.\n", - "question = {\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"what is rag?\",\n", - " },\n", - " ]\n", - "}\n", - "\n", - "chain.invoke(question)" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "5d68e37d-0980-4a02-bf8d-885c3853f6c1", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "mlflow.models.set_model(model=model)" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "environmentMetadata": null, - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 4 - }, - "notebookName": "Untitled Notebook 2024-10-16 19:35:16", - "widgets": {} - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py b/cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py deleted file mode 100644 index 7bf9cc32484..00000000000 --- a/cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py +++ /dev/null @@ -1,540 +0,0 @@ -#!/usr/bin/env python3 -""" -Mock Bedrock Guardrail API Server - -This is a FastAPI server that mimics the AWS Bedrock Guardrail API for testing purposes. -It follows the same API spec as the real Bedrock guardrail endpoint. - -Usage: - python mock_bedrock_guardrail_server.py - -The server will start on http://localhost:8080 -""" - -import os -import re -from typing import Any, Dict, List, Literal, Optional - -from fastapi import Depends, FastAPI, Header, HTTPException, status -from fastapi.responses import JSONResponse -from pydantic import BaseModel, Field - -# ============================================================================ -# Request/Response Models (matching Bedrock API spec) -# ============================================================================ - - -class BedrockTextContent(BaseModel): - text: str - - -class BedrockContentItem(BaseModel): - text: BedrockTextContent - - -class BedrockRequest(BaseModel): - source: Literal["INPUT", "OUTPUT"] - content: List[BedrockContentItem] = Field(default_factory=list) - - -class BedrockGuardrailOutput(BaseModel): - text: Optional[str] = None - - -class TopicPolicyItem(BaseModel): - name: str - type: str - action: Literal["BLOCKED", "NONE"] - - -class TopicPolicy(BaseModel): - topics: List[TopicPolicyItem] = Field(default_factory=list) - - -class ContentFilterItem(BaseModel): - type: str - confidence: str - action: Literal["BLOCKED", "NONE"] - - -class ContentPolicy(BaseModel): - filters: List[ContentFilterItem] = Field(default_factory=list) - - -class CustomWord(BaseModel): - match: str - action: Literal["BLOCKED", "NONE"] - - -class WordPolicy(BaseModel): - customWords: List[CustomWord] = Field(default_factory=list) - managedWordLists: List[Dict[str, Any]] = Field(default_factory=list) - - -class PiiEntity(BaseModel): - type: str - match: str - action: Literal["BLOCKED", "ANONYMIZED", "NONE"] - - -class RegexMatch(BaseModel): - name: str - match: str - regex: str - action: Literal["BLOCKED", "ANONYMIZED", "NONE"] - - -class SensitiveInformationPolicy(BaseModel): - piiEntities: List[PiiEntity] = Field(default_factory=list) - regexes: List[RegexMatch] = Field(default_factory=list) - - -class ContextualGroundingFilter(BaseModel): - type: str - threshold: float - score: float - action: Literal["BLOCKED", "NONE"] - - -class ContextualGroundingPolicy(BaseModel): - filters: List[ContextualGroundingFilter] = Field(default_factory=list) - - -class Assessment(BaseModel): - topicPolicy: Optional[TopicPolicy] = None - contentPolicy: Optional[ContentPolicy] = None - wordPolicy: Optional[WordPolicy] = None - sensitiveInformationPolicy: Optional[SensitiveInformationPolicy] = None - contextualGroundingPolicy: Optional[ContextualGroundingPolicy] = None - - -class BedrockGuardrailResponse(BaseModel): - usage: Dict[str, int] = Field( - default_factory=lambda: {"topicPolicyUnits": 1, "contentPolicyUnits": 1} - ) - action: Literal["NONE", "GUARDRAIL_INTERVENED"] = "NONE" - outputs: List[BedrockGuardrailOutput] = Field(default_factory=list) - assessments: List[Assessment] = Field(default_factory=list) - - -# ============================================================================ -# Mock Guardrail Configuration -# ============================================================================ - - -class GuardrailConfig(BaseModel): - """Configuration for mock guardrail behavior""" - - blocked_words: List[str] = Field( - default_factory=lambda: ["offensive", "inappropriate", "badword"] - ) - blocked_topics: List[str] = Field(default_factory=lambda: ["violence", "illegal"]) - pii_patterns: Dict[str, str] = Field( - default_factory=lambda: { - "EMAIL": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", - "PHONE": r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b", - "SSN": r"\b\d{3}-\d{2}-\d{4}\b", - "CREDIT_CARD": r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b", - } - ) - anonymize_pii: bool = True # If True, ANONYMIZE PII; if False, BLOCK it - bearer_token: str = "mock-bedrock-token-12345" - - -# Global config -GUARDRAIL_CONFIG = GuardrailConfig() - -# ============================================================================ -# FastAPI App Setup -# ============================================================================ - -app = FastAPI( - title="Mock Bedrock Guardrail API", - description="Mock server mimicking AWS Bedrock Guardrail API", - version="1.0.0", -) - - -# ============================================================================ -# Authentication -# ============================================================================ - - -async def verify_bearer_token(authorization: Optional[str] = Header(None)) -> str: - """ - Verify the Bearer token from the Authorization header. - - Args: - authorization: The Authorization header value - - Returns: - The token if valid - - Raises: - HTTPException: If token is missing or invalid - """ - if authorization is None: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Missing Authorization header", - headers={"WWW-Authenticate": "Bearer"}, - ) - - # Check if it's a Bearer token - parts = authorization.split() - print(f"parts: {parts}") - if len(parts) != 2 or parts[0].lower() != "bearer": - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid Authorization header format. Expected: Bearer ", - headers={"WWW-Authenticate": "Bearer"}, - ) - - token = parts[1] - - # Verify token - if token != GUARDRAIL_CONFIG.bearer_token: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Invalid bearer token", - ) - - return token - - -# ============================================================================ -# Guardrail Logic -# ============================================================================ - - -def check_blocked_words(text: str) -> Optional[WordPolicy]: - """Check if text contains blocked words""" - found_words = [] - text_lower = text.lower() - - for word in GUARDRAIL_CONFIG.blocked_words: - if word.lower() in text_lower: - found_words.append(CustomWord(match=word, action="BLOCKED")) - - if found_words: - return WordPolicy(customWords=found_words) - return None - - -def check_blocked_topics(text: str) -> Optional[TopicPolicy]: - """Check if text contains blocked topics""" - found_topics = [] - text_lower = text.lower() - - for topic in GUARDRAIL_CONFIG.blocked_topics: - if topic.lower() in text_lower: - found_topics.append( - TopicPolicyItem(name=topic, type=topic.upper(), action="BLOCKED") - ) - - if found_topics: - return TopicPolicy(topics=found_topics) - return None - - -def check_pii(text: str) -> tuple[Optional[SensitiveInformationPolicy], str]: - """ - Check for PII in text and return policy + anonymized text - - Returns: - Tuple of (SensitiveInformationPolicy or None, anonymized_text) - """ - pii_entities = [] - anonymized_text = text - action = "ANONYMIZED" if GUARDRAIL_CONFIG.anonymize_pii else "BLOCKED" - - for pii_type, pattern in GUARDRAIL_CONFIG.pii_patterns.items(): - try: - # Compile the regex pattern with a timeout to prevent ReDoS attacks - compiled_pattern = re.compile(pattern) - matches = compiled_pattern.finditer(text) - for match in matches: - matched_text = match.group() - pii_entities.append( - PiiEntity(type=pii_type, match=matched_text, action=action) - ) - - # Anonymize the text if configured - if GUARDRAIL_CONFIG.anonymize_pii: - anonymized_text = anonymized_text.replace( - matched_text, f"[{pii_type}_REDACTED]" - ) - except re.error: - # Invalid regex pattern - skip it and log a warning - print(f"Warning: Invalid regex pattern for PII type {pii_type}: {pattern}") - continue - - if pii_entities: - return SensitiveInformationPolicy(piiEntities=pii_entities), anonymized_text - - return None, text - - -def process_guardrail_request( - request: BedrockRequest, -) -> tuple[BedrockGuardrailResponse, List[str]]: - """ - Process a guardrail request and return the response. - - Returns: - Tuple of (response, list of output texts) - """ - all_text_content = [] - output_texts = [] - - # Extract all text from content items - for content_item in request.content: - if content_item.text and content_item.text.text: - all_text_content.append(content_item.text.text) - - # Combine all text for analysis - combined_text = " ".join(all_text_content) - - # Initialize response - response = BedrockGuardrailResponse() - assessment = Assessment() - has_intervention = False - - # Check for blocked words - word_policy = check_blocked_words(combined_text) - if word_policy: - assessment.wordPolicy = word_policy - has_intervention = True - - # Check for blocked topics - topic_policy = check_blocked_topics(combined_text) - if topic_policy: - assessment.topicPolicy = topic_policy - has_intervention = True - - # Check for PII - for text in all_text_content: - pii_policy, anonymized_text = check_pii(text) - if pii_policy: - assessment.sensitiveInformationPolicy = pii_policy - if GUARDRAIL_CONFIG.anonymize_pii: - # If anonymizing, we don't block, we modify the text - output_texts.append(anonymized_text) - has_intervention = True - else: - # If not anonymizing PII, we block it - output_texts.append(text) - has_intervention = True - else: - output_texts.append(text) - - # Build response - if has_intervention: - response.action = "GUARDRAIL_INTERVENED" - # Only add assessment if there were interventions - response.assessments = [assessment] - - # Add outputs (modified or original text) - response.outputs = [BedrockGuardrailOutput(text=txt) for txt in output_texts] - - return response, output_texts - - -# ============================================================================ -# API Endpoints -# ============================================================================ - - -@app.get("/") -async def root(): - """Health check endpoint""" - return { - "service": "Mock Bedrock Guardrail API", - "status": "running", - "endpoint_format": "/guardrail/{guardrailIdentifier}/version/{guardrailVersion}/apply", - } - - -@app.get("/health") -async def health(): - """Health check endpoint""" - return {"status": "healthy"} - - -""" -LiteLLM exposes a basic guardrail API with the text extracted from the request and sent to the guardrail API, as well as the received request body for any further processing. - -This works across all LiteLLM endpoints (completion, anthropic /v1/messages, responses api, image generation, embedding, etc.) - -This makes it easy to support your own guardrail API without having to make a PR to LiteLLM. - -LiteLLM supports passing any provider specific params from LiteLLM config.yaml to the guardrail API. - -Example: - -```yaml -guardrails: - - guardrail_name: "bedrock-content-guard" - litellm_params: - guardrail: generic_guardrail_api - mode: "pre_call" - api_key: os.environ/GUARDRAIL_API_KEY - api_base: os.environ/GUARDRAIL_API_BASE - additional_provider_specific_params: - api_version: os.environ/GUARDRAIL_API_VERSION # additional provider specific params -``` - -This is a beta API. Please help us improve it. -""" - - -class LitellmBasicGuardrailRequest(BaseModel): - texts: List[str] - images: Optional[List[str]] = None - tools: Optional[List[dict]] = None - tool_calls: Optional[List[dict]] = None - request_data: Dict[str, Any] = Field(default_factory=dict) - additional_provider_specific_params: Dict[str, Any] = Field(default_factory=dict) - input_type: Literal["request", "response"] - litellm_call_id: Optional[str] = None - litellm_trace_id: Optional[str] = None - structured_messages: Optional[List[Dict[str, Any]]] = None - - -class LitellmBasicGuardrailResponse(BaseModel): - action: Literal[ - "BLOCKED", "NONE", "GUARDRAIL_INTERVENED" - ] # BLOCKED = litellm will raise an error, NONE = litellm will continue, GUARDRAIL_INTERVENED = litellm will continue, but the text was modified by the guardrail - blocked_reason: Optional[str] = None # only if action is BLOCKED, otherwise None - texts: Optional[List[str]] = None - images: Optional[List[str]] = None - - -@app.post( - "/beta/litellm_basic_guardrail_api", - response_model=LitellmBasicGuardrailResponse, -) -async def beta_litellm_basic_guardrail_api( - request: LitellmBasicGuardrailRequest, -) -> LitellmBasicGuardrailResponse: - """ - Apply guardrail to input or output content. - - This endpoint mimics the AWS Bedrock ApplyGuardrail API. - - Args: - request: The guardrail request containing content to analyze - token: Bearer token (verified by dependency) - - Returns: - LitellmBasicGuardrailResponse with analysis results - """ - print(f"request: {request}") - if any("ishaan" in text.lower() for text in request.texts): - return LitellmBasicGuardrailResponse( - action="BLOCKED", blocked_reason="Ishaan is not allowed" - ) - elif any("pii_value" in text for text in request.texts): - return LitellmBasicGuardrailResponse( - action="GUARDRAIL_INTERVENED", - texts=[ - text.replace("pii_value", "pii_value_redacted") - for text in request.texts - ], - ) - return LitellmBasicGuardrailResponse(action="NONE") - - -@app.post("/config/update") -async def update_config( - config: GuardrailConfig, token: str = Depends(verify_bearer_token) -): - """ - Update the guardrail configuration. - - This is a testing endpoint to modify the mock guardrail behavior. - - Args: - config: New guardrail configuration - token: Bearer token (verified by dependency) - - Returns: - Updated configuration - """ - global GUARDRAIL_CONFIG - GUARDRAIL_CONFIG = config - return {"status": "updated", "config": GUARDRAIL_CONFIG} - - -@app.get("/config") -async def get_config(token: str = Depends(verify_bearer_token)): - """ - Get the current guardrail configuration. - - Args: - token: Bearer token (verified by dependency) - - Returns: - Current configuration - """ - return GUARDRAIL_CONFIG - - -# ============================================================================ -# Error Handlers -# ============================================================================ - - -@app.exception_handler(HTTPException) -async def http_exception_handler(request, exc: HTTPException): - """Custom error handler for HTTP exceptions""" - return JSONResponse( - status_code=exc.status_code, - content={"error": exc.detail}, - headers=exc.headers, - ) - - -# ============================================================================ -# Main -# ============================================================================ - -if __name__ == "__main__": - import uvicorn - - # Get configuration from environment - host = os.getenv("MOCK_BEDROCK_HOST", "0.0.0.0") - port = int(os.getenv("MOCK_BEDROCK_PORT", "8080")) - bearer_token = os.getenv("MOCK_BEDROCK_TOKEN", "mock-bedrock-token-12345") - - # Update config with environment token - GUARDRAIL_CONFIG.bearer_token = bearer_token - - print("=" * 80) - print("Mock Bedrock Guardrail API Server") - print("=" * 80) - print(f"Server starting on: http://{host}:{port}") - print(f"Bearer Token: {bearer_token}") - print(f"Endpoint: POST /guardrail/{{id}}/version/{{version}}/apply") - print("=" * 80) - print("\nExample curl command:") - print( - f""" -curl -X POST "http://{host}:{port}/guardrail/test-guardrail/version/1/apply" \\ - -H "Authorization: Bearer {bearer_token}" \\ - -H "Content-Type: application/json" \\ - -d '{{ - "source": "INPUT", - "content": [ - {{ - "text": {{ - "text": "Hello, my email is test@example.com" - }} - }} - ] - }}' - """ - ) - print("=" * 80) - - uvicorn.run(app, host=host, port=port) diff --git a/cookbook/mock_prompt_management_server/README.md b/cookbook/mock_prompt_management_server/README.md deleted file mode 100644 index c13061cfe89..00000000000 --- a/cookbook/mock_prompt_management_server/README.md +++ /dev/null @@ -1,292 +0,0 @@ -# Mock Prompt Management Server - -A reference implementation of the [LiteLLM Generic Prompt Management API](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api). - -This FastAPI server demonstrates how to build a prompt management API that integrates with LiteLLM without requiring a PR to the LiteLLM repository. - -## Quick Start - -### 1. Install Dependencies - -```bash -pip install fastapi uvicorn pydantic -``` - -### 2. Start the Server - -```bash -python mock_prompt_management_server.py -``` - -The server will start on `http://localhost:8080` - -### 3. Test the Endpoint - -```bash -# Get a prompt -curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt" - -# Get a prompt with authentication -curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt" \ - -H "Authorization: Bearer test-token-12345" - -# List all prompts -curl "http://localhost:8080/prompts" - -# Get prompt variables -curl "http://localhost:8080/prompts/hello-world-prompt/variables" -``` - -## Using with LiteLLM - -### Configuration - -Create a `config.yaml` file: - -```yaml -model_list: - - model_name: gpt-3.5-turbo - litellm_params: - model: openai/gpt-3.5-turbo - api_key: os.environ/OPENAI_API_KEY - -prompts: - - prompt_id: "hello-world-prompt" - litellm_params: - prompt_integration: "generic_prompt_management" - api_base: http://localhost:8080 - api_key: test-token-12345 -``` - -### Start LiteLLM Proxy - -```bash -litellm --config config.yaml -``` - -### Make a Request - -```bash -curl http://0.0.0.0:4000/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $LITELLM_MASTER_KEY" \ - -d '{ - "model": "gpt-3.5-turbo", - "prompt_id": "hello-world-prompt", - "prompt_variables": { - "domain": "data science", - "task": "analyzing customer behavior" - }, - "messages": [ - {"role": "user", "content": "Please help me get started"} - ] - }' -``` - -## Available Prompts - -The server includes several example prompts: - -| Prompt ID | Description | Variables | -|-----------|-------------|-----------| -| `hello-world-prompt` | Basic helpful assistant | `domain`, `task` | -| `code-review-prompt` | Code review assistant | `years_experience`, `language`, `code` | -| `customer-support-prompt` | Customer support agent | `company_name`, `customer_message` | -| `data-analysis-prompt` | Data analysis expert | `analysis_type`, `dataset_name`, `data` | -| `creative-writing-prompt` | Creative writing assistant | `genre`, `length`, `topic` | - -## Authentication - -The server supports optional Bearer token authentication. Valid tokens for testing: - -- `test-token-12345` -- `dev-token-67890` -- `prod-token-abcdef` - -If no `Authorization` header is provided, requests are allowed (for testing purposes). - -## API Endpoints - -### LiteLLM Spec Endpoints - -#### `GET /beta/litellm_prompt_management` - -Get a prompt by ID (required by LiteLLM). - -**Query Parameters:** -- `prompt_id` (required): The prompt ID -- `project_name` (optional): Project filter -- `slug` (optional): Slug filter -- `version` (optional): Version filter - -**Response:** -```json -{ - "prompt_id": "hello-world-prompt", - "prompt_template": [ - { - "role": "system", - "content": "You are a helpful assistant specialized in {domain}." - }, - { - "role": "user", - "content": "Help me with: {task}" - } - ], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": { - "temperature": 0.7, - "max_tokens": 500 - } -} -``` - -### Convenience Endpoints (Not in LiteLLM Spec) - -#### `GET /health` - -Health check endpoint. - -#### `GET /prompts` - -List all available prompts. - -#### `GET /prompts/{prompt_id}/variables` - -Get all variables used in a prompt template. - -#### `POST /prompts` - -Create a new prompt (in-memory only, for testing). - -## Example: Full Integration Test - -### 1. Start the Mock Server - -```bash -python mock_prompt_management_server.py -``` - -### 2. Test with Python - -```python -from litellm import completion - -# The completion will: -# 1. Fetch the prompt from your API -# 2. Replace {domain} with "machine learning" -# 3. Replace {task} with "building a recommendation system" -# 4. Merge with your messages -# 5. Use the model and params from the prompt - -response = completion( - model="gpt-4", - prompt_id="hello-world-prompt", - prompt_variables={ - "domain": "machine learning", - "task": "building a recommendation system" - }, - messages=[ - {"role": "user", "content": "I have user behavior data from the past year."} - ], - # Configure the generic prompt manager - generic_prompt_config={ - "api_base": "http://localhost:8080", - "api_key": "test-token-12345", - } -) - -print(response.choices[0].message.content) -``` - -## Customization - -### Adding New Prompts - -Edit the `PROMPTS_DB` dictionary in `mock_prompt_management_server.py`: - -```python -PROMPTS_DB = { - "my-custom-prompt": { - "prompt_id": "my-custom-prompt", - "prompt_template": [ - { - "role": "system", - "content": "You are a {role}." - }, - { - "role": "user", - "content": "{user_input}" - } - ], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": { - "temperature": 0.8, - "max_tokens": 1000 - } - } -} -``` - -### Using a Database - -Replace the `PROMPTS_DB` dictionary with database queries: - -```python -@app.get("/beta/litellm_prompt_management") -async def get_prompt(prompt_id: str): - # Fetch from database - prompt = await db.prompts.find_one({"prompt_id": prompt_id}) - - if not prompt: - raise HTTPException(status_code=404, detail="Prompt not found") - - return PromptResponse(**prompt) -``` - -### Adding Access Control - -Use the custom query parameters for access control: - -```python -@app.get("/beta/litellm_prompt_management") -async def get_prompt( - prompt_id: str, - project_name: Optional[str] = None, - user_id: Optional[str] = None, - authorization: Optional[str] = Header(None) -): - token = verify_api_key(authorization) - - # Check if user has access to this project - if not has_project_access(token, project_name): - raise HTTPException(status_code=403, detail="Access denied") - - # Fetch and return prompt - ... -``` - -## Production Considerations - -Before deploying to production: - -1. **Use a real database** instead of in-memory storage -2. **Implement proper authentication** with JWT tokens or API keys -3. **Add rate limiting** to prevent abuse -4. **Use HTTPS** for encrypted communication -5. **Add logging and monitoring** for observability -6. **Implement caching** for frequently accessed prompts -7. **Add versioning** for prompt management -8. **Implement access control** based on teams/users -9. **Add input validation** for all parameters -10. **Use environment variables** for configuration - -## Related Documentation - -- [Generic Prompt Management API Documentation](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api) -- [LiteLLM Prompt Management](https://docs.litellm.ai/docs/proxy/prompt_management) -- [Generic Guardrail API](https://docs.litellm.ai/docs/adding_provider/generic_guardrail_api) - -## Questions? - -This is a reference implementation for the LiteLLM Generic Prompt Management API. For questions or issues, please open an issue on the [LiteLLM GitHub repository](https://github.com/BerriAI/litellm). diff --git a/cookbook/mock_prompt_management_server/mock_prompt_management_server.py b/cookbook/mock_prompt_management_server/mock_prompt_management_server.py deleted file mode 100644 index 295a96e12a9..00000000000 --- a/cookbook/mock_prompt_management_server/mock_prompt_management_server.py +++ /dev/null @@ -1,390 +0,0 @@ -#!/usr/bin/env python3 -""" -Mock Prompt Management API Server - -This is a FastAPI server that implements the LiteLLM Generic Prompt Management API -for testing and demonstration purposes. - -Usage: - python mock_prompt_management_server.py - -The server will start on http://localhost:8080 - -Test the endpoint: - curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt" -""" - -import os -import json -from typing import Any, Dict, List, Optional - -from fastapi import FastAPI, HTTPException, Header, Query, status -from fastapi.responses import JSONResponse -from pydantic import BaseModel, Field - -# ============================================================================ -# Response Models -# ============================================================================ - - -class MessageContent(BaseModel): - """A single message in the prompt template""" - - role: str = Field(..., description="Message role (system, user, assistant)") - content: str = Field( - ..., description="Message content with optional {variable} placeholders" - ) - - -class PromptResponse(BaseModel): - """Response format for the prompt management API""" - - prompt_id: str = Field(..., description="The ID of the prompt") - prompt_template: List[MessageContent] = Field( - ..., description="Array of messages in OpenAI format" - ) - prompt_template_model: Optional[str] = Field( - None, description="Optional model to use for this prompt" - ) - prompt_template_optional_params: Optional[Dict[str, Any]] = Field( - None, description="Optional parameters like temperature, max_tokens, etc." - ) - - -# ============================================================================ -# Mock Prompt Database -# ============================================================================ - -PROMPTS_DB = { - "hello-world-prompt": { - "prompt_id": "hello-world-prompt", - "prompt_template": [ - { - "role": "system", - "content": "You are a helpful assistant specialized in {domain}.", - }, - {"role": "user", "content": "Help me with: {task}"}, - ], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": {"temperature": 0.7, "max_tokens": 500}, - }, - "code-review-prompt": { - "prompt_id": "code-review-prompt", - "prompt_template": [ - { - "role": "system", - "content": "You are an expert code reviewer with {years_experience} years of experience in {language}.", - }, - { - "role": "user", - "content": "Please review the following code for bugs, security issues, and best practices:\n\n{code}", - }, - ], - "prompt_template_model": "gpt-4-turbo", - "prompt_template_optional_params": { - "temperature": 0.3, - "max_tokens": 1500, - }, - }, - "customer-support-prompt": { - "prompt_id": "customer-support-prompt", - "prompt_template": [ - { - "role": "system", - "content": "You are a friendly customer support agent for {company_name}. Always be professional, empathetic, and solution-oriented.", - }, - { - "role": "user", - "content": "Customer inquiry: {customer_message}", - }, - ], - "prompt_template_model": "gpt-3.5-turbo", - "prompt_template_optional_params": { - "temperature": 0.8, - "max_tokens": 800, - "top_p": 0.9, - }, - }, - "data-analysis-prompt": { - "prompt_id": "data-analysis-prompt", - "prompt_template": [ - { - "role": "system", - "content": "You are a data scientist expert in {analysis_type} analysis.", - }, - { - "role": "user", - "content": "Analyze the following data and provide insights:\n\nDataset: {dataset_name}\nData: {data}", - }, - ], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": { - "temperature": 0.5, - "max_tokens": 2000, - }, - }, - "creative-writing-prompt": { - "prompt_id": "creative-writing-prompt", - "prompt_template": [ - { - "role": "system", - "content": "You are a creative writer specializing in {genre} fiction.", - }, - { - "role": "user", - "content": "Write a {length} story about: {topic}", - }, - ], - "prompt_template_model": "gpt-4", - "prompt_template_optional_params": { - "temperature": 0.9, - "max_tokens": 3000, - "top_p": 0.95, - }, - }, -} - -# Valid API tokens for authentication (in production, use a secure token store) -VALID_API_TOKENS = { - "test-token-12345", - "dev-token-67890", - "prod-token-abcdef", -} - -# ============================================================================ -# FastAPI App -# ============================================================================ - -app = FastAPI( - title="Mock Prompt Management API", - description="A mock server implementing the LiteLLM Generic Prompt Management API", - version="1.0.0", -) - - -def verify_api_key(authorization: Optional[str] = Header(None)) -> bool: - """ - Verify the API key from the Authorization header. - - Args: - authorization: Authorization header (Bearer token) - - Returns: - True if valid, raises HTTPException if invalid - """ - if authorization is None: - # Allow requests without authentication for testing - return True - - # Extract token from "Bearer " - if not authorization.startswith("Bearer "): - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid authorization header format. Expected 'Bearer '", - ) - - token = authorization.replace("Bearer ", "").strip() - - if token not in VALID_API_TOKENS: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid API key", - ) - - return True - - -@app.get("/beta/litellm_prompt_management", response_model=PromptResponse) -async def get_prompt( - prompt_id: str = Query(..., description="The ID of the prompt to fetch"), - project_name: Optional[str] = Query( - None, description="Optional project name filter" - ), - slug: Optional[str] = Query(None, description="Optional slug filter"), - version: Optional[str] = Query(None, description="Optional version filter"), - authorization: Optional[str] = Header(None), -) -> PromptResponse: - """ - Get a prompt by ID with optional filtering. - - This endpoint implements the LiteLLM Generic Prompt Management API specification. - - Args: - prompt_id: The ID of the prompt to fetch - project_name: Optional project name for filtering - slug: Optional slug for filtering - version: Optional version for filtering - authorization: Optional Bearer token for authentication - - Returns: - PromptResponse with the prompt template and configuration - - Raises: - HTTPException: 401 if authentication fails, 404 if prompt not found - """ - # Verify authentication - verify_api_key(authorization) - - # Log the request parameters (useful for debugging) - print(f"Fetching prompt: {prompt_id}") - if project_name: - print(f" Project: {project_name}") - if slug: - print(f" Slug: {slug}") - if version: - print(f" Version: {version}") - - # Check if prompt exists - if prompt_id not in PROMPTS_DB: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Prompt '{prompt_id}' not found. Available prompts: {list(PROMPTS_DB.keys())}", - ) - - # Get the prompt from the database - prompt_data = PROMPTS_DB[prompt_id] - - # Optional: Apply filtering based on project_name, slug, or version - # In a real implementation, you might use these to filter prompts by access control - # or to fetch specific versions from your database - - return PromptResponse(**prompt_data) - - -@app.get("/health") -async def health_check(): - """Health check endpoint""" - return { - "status": "healthy", - "service": "mock-prompt-management-api", - "version": "1.0.0", - } - - -@app.get("/prompts") -async def list_prompts(authorization: Optional[str] = Header(None)): - """ - List all available prompts. - - This is a convenience endpoint (not part of the LiteLLM spec) for - discovering available prompts. - """ - # Verify authentication - verify_api_key(authorization) - - prompts_list = [ - { - "prompt_id": pid, - "model": p.get("prompt_template_model"), - "has_variables": any( - "{" in msg.get("content", "") for msg in p.get("prompt_template", []) - ), - } - for pid, p in PROMPTS_DB.items() - ] - - return {"prompts": prompts_list, "total": len(prompts_list)} - - -@app.get("/prompts/{prompt_id}/variables") -async def get_prompt_variables( - prompt_id: str, authorization: Optional[str] = Header(None) -): - """ - Get all variables in a prompt template. - - This is a convenience endpoint (not part of the LiteLLM spec) for - discovering what variables a prompt expects. - """ - # Verify authentication - verify_api_key(authorization) - - if prompt_id not in PROMPTS_DB: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Prompt '{prompt_id}' not found", - ) - - prompt_data = PROMPTS_DB[prompt_id] - variables = set() - - # Extract variables from the prompt template - import re - - for message in prompt_data["prompt_template"]: - content = message.get("content", "") - # Find all {variable} patterns - found_vars = re.findall(r"\{(\w+)\}", content) - variables.update(found_vars) - - return { - "prompt_id": prompt_id, - "variables": sorted(list(variables)), - "example_usage": { - "prompt_id": prompt_id, - "prompt_variables": {var: f"<{var}_value>" for var in variables}, - }, - } - - -@app.post("/prompts") -async def create_prompt( - prompt: PromptResponse, authorization: Optional[str] = Header(None) -): - """ - Create a new prompt (convenience endpoint for testing). - - This is NOT part of the LiteLLM spec - it's just for testing purposes. - """ - # Verify authentication - verify_api_key(authorization) - - if prompt.prompt_id in PROMPTS_DB: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=f"Prompt '{prompt.prompt_id}' already exists", - ) - - PROMPTS_DB[prompt.prompt_id] = prompt.dict() - - return { - "status": "created", - "prompt_id": prompt.prompt_id, - "message": "Prompt created successfully (in-memory only)", - } - - -# ============================================================================ -# Main -# ============================================================================ - -if __name__ == "__main__": - import uvicorn - - print("=" * 70) - print("Mock Prompt Management API Server") - print("=" * 70) - print(f"\nStarting server on http://localhost:8080") - print(f"\nAvailable prompts: {len(PROMPTS_DB)}") - for prompt_id in PROMPTS_DB.keys(): - print(f" - {prompt_id}") - print(f"\nValid API tokens: {len(VALID_API_TOKENS)}") - print(" - test-token-12345") - print(" - dev-token-67890") - print(" - prod-token-abcdef") - print("\nEndpoints:") - print(" GET /beta/litellm_prompt_management?prompt_id= (LiteLLM spec)") - print(" GET /health (health check)") - print(" GET /prompts (list all prompts)") - print( - " GET /prompts/{id}/variables (get prompt variables)" - ) - print(" POST /prompts (create prompt)") - print("\nExample usage:") - print( - ' curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"' - ) - print("\nPress CTRL+C to stop the server") - print("=" * 70) - - uvicorn.run(app, host="0.0.0.0", port=8080, log_level="info") diff --git a/cookbook/nova_sonic_realtime.py b/cookbook/nova_sonic_realtime.py deleted file mode 100644 index 50844121cbf..00000000000 --- a/cookbook/nova_sonic_realtime.py +++ /dev/null @@ -1,292 +0,0 @@ -""" -Client script to test Nova Sonic realtime API through LiteLLM proxy. - -This script connects to LiteLLM proxy's realtime endpoint and enables -speech-to-speech conversation with Bedrock Nova Sonic. - -Prerequisites: -- LiteLLM proxy running with Bedrock configured -- pyaudio installed: pip install pyaudio -- websockets installed: pip install websockets - -Usage: - export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 32)" - python nova_sonic_realtime.py -""" - -import asyncio -import base64 -import json -import os -import pyaudio -import websockets -from typing import Optional - -# Bounded queue size for audio chunks (configurable via env to avoid unbounded memory) -AUDIO_QUEUE_MAXSIZE = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 10_000)) - -# Audio configuration (matching Nova Sonic requirements) -INPUT_SAMPLE_RATE = 16000 # Nova Sonic expects 16kHz input -OUTPUT_SAMPLE_RATE = 24000 # Nova Sonic outputs 24kHz -CHANNELS = 1 -FORMAT = pyaudio.paInt16 -CHUNK_SIZE = 1024 - -# LiteLLM proxy configuration -LITELLM_PROXY_URL = "ws://localhost:4000/v1/realtime?model=bedrock-sonic" -LITELLM_API_KEY = os.environ["LITELLM_MASTER_KEY"] - - -class RealtimeClient: - """Client for LiteLLM realtime API with audio support.""" - - def __init__(self, url: str, api_key: str): - self.url = url - self.api_key = api_key - self.ws: Optional[websockets.WebSocketClientProtocol] = None - self.is_active = False - self.audio_queue = asyncio.Queue(maxsize=AUDIO_QUEUE_MAXSIZE) - self.pyaudio = pyaudio.PyAudio() - self.input_stream = None - self.output_stream = None - - async def connect(self): - """Connect to LiteLLM proxy realtime endpoint.""" - print(f"Connecting to {self.url}...") - - headers = {} - if self.api_key: - headers["Authorization"] = f"Bearer {self.api_key}" - - self.ws = await websockets.connect( - self.url, - additional_headers=headers, - max_size=10 * 1024 * 1024, # 10MB max message size - ) - self.is_active = True - print("✓ Connected to LiteLLM proxy") - - async def send_session_update(self): - """Send session configuration.""" - session_update = { - "type": "session.update", - "session": { - "instructions": "You are a friendly assistant. Keep your responses short and conversational.", - "voice": "matthew", - "temperature": 0.8, - "max_response_output_tokens": 1024, - "modalities": ["text", "audio"], - "input_audio_format": "pcm16", - "output_audio_format": "pcm16", - "turn_detection": { - "type": "server_vad", - "threshold": 0.5, - "prefix_padding_ms": 300, - "silence_duration_ms": 500, - }, - }, - } - await self.ws.send(json.dumps(session_update)) - print("✓ Session configuration sent") - - async def receive_messages(self): - """Receive and process messages from the server.""" - try: - async for message in self.ws: - if not self.is_active: - break - - try: - data = json.loads(message) - event_type = data.get("type") - - if event_type == "session.created": - print(f"✓ Session created: {data.get('session', {}).get('id')}") - - elif event_type == "response.created": - print("🤖 Assistant is responding...") - - elif event_type == "response.text.delta": - # Print text transcription - delta = data.get("delta", "") - print(delta, end="", flush=True) - - elif event_type == "response.audio.delta": - # Queue audio for playback - audio_b64 = data.get("delta", "") - if audio_b64: - audio_bytes = base64.b64decode(audio_b64) - await self.audio_queue.put(audio_bytes) - - elif event_type == "response.text.done": - print() # New line after text - - elif event_type == "response.done": - print("✓ Response complete") - - elif event_type == "error": - print(f"❌ Error: {data.get('error', {})}") - - else: - # Debug: print other event types - print(f"[{event_type}]", end=" ") - - except json.JSONDecodeError: - print(f"Failed to parse message: {message[:100]}") - - except websockets.exceptions.ConnectionClosed: - print("\n✗ Connection closed") - except Exception as e: - print(f"\n✗ Error receiving messages: {e}") - finally: - self.is_active = False - - async def send_audio_chunk(self, audio_bytes: bytes): - """Send audio chunk to server.""" - if not self.is_active or not self.ws: - return - - audio_b64 = base64.b64encode(audio_bytes).decode("utf-8") - message = { - "type": "input_audio_buffer.append", - "audio": audio_b64, - } - await self.ws.send(json.dumps(message)) - - async def commit_audio_buffer(self): - """Commit the audio buffer to trigger processing.""" - if not self.is_active or not self.ws: - return - - message = {"type": "input_audio_buffer.commit"} - await self.ws.send(json.dumps(message)) - - async def capture_audio(self): - """Capture audio from microphone and send to server.""" - print("\n🎤 Starting audio capture...") - print("Speak into your microphone. Press Ctrl+C to stop.\n") - - self.input_stream = self.pyaudio.open( - format=FORMAT, - channels=CHANNELS, - rate=INPUT_SAMPLE_RATE, - input=True, - frames_per_buffer=CHUNK_SIZE, - ) - - try: - while self.is_active: - audio_data = self.input_stream.read( - CHUNK_SIZE, exception_on_overflow=False - ) - await self.send_audio_chunk(audio_data) - await asyncio.sleep(0.01) # Small delay to prevent overwhelming - except Exception as e: - print(f"Error capturing audio: {e}") - finally: - if self.input_stream: - self.input_stream.stop_stream() - self.input_stream.close() - - async def play_audio(self): - """Play audio responses from the server.""" - print("🔊 Starting audio playback...") - - self.output_stream = self.pyaudio.open( - format=FORMAT, - channels=CHANNELS, - rate=OUTPUT_SAMPLE_RATE, - output=True, - frames_per_buffer=CHUNK_SIZE, - ) - - try: - while self.is_active: - try: - audio_data = await asyncio.wait_for( - self.audio_queue.get(), timeout=0.1 - ) - if audio_data: - self.output_stream.write(audio_data) - except asyncio.TimeoutError: - continue - except Exception as e: - print(f"Error playing audio: {e}") - finally: - if self.output_stream: - self.output_stream.stop_stream() - self.output_stream.close() - - async def close(self): - """Close the connection and cleanup.""" - self.is_active = False - - if self.ws: - await self.ws.close() - - if self.input_stream: - self.input_stream.stop_stream() - self.input_stream.close() - - if self.output_stream: - self.output_stream.stop_stream() - self.output_stream.close() - - self.pyaudio.terminate() - print("\n✓ Connection closed") - - -async def main(): - """Main function to run the realtime client.""" - print("=" * 80) - print("Bedrock Nova Sonic Realtime Client") - print("=" * 80) - print() - - client = RealtimeClient(LITELLM_PROXY_URL, LITELLM_API_KEY) - - try: - # Connect to server - await client.connect() - - # Send session configuration - await client.send_session_update() - - # Wait a moment for session to be established - await asyncio.sleep(0.5) - - # Start tasks - receive_task = asyncio.create_task(client.receive_messages()) - capture_task = asyncio.create_task(client.capture_audio()) - playback_task = asyncio.create_task(client.play_audio()) - - # Wait for user to interrupt - await asyncio.gather( - receive_task, - capture_task, - playback_task, - return_exceptions=True, - ) - - except KeyboardInterrupt: - print("\n\n⚠ Interrupted by user") - except Exception as e: - print(f"\n❌ Error: {e}") - import traceback - - traceback.print_exc() - finally: - await client.close() - - -if __name__ == "__main__": - print("\nMake sure:") - print("1. LiteLLM proxy is running on port 4000") - print("2. Bedrock is configured in proxy_server_config.yaml") - print("3. AWS credentials are set") - print() - - try: - asyncio.run(main()) - except KeyboardInterrupt: - print("\n\nGoodbye!") diff --git a/cookbook/result.html b/cookbook/result.html deleted file mode 100644 index 0bd099bacc7..00000000000 --- a/cookbook/result.html +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - -
- - - - - - - - \ No newline at end of file diff --git a/cookbook/veo_video_generation.py b/cookbook/veo_video_generation.py deleted file mode 100644 index 8587032e0be..00000000000 --- a/cookbook/veo_video_generation.py +++ /dev/null @@ -1,321 +0,0 @@ -#!/usr/bin/env python3 -""" -Complete example for Veo video generation through LiteLLM proxy. - -This script demonstrates how to: -1. Generate videos using Google's Veo model -2. Poll for completion status -3. Download the generated video file - -Requirements: -- LiteLLM proxy running with Google AI Studio pass-through configured -- Google AI Studio API key with Veo access -""" - -import json -import os -import time -import requests -from typing import Optional - - -class VeoVideoGenerator: - """Complete Veo video generation client using LiteLLM proxy.""" - - def __init__( - self, - api_key: str, - base_url: str = "http://localhost:4000/gemini/v1beta", - ): - """ - Initialize the Veo video generator. - - Args: - base_url: Base URL for the LiteLLM proxy with Gemini pass-through - api_key: API key for LiteLLM proxy authentication - """ - self.base_url = base_url - self.api_key = api_key - self.headers = {"x-goog-api-key": api_key, "Content-Type": "application/json"} - - def generate_video(self, prompt: str) -> Optional[str]: - """ - Initiate video generation with Veo. - - Args: - prompt: Text description of the video to generate - - Returns: - Operation name if successful, None otherwise - """ - print(f"🎬 Generating video with prompt: '{prompt}'") - - url = f"{self.base_url}/models/veo-3.0-generate-preview:predictLongRunning" - payload = {"instances": [{"prompt": prompt}]} - - try: - response = requests.post(url, headers=self.headers, json=payload) - response.raise_for_status() - - data = response.json() - operation_name = data.get("name") - - if operation_name: - print(f"✅ Video generation started: {operation_name}") - return operation_name - else: - print("❌ No operation name returned") - print(f"Response: {json.dumps(data, indent=2)}") - return None - - except requests.RequestException as e: - print(f"❌ Failed to start video generation: {e}") - if hasattr(e, "response") and e.response is not None: - try: - error_data = e.response.json() - print(f"Error details: {json.dumps(error_data, indent=2)}") - except: - print(f"Error response: {e.response.text}") - return None - - def wait_for_completion( - self, operation_name: str, max_wait_time: int = 600 - ) -> Optional[str]: - """ - Poll operation status until video generation is complete. - - Args: - operation_name: Name of the operation to monitor - max_wait_time: Maximum time to wait in seconds (default: 10 minutes) - - Returns: - Video URI if successful, None otherwise - """ - print("⏳ Waiting for video generation to complete...") - - operation_url = f"{self.base_url}/{operation_name}" - start_time = time.time() - poll_interval = 10 # Start with 10 seconds - - while time.time() - start_time < max_wait_time: - try: - print( - f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)" - ) - - response = requests.get(operation_url, headers=self.headers) - response.raise_for_status() - - data = response.json() - - # Check for errors - if "error" in data: - print("❌ Error in video generation:") - print(json.dumps(data["error"], indent=2)) - return None - - # Check if operation is complete - is_done = data.get("done", False) - - if is_done: - print("🎉 Video generation complete!") - - try: - # Extract video URI from nested response - video_uri = data["response"]["generateVideoResponse"][ - "generatedSamples" - ][0]["video"]["uri"] - print(f"📹 Video URI: {video_uri}") - return video_uri - except KeyError as e: - print(f"❌ Could not extract video URI: {e}") - print("Full response:") - print(json.dumps(data, indent=2)) - return None - - # Wait before next poll, with exponential backoff - time.sleep(poll_interval) - poll_interval = min(poll_interval * 1.2, 30) # Cap at 30 seconds - - except requests.RequestException as e: - print(f"❌ Error polling operation status: {e}") - time.sleep(poll_interval) - - print(f"⏰ Timeout after {max_wait_time} seconds") - return None - - def download_video( - self, video_uri: str, output_filename: str = "generated_video.mp4" - ) -> bool: - """ - Download the generated video file. - - Args: - video_uri: URI of the video to download (from Google's response) - output_filename: Local filename to save the video - - Returns: - True if download successful, False otherwise - """ - print(f"⬇️ Downloading video...") - print(f"Original URI: {video_uri}") - - # Convert Google URI to LiteLLM proxy URI - # Example: files/abc123 -> /gemini/v1beta/files/abc123:download?alt=media - if video_uri.startswith("files/"): - download_path = f"{video_uri}:download?alt=media" - else: - download_path = video_uri - - litellm_download_url = f"{self.base_url}/{download_path}" - print(f"Download URL: {litellm_download_url}") - - try: - # Download with streaming and redirect handling - response = requests.get( - litellm_download_url, - headers=self.headers, - stream=True, - allow_redirects=True, # Handle redirects automatically - ) - response.raise_for_status() - - # Save video file - with open(output_filename, "wb") as f: - downloaded_size = 0 - for chunk in response.iter_content(chunk_size=8192): - if chunk: - f.write(chunk) - downloaded_size += len(chunk) - - # Progress indicator for large files - if downloaded_size % (1024 * 1024) == 0: # Every MB - print( - f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB..." - ) - - # Verify file was created and has content - if os.path.exists(output_filename): - file_size = os.path.getsize(output_filename) - if file_size > 0: - print(f"✅ Video downloaded successfully!") - print(f"📁 Saved as: {output_filename}") - print(f"📏 File size: {file_size / (1024*1024):.2f} MB") - return True - else: - print("❌ Downloaded file is empty") - os.remove(output_filename) - return False - else: - print("❌ File was not created") - return False - - except requests.RequestException as e: - print(f"❌ Download failed: {e}") - if hasattr(e, "response") and e.response is not None: - print(f"Status code: {e.response.status_code}") - print(f"Response headers: {dict(e.response.headers)}") - return False - - def generate_and_download(self, prompt: str, output_filename: str = None) -> bool: - """ - Complete workflow: generate video and download it. - - Args: - prompt: Text description for video generation - output_filename: Output filename (auto-generated if None) - - Returns: - True if successful, False otherwise - """ - # Auto-generate filename if not provided - if output_filename is None: - timestamp = int(time.time()) - safe_prompt = "".join( - c for c in prompt[:30] if c.isalnum() or c in (" ", "-", "_") - ).rstrip() - output_filename = ( - f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4" - ) - - print("=" * 60) - print("🎬 VEO VIDEO GENERATION WORKFLOW") - print("=" * 60) - - # Step 1: Generate video - operation_name = self.generate_video(prompt) - if not operation_name: - return False - - # Step 2: Wait for completion - video_uri = self.wait_for_completion(operation_name) - if not video_uri: - return False - - # Step 3: Download video - success = self.download_video(video_uri, output_filename) - - if success: - print("=" * 60) - print("🎉 SUCCESS! Video generation complete!") - print(f"📁 Video saved as: {output_filename}") - print("=" * 60) - else: - print("=" * 60) - print("❌ FAILED! Video generation or download failed") - print("=" * 60) - - return success - - -def main(): - """ - Example usage of the VeoVideoGenerator. - - Configure these environment variables: - - LITELLM_BASE_URL: Your LiteLLM proxy URL (default: http://localhost:4000/gemini/v1beta) - - LITELLM_API_KEY: API key for LiteLLM proxy authentication - """ - - # Configuration from environment or defaults - base_url = os.getenv("LITELLM_BASE_URL", "http://localhost:4000/gemini/v1beta") - api_key = os.environ["LITELLM_API_KEY"] - - print("🚀 Starting Veo Video Generation Example") - print(f"📡 Using LiteLLM proxy at: {base_url}") - - # Initialize generator - generator = VeoVideoGenerator(base_url=base_url, api_key=api_key) - - # Example prompts - try different ones! - example_prompts = [ - "A cat playing with a ball of yarn in a sunny garden", - "Ocean waves crashing against rocky cliffs at sunset", - "A bustling city street with people walking and cars passing by", - "A peaceful forest with sunlight filtering through the trees", - ] - - # Use first example or get from user - prompt = example_prompts[0] - print(f"🎬 Using prompt: '{prompt}'") - - # Generate and download video - success = generator.generate_and_download(prompt) - - if success: - print("\n✅ Example completed successfully!") - print("💡 Try modifying the prompt in the script for different videos!") - else: - print("\n❌ Example failed!") - print("🔧 Check your LiteLLM proxy configuration and Google AI Studio API key") - - # Troubleshooting tips - print("\n🔍 Troubleshooting:") - print("1. Ensure LiteLLM proxy is running with Google AI Studio pass-through") - print("2. Verify your Google AI Studio API key has Veo access") - print("3. Check that your prompt meets Veo's content guidelines") - print("4. Review the LiteLLM proxy logs for detailed error information") - - -if __name__ == "__main__": - main() diff --git a/docker-compose.hardened.yml b/docker-compose.hardened.yml deleted file mode 100644 index 84a23faa054..00000000000 --- a/docker-compose.hardened.yml +++ /dev/null @@ -1,44 +0,0 @@ -services: - # Hardened stack: for testing the proxy under non-root, read-only, proxy-enforced constraints. - # Keep this file focused on hardening/QA scenarios; leave the main docker-compose.yml for default dev usage. - litellm: - build: - context: . - dockerfile: docker/Dockerfile.non_root - target: runtime - depends_on: - - squid - user: "101:101" - group_add: - - "2345" - read_only: true - cap_drop: - - ALL - security_opt: - - no-new-privileges:true - tmpfs: - - /app/cache:rw,noexec,nosuid,nodev,size=128m,uid=101,gid=101,mode=1777 - - /app/migrations:rw,noexec,nosuid,nodev,size=64m,uid=101,gid=101,mode=1777 - volumes: - - ./proxy_server_config.yaml:/app/config.yaml:ro - environment: - LITELLM_NON_ROOT: "true" - PRISMA_BINARY_CACHE_DIR: "/app/cache/prisma-python/binaries" - XDG_CACHE_HOME: "/app/cache" - LITELLM_MIGRATION_DIR: "/app/migrations" - HTTP_PROXY: "http://squid:3128" - HTTPS_PROXY: "http://squid:3128" - NO_PROXY: "localhost,127.0.0.1,db" - command: - - "--port" - - "4000" - - "--config" - - "/app/config.yaml" - squid: - image: sameersbn/squid:3.5.27-2 - restart: unless-stopped - ports: - - "3128:3128" - tmpfs: - - /var/spool/squid:rw,noexec,nosuid,nodev,size=64m - - /var/log/squid:rw,noexec,nosuid,nodev,size=16m diff --git a/docker-compose.yml b/docker-compose.yml index 80e1f289aad..b012c3763de 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,52 +1,71 @@ +# LiteLLM plus a Postgres database that stores models, virtual keys and spend +# logs. Used by https://docs.litellm.ai/docs/proxy/docker_quick_start +# +# curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker-compose.yml +# printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env +# docker compose up -d +# +# `docker compose up` runs the published image. From a checkout of the repo, +# `docker compose up --build` builds the Dockerfile next to this file instead. +# `docker compose --profile monitoring up -d` also starts Prometheus on 9090. +# +# Compose reads .env from this directory. Keep it: regenerating LITELLM_SALT_KEY +# makes credentials already stored in the database unreadable. A database +# created without LITELLM_SALT_KEY encrypted them with the master key, so when +# upgrading one, write only LITELLM_MASTER_KEY (the existing one) to .env. For anything +# beyond local evaluation, pin the image to a specific release tag. services: litellm: + image: docker.litellm.ai/berriai/litellm:main-stable build: context: . - args: - target: runtime - image: docker.litellm.ai/berriai/litellm:main-stable - ######################################### - ## Uncomment these lines to start proxy with a config.yaml file ## - # volumes: - # - ./config.yaml:/app/config.yaml - # command: - # - "--config=/app/config.yaml" - ############################################## + target: runtime + pull_policy: missing ports: - - "4000:4000" # Map the container port to the host, change the host port if necessary + - "4000:4000" + # The same image runs every component. The first word of `command` picks + # it: proxy (default), gateway, backend, ui, migrations, metrics, + # collector. See docker/README.md + # command: ["--config", "/app/config.yaml"] + # volumes: + # - ./config.yaml:/app/config.yaml:ro environment: - DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm" - # Optional: route read-only queries (find_*, count, group_by, query_raw/_first) - # to a separate reader endpoint, e.g. an Aurora reader. Leave unset for - # single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL - # is auto-refreshed alongside the writer. - # DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm" - STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI + LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?set it in .env, see the header of this file} + LITELLM_MIGRATION_DIR: /tmp/litellm-migrations + DATABASE_URL: postgresql://llmproxy:dbpassword9090@db:5432/litellm + STORE_MODEL_IN_DB: "True" env_file: - - .env # Load local .env file + - path: .env + required: false + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777 + - /app/.cache:rw,noexec,nosuid,nodev,size=128m,mode=1777 depends_on: - - db # Indicates that this service depends on the 'db' service, ensuring 'db' starts first - healthcheck: # Defines the health check configuration for the container + db: + condition: service_healthy + healthcheck: test: - CMD-SHELL - - python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')" # Command to execute for health check - interval: 30s # Perform health check every 30 seconds - timeout: 10s # Health check command times out after 10 seconds - retries: 3 # Retry up to 3 times if health check fails - start_period: 40s # Wait 40 seconds after container start before beginning health checks + - python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')" + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s db: image: postgres:16 restart: always - container_name: litellm_db environment: POSTGRES_DB: litellm POSTGRES_USER: llmproxy POSTGRES_PASSWORD: dbpassword9090 - ports: - - "5432:5432" volumes: - - postgres_data:/var/lib/postgresql/data # Persists Postgres data across container restarts + - postgres_data:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"] interval: 1s @@ -55,6 +74,8 @@ services: prometheus: image: prom/prometheus + profiles: + - monitoring volumes: - prometheus_data:/prometheus - ./prometheus.yml:/etc/prometheus/prometheus.yml @@ -68,6 +89,5 @@ services: volumes: prometheus_data: - driver: local postgres_data: - name: litellm_postgres_data # Named volume for Postgres data persistence + name: litellm_postgres_data diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 00000000000..a48aad33324 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,140 @@ +#!/bin/sh +# LiteLLM image entrypoint: docker-entrypoint.sh [COMPONENT] [ARGS...] +# +# COMPONENT selects the process this container runs; it can also be given as +# LITELLM_COMPONENT when the command carries no component. Anything after it is +# passed to that process. A first argument starting with "-" (or no argument at +# all) keeps the historical behaviour of running the monolithic proxy. +# +# proxy litellm ARGS everything in one process (default) +# gateway python -m gateway.launch ARGS inference routes, 0.0.0.0:4000 +# backend uvicorn backend.main:app ARGS management routes, 0.0.0.0:4001 +# ui nginx serving the admin UI port 3000 +# migrations python migrations/run.py prisma migrate deploy, then exit +# metrics python -m litellm.proxy.prometheus_metrics_server ARGS +# collector python -m litellm.proxy.collector ARGS +# --admin-agent the bundled LiteAdmin Slack agent (litellm-admin-agent --web ARGS) +# +# gateway and backend get their host and port defaults first, so ARGS such as +# --port 8080 override them. With no DATABASE_URL, proxy assembles DATABASE_URL +# and DATABASE_URL_READ_REPLICA from DATABASE_HOST and the other discrete +# variables the way gateway and backend do. An unknown first word is executed +# as-is (docker run sh), through ddtrace-run like every component when +# USE_DDTRACE=true. PgBouncer is not a component: LITELLM_PGBOUNCER_ENABLED=true +# starts it inside proxy and gateway. Components that write Prometheus samples +# start with an empty PROMETHEUS_MULTIPROC_DIR; metrics, collector and raw +# readers keep the workers' files, other raw commands clear stale samples. +set -eu + +usage() { + awk 'NR == 1 { next } /^#/ { sub(/^# ?/, ""); print; next } { exit }' "$0" >&2 +} + +component="${LITELLM_COMPONENT:-proxy}" +case "${1:-}" in + proxy|gateway|backend|ui|migrations|metrics|collector) + component="$1" + shift + ;; + --admin-agent) + shift + export CONNECTION_AUTH_MODE=native + exec /opt/liteadmin/bin/litellm-admin-agent --web "$@" + ;; + ""|-*) + ;; + help) + usage + exit 0 + ;; + *) + if [ -n "${PROMETHEUS_MULTIPROC_DIR:-}" ]; then + mkdir -p "$PROMETHEUS_MULTIPROC_DIR" + case "$*" in + *litellm.proxy.prometheus_metrics_server*|*litellm.proxy.collector*) ;; + *) rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db "$PROMETHEUS_MULTIPROC_DIR"/litellm_admitted_series_* ;; + esac + fi + component=raw + ;; +esac + +case "$component" in + raw) + ;; + proxy) + if { [ -z "${DATABASE_URL:-}" ] && [ -n "${DATABASE_HOST:-}" ]; } || + { [ -z "${DATABASE_URL_READ_REPLICA:-}" ] && [ -n "${DATABASE_HOST_READ_REPLICA:-}" ]; }; then + set -- python -c 'import os, sys +from litellm.proxy.db.db_url_settings import ( + DatabaseURLSettings, + add_missing_query_params, + connection_params_from_url, + idle_lifetime_params, + translate_libpq_ssl_params, +) +if os.environ.get("USE_AWS_KMS") == "True": + from litellm.secret_managers.aws_secret_manager import decrypt_env_var + os.environ.update(decrypt_env_var()) +settings = DatabaseURLSettings.from_env() +settings.apply_writer_url_to_env() +reader = settings.build_reader_url() +if reader: + os.environ["DATABASE_URL_READ_REPLICA"] = translate_libpq_ssl_params( + add_missing_query_params(reader, connection_params_from_url(os.environ.get("DATABASE_URL", ""))) + ) +if settings.max_idle_connection_lifetime is not None: + lifetime = idle_lifetime_params(settings.max_idle_connection_lifetime) + for name in ("DATABASE_URL", "DATABASE_URL_READ_REPLICA"): + if os.environ.get(name): + os.environ[name] = add_missing_query_params(os.environ[name], lifetime) +os.execvp("litellm", ["litellm", *sys.argv[1:]])' "$@" + else + set -- litellm "$@" + fi + ;; + gateway) + set -- python -m gateway.launch --workers "${NUM_WORKERS:-1}" --host 0.0.0.0 --port 4000 "$@" + ;; + backend) + set -- uvicorn backend.main:app --host 0.0.0.0 --port 4001 "$@" + ;; + ui) + sh "$(dirname "$0")/ui/docker-entrypoint.d/40-litellm-runtime-config.sh" + exec nginx -g 'daemon off;' "$@" + ;; + migrations) + set -- python /app/migrations/run.py "$@" + ;; + metrics) + set -- python -m litellm.proxy.prometheus_metrics_server "$@" + ;; + collector) + set -- python -m litellm.proxy.collector "$@" + ;; + *) + echo "docker-entrypoint.sh: unknown LITELLM_COMPONENT '$component'" >&2 + usage + exit 64 + ;; +esac + +if [ -n "${PROMETHEUS_MULTIPROC_DIR:-}" ]; then + case "$component" in + raw) ;; + metrics|collector) mkdir -p "$PROMETHEUS_MULTIPROC_DIR" ;; + *) + mkdir -p "$PROMETHEUS_MULTIPROC_DIR" + rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db "$PROMETHEUS_MULTIPROC_DIR"/litellm_admitted_series_* + ;; + esac +fi + +case "${USE_DDTRACE:-}" in + [Tt][Rr][Uu][Ee]) + export DD_TRACE_OPENAI_ENABLED="False" + exec ddtrace-run "$@" + ;; +esac + +exec "$@" diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database deleted file mode 100644 index 20f810686a3..00000000000 --- a/docker/Dockerfile.database +++ /dev/null @@ -1,168 +0,0 @@ -# syntax=docker/dockerfile:1.7 - -# Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 - -# Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a -# Pinned by digest like the other base images; bump explicitly on Node upgrades. -ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 -# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) -ARG PGBOUNCER_VERSION=1.26.0 -ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d - -FROM $UV_IMAGE AS uvbin - -FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder -ARG PGBOUNCER_VERSION -ARG PGBOUNCER_SHA256 -USER root -RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl -WORKDIR /build -RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \ - echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \ - tar xzf pgbouncer.tar.gz --strip-components=1 && \ - ./configure --prefix=/usr/local --with-openssl=/usr && \ - make -j"$(nproc)" pgbouncer && \ - install -m 0755 pgbouncer /usr/local/bin/pgbouncer - -# Admin UI builder. Pinned to the build platform so the architecture-independent -# Next.js static export compiles once natively even in a multi-arch build, -# instead of once per target arch under QEMU. -FROM --platform=$BUILDPLATFORM $UI_BUILD_IMAGE AS ui-builder - -ENV NEXT_TELEMETRY_DISABLED=1 \ - npm_config_fund=false \ - npm_config_audit=false - -WORKDIR /ui - -COPY ui/litellm-dashboard/package.json ui/litellm-dashboard/package-lock.json ./ -COPY ui/litellm-dashboard/vendor/ ./vendor/ -RUN --mount=type=cache,target=/root/.npm npm ci --prefer-offline - -COPY ui/litellm-dashboard/ ./ -RUN npm run build - -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /usr/local/bin/uv -COPY --from=uvbin /uvx /usr/local/bin/uvx - -RUN apk add --no-cache \ - bash \ - gcc \ - python-3.13 \ - python-3.13-dev \ - openssl \ - libssl3 \ - openssl-dev \ - nodejs \ - npm \ - libsndfile - -ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - UV_PYTHON_DOWNLOADS=0 \ - PATH="/app/.venv/bin:${PATH}" - -# Copy dependency metadata first for layer caching -COPY pyproject.toml uv.lock ./ -COPY enterprise/pyproject.toml enterprise/ -COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ - -# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) -RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra saml \ - --extra bedrock-realtime \ - --python python3.13 - -# Copy full source tree -COPY . . - -# Replace the committed UI bundle with the one built from this exact source. -# Clearing first drops the committed bundle's content-hashed chunks that COPY -# would otherwise leave behind alongside the fresh ones. -RUN rm -rf litellm/proxy/_experimental/out -COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/ - -# Build Admin UI before final sync (applies the enterprise color override when present) -RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh - -# Install project and workspace packages (fast - deps already cached) -RUN uv sync --frozen --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra saml \ - --extra bedrock-realtime \ - --python python3.13 - -RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - npm_config_cache=/root/.npm \ - prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -FROM $LITELLM_RUNTIME_IMAGE AS runtime -ARG LITELLM_RELEASE_TAG="" -ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} - -USER root - -# node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent -COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer - -WORKDIR /app -ENV PATH="/app/.venv/bin:${PATH}" \ - PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \ - PRISMA_CLI_QUERY_ENGINE_TYPE=binary \ - PRISMA_OFFLINE_MODE=true - -# Copy only what runtime needs. The application is installed inside the venv; -# the rest of the builder's /app is source and build metadata that must not -# ship (manifest-scanning tools attribute everything in it to this image). -# entrypoint.sh invokes litellm/proxy/prisma_migration.py by source path. -COPY --from=builder /app/.venv /app/.venv -COPY --from=builder /app/docker /app/docker -COPY --from=builder /app/schema.prisma /app/schema.prisma -COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py -# enterprise/ is imported by source path at runtime (proxy_cli puts the -# working directory on sys.path; litellm/proxy/hooks resolves -# enterprise.enterprise_hooks from it) -COPY --from=builder /app/enterprise /app/enterprise -COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras -# Prisma CLI + engines are baked under /opt/prisma, a fixed path every -# runtime uid can read and that no cache volume mount shadows (unlike -# /app/.cache or $HOME/.cache in readOnlyRootFilesystem + emptyDir setups). -# The paths are pinned via PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH and -# recorded into the generated client at build time, so `prisma migrate -# deploy` on a fresh database needs no npm and no network access -# (#33650, #24554). -COPY --from=builder /opt/prisma /opt/prisma - -RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ - find /app/.venv -type d -path "*/tornado/test" -delete && \ - chmod -R a+rX /opt/prisma && \ - test -x /opt/prisma/binaries/node_modules/.bin/prisma && \ - test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \ - python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" - -EXPOSE 4000/tcp - -ENTRYPOINT ["docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root deleted file mode 100644 index 90bcc523d9e..00000000000 --- a/docker/Dockerfile.non_root +++ /dev/null @@ -1,208 +0,0 @@ -# syntax=docker/dockerfile:1.7 - -# Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a -# Pinned by digest like the other base images; bump explicitly on Node upgrades. -ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 -# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) -ARG PGBOUNCER_VERSION=1.26.0 -ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d - -FROM $UV_IMAGE AS uvbin - -FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder -ARG PGBOUNCER_VERSION -ARG PGBOUNCER_SHA256 -USER root -RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl -WORKDIR /build -RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \ - echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \ - tar xzf pgbouncer.tar.gz --strip-components=1 && \ - ./configure --prefix=/usr/local --with-openssl=/usr && \ - make -j"$(nproc)" pgbouncer && \ - install -m 0755 pgbouncer /usr/local/bin/pgbouncer - -# Admin UI builder. Pinned to the build platform so the architecture-independent -# Next.js static export compiles once natively even in a multi-arch build, -# instead of once per target arch under QEMU. -FROM --platform=$BUILDPLATFORM $UI_BUILD_IMAGE AS ui-builder - -ENV NEXT_TELEMETRY_DISABLED=1 \ - npm_config_fund=false \ - npm_config_audit=false - -WORKDIR /ui - -COPY ui/litellm-dashboard/package.json ui/litellm-dashboard/package-lock.json ./ -COPY ui/litellm-dashboard/vendor/ ./vendor/ -RUN --mount=type=cache,target=/root/.npm npm ci --prefer-offline - -COPY ui/litellm-dashboard/ ./ -RUN npm run build - -FROM $LITELLM_BUILD_IMAGE AS builder -WORKDIR /app -USER root - -COPY --from=uvbin /uv /usr/local/bin/uv -COPY --from=uvbin /uvx /usr/local/bin/uvx - -RUN for i in 1 2 3; do \ - apk add --no-cache \ - python-3.13 \ - python-3.13-dev \ - gcc \ - rust \ - bash \ - coreutils \ - curl \ - openssl \ - libssl3 \ - libsndfile \ - nodejs \ - npm && break || sleep 5; \ - done - -ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - UV_PYTHON_DOWNLOADS=0 \ - PATH="/app/.venv/bin:${PATH}" \ - LITELLM_NON_ROOT=true \ - XDG_CACHE_HOME=/app/.cache - -# Copy dependency metadata first for layer caching -COPY pyproject.toml uv.lock ./ -COPY enterprise/pyproject.toml enterprise/ -COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ - -# Install third-party dependencies (cached unless pyproject.toml/uv.lock change) -RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ - uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra saml \ - --extra bedrock-realtime \ - --python python3.13 - -# Copy full source tree -COPY . . - -# Replace the committed UI bundle with the one built from this exact source. -# Clearing first drops the committed bundle's content-hashed chunks that COPY -# would otherwise leave behind alongside the fresh ones. -RUN rm -rf litellm/proxy/_experimental/out -COPY --from=ui-builder /ui/out/. litellm/proxy/_experimental/out/ - -# Set non-root flag for build time consistency -ENV LITELLM_NON_ROOT=true - -RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \ - cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \ - cp /app/litellm/proxy/logo.png /var/lib/litellm/assets/logo.png && \ - touch /var/lib/litellm/ui/.litellm_ui_ready - -RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ - uv sync --frozen --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra saml \ - --extra bedrock-realtime \ - --python python3.13 - -RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - npm_config_cache=/root/.npm \ - prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ - sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh - -FROM $LITELLM_RUNTIME_IMAGE AS runtime -ARG LITELLM_RELEASE_TAG="" -ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} -WORKDIR /app -USER root - -RUN for i in 1 2 3; do \ - apk upgrade --no-cache && break || sleep 5; \ - done && \ - for i in 1 2 3; do \ - apk add --no-cache python-3.13 bash openssl libssl3 tzdata libsndfile nodejs libevent && break || sleep 5; \ - done -COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer - -# Copy only what runtime needs. The application is installed inside the venv; -# the rest of the builder's /app is source and build metadata that must not -# ship (manifest-scanning tools attribute everything in it to this image). -# entrypoint.sh invokes litellm/proxy/prisma_migration.py by source path. -COPY --from=builder /app/.venv /app/.venv -COPY --from=builder /app/docker /app/docker -COPY --from=builder /app/schema.prisma /app/schema.prisma -COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py -# enterprise/ is imported by source path at runtime (proxy_cli puts the -# working directory on sys.path; litellm/proxy/hooks resolves -# enterprise.enterprise_hooks from it) -COPY --from=builder /app/enterprise /app/enterprise -COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras -# Prisma CLI + engines are baked under /opt/prisma, a fixed path every runtime -# uid can read and that no cache volume mount shadows (unlike /app/.cache or -# $HOME/.cache under readOnlyRootFilesystem + emptyDir or arbitrary-uid setups). -# PRISMA_CLI_QUERY_ENGINE_TYPE=binary makes the CLI use the baked binary query -# engine directly, so `prisma migrate deploy` on a fresh database needs no npm -# and no network access; without it the CLI looks for the library engine, which -# prisma stopped baking, and falls back to a download that fails offline or as a -# non-writable uid (#33650, #24554). -COPY --from=builder /opt/prisma /opt/prisma -COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui -COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets - -# XDG_CACHE_HOME is intentionally left unset so it falls back to $HOME/.cache -# (/app/.cache, writable by the runtime uid). The prisma bake at the read-only -# /opt/prisma is anchored by PRISMA_BINARY_CACHE_DIR / PRISMA_CLI_PATH, so -# nothing needs XDG to point there; pointing it at the read-only bake would -# deny any XDG-aware library that writes a cache at runtime. -ENV PATH="/app/.venv/bin:${PATH}" \ - PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - PRISMA_CLI_PATH=/opt/prisma/binaries/node_modules/.bin/prisma \ - PRISMA_CLI_QUERY_ENGINE_TYPE=binary \ - HOME=/app \ - LITELLM_NON_ROOT=true \ - PRISMA_SKIP_POSTINSTALL_GENERATE=1 \ - PRISMA_HIDE_UPDATE_MESSAGE=1 \ - PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \ - PRISMA_OFFLINE_MODE=true - -RUN mkdir -p /nonexistent /app/.cache /var/lib/litellm/assets /var/lib/litellm/ui && \ - chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent && \ - PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \ - chown -R nobody:nogroup "$PRISMA_PATH" && \ - LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \ - [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup "$LITELLM_PKG_MIGRATIONS_PATH" || true && \ - LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \ - chgrp -R 0 "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \ - [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 "$LITELLM_PROXY_EXTRAS_PATH" || true && \ - chmod -R g=u "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \ - [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u "$LITELLM_PROXY_EXTRAS_PATH" || true && \ - chmod -R g+w "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \ - [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w "$LITELLM_PROXY_EXTRAS_PATH" || true && \ - chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \ - chmod -R a+rX /opt/prisma && \ - test -x /opt/prisma/binaries/node_modules/.bin/prisma && \ - test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \ - ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null 2>&1 && \ - python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" - -USER 65534 - -EXPOSE 4000/tcp - -ENTRYPOINT ["/app/docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] diff --git a/docker/README.md b/docker/README.md index ac82c6caebf..3ec26219420 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,15 +2,15 @@ This guide provides instructions for building and running the LiteLLM application using Docker and Docker Compose. -> **Just want to run LiteLLM?** This guide builds from source. To run the published -> image instead, use `docker-compose.quickstart.yml` in this directory — the -> two-service stack (gateway + Postgres) that the -> [Docker quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start) documents: +> **Just want to run LiteLLM?** `docker-compose.yml` in the repository root runs +> the published image with a Postgres database, the stack the +> [Docker quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start) documents, +> and it works on its own outside a checkout: > > ```bash -> curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker/docker-compose.quickstart.yml +> curl -sSLO https://github.com/BerriAI/litellm/raw/main/docker-compose.yml > printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env -> docker compose -f docker-compose.quickstart.yml up -d +> docker compose up -d > ``` ## Prerequisites @@ -20,33 +20,52 @@ This guide provides instructions for building and running the LiteLLM applicatio ## Building and Running the Application -To build and run the application, you will use the `docker-compose.yml` file located in the root of the project. This file is configured to use the `Dockerfile.non_root` for a secure, non-root container environment. +The same `docker-compose.yml` builds from source when you pass `--build`: it builds the `Dockerfile` in the repository root, the one image LiteLLM ships. `docker-entrypoint.sh` next to it is the only entrypoint script; every container starts through it -### 1. Set the Master Key +## One image, many components -The application requires a `LITELLM_MASTER_KEY` for signing and validating tokens. You must set this key as an environment variable before running the application. +Every LiteLLM container runs the same image. The first word of the container command (or the `LITELLM_COMPONENT` environment variable when the command carries only flags) picks the process the container runs, and everything after it is handed to that process unchanged: -Create a `.env` file in the root of the project and add the following line: +| Component | Runs | Port | +|--------------|-----------------------------------------------------|------| +| `proxy` | `litellm ...` (everything in one process, default) | 4000 | +| `gateway` | `python -m gateway.launch ...` (inference routes) | 4000 | +| `backend` | `uvicorn backend.main:app ...` (management routes) | 4001 | +| `ui` | nginx serving the static admin UI | 3000 | +| `migrations` | `python migrations/run.py`, `prisma migrate deploy` then exit | | +| `metrics` | `python -m litellm.proxy.prometheus_metrics_server ...` | `--port` | +| `collector` | `python -m litellm.proxy.collector ...` | | -``` -LITELLM_MASTER_KEY=your-secret-key +```bash +docker run -p 4000:4000 litellm --config /app/config.yaml # proxy, exactly as before +docker run -p 4000:4000 litellm gateway --port 4000 # componentized data plane +docker run -p 4001:4001 -e LITELLM_COMPONENT=backend litellm # same, chosen through the env +docker run -p 3000:3000 --read-only --tmpfs /tmp litellm ui # admin UI behind nginx +docker run -e DATABASE_URL=... litellm migrations # one-off schema migration job +docker run -it litellm sh # anything else runs verbatim ``` -Replace `your-secret-key` with a strong, randomly generated secret. +PgBouncer is not a separate component: `LITELLM_PGBOUNCER_ENABLED=true` starts an in-container PgBouncer in front of `DATABASE_URL` inside `proxy` and `gateway`. `USE_DDTRACE=true` wraps whichever component runs with `ddtrace-run`, and `PROMETHEUS_MULTIPROC_DIR` is emptied of stale samples before any workers fork (the `metrics` and `collector` sidecars only read it, so their restart keeps the live samples) + +The image runs as uid `65532` (`nonroot` in the Wolfi base) and also works as an arbitrary uid in gid 0, the shape OpenShift `restricted-v2` assigns, because everything it writes at runtime lives under `/app/.cache`, `/var/lib/litellm` and `/tmp`. Mount those (or set `readOnlyRootFilesystem` with emptyDirs there) for a read-only root filesystem. Prisma's CLI and engines are baked under `/opt/prisma`, so migrations need neither network nor a writable home + +### 1. Set the Master and Salt Keys + +The proxy signs virtual keys with `LITELLM_MASTER_KEY` and encrypts stored provider credentials with `LITELLM_SALT_KEY`. Compose reads both from a `.env` file in the directory you run it from, so generate them once: + +```bash +printf 'LITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env +``` + +Keep the file: regenerating `LITELLM_SALT_KEY` makes credentials already stored in the database unreadable. A database created without `LITELLM_SALT_KEY` encrypted them with the master key, so when upgrading one, write only the existing `LITELLM_MASTER_KEY` to `.env` and leave `LITELLM_SALT_KEY` out. Provider keys such as `OPENAI_API_KEY` go in the same file, and the whole file is passed to the container ### 2. Build and Run the Containers -Once you have set the `LITELLM_MASTER_KEY`, you can build and run the containers using the following command: - ```bash docker compose up -d --build ``` -This command will: - -- Build the Docker image using `Dockerfile.non_root`. -- Start the `litellm`, `litellm_db`, and `prometheus` services in detached mode (`-d`). -- The `--build` flag ensures that the image is rebuilt if there are any changes to the Dockerfile or the application code. +This command builds the image from the root `Dockerfile` and starts the `litellm` and `db` services in detached mode. Without `--build`, `docker compose up` pulls the published `main-stable` image instead. Add `--profile monitoring` to also start Prometheus on port 9090, scraping the proxy with the root `prometheus.yml` ### 3. Verifying the Application is Running @@ -80,13 +99,13 @@ LITELLM_LICENSE="your-enterprise-license" PROXY_BASE_URL=https://gateway.example.com ``` -For the unified source deployment described above, put them in its `.env` file and rebuild: +For the compose deployment described above, put them in its `.env` file and rebuild: ```bash docker compose up -d --build ``` -In componentized deployments, set the flag and license on the backend container and route `/admin/mcp` to the backend service. The gateway component excludes this endpoint. The unified, database, non-root, and backend image builds bundle the connector +In componentized deployments, set the flag and license on the `backend` component and route `/admin/mcp` to it. The `gateway` component excludes this endpoint. The image bundles the connector for every component Hosting is disabled by default. Opting in requires a valid base Enterprise license; an unlicensed opt-in or invalid flag value prevents startup. Enabling it reserves `/admin`, so rename any MCP server alias called `admin` first @@ -96,34 +115,18 @@ Embedded responses default to `full`; selecting `LITELLM_ADMIN_RESPONSE_VIEW=com See the [LiteAdmin MCP guide](https://docs.litellm.ai/docs/proxy/liteadmin_mcp#run-liteadmin-mcp-inside-litellm) for client configuration, tool restrictions, and verification -## Hardened / Offline Testing +## Hardening -To ensure changes are safe for non-root, read-only root filesystems and restricted egress, always validate with the hardened compose file: +The compose file runs the proxy the way a locked-down cluster would: as the image's non-root user with a read-only root filesystem, every capability dropped, `no-new-privileges` set, and tmpfs mounts only at `/tmp` and `/app/.cache`. The image is built for that, so a change that makes the proxy write anywhere else fails here before it fails in Kubernetes. To try an arbitrary uid in gid 0, the shape OpenShift `restricted-v2` assigns, add `user: "101:0"` to the `litellm` service + +Prisma's CLI and engines are baked under `/opt/prisma`, so migrations run without network access. Verify with: ```bash -docker compose -f docker-compose.yml -f docker-compose.hardened.yml build --no-cache -docker compose -f docker-compose.yml -f docker-compose.hardened.yml up -d +docker run --rm --network none --entrypoint prisma docker.litellm.ai/berriai/litellm:main-stable --version ``` -This setup: -- Builds from `docker/Dockerfile.non_root` with Prisma engines and Node toolchain baked into the image. -- Runs the proxy as a non-root user with a read-only rootfs and only writable tmpfs mounts: - - `/app/cache` (Prisma/NPM cache; backing `PRISMA_BINARY_CACHE_DIR`, `NPM_CONFIG_CACHE`, `XDG_CACHE_HOME`) - - `/app/migrations` (Prisma migration workspace; backing `LITELLM_MIGRATION_DIR`) -- Pre-builds and serves the admin UI from read-only paths: - - `/var/lib/litellm/ui` (pre-restructured Next.js UI with `.litellm_ui_ready` marker) - - `/var/lib/litellm/assets` (UI logos and assets) -- Routes all outbound traffic through a local Squid proxy that denies egress, so Prisma migrations must use the cached CLI and engines. - -You should also verify offline Prisma behaviour with: - -```bash -docker run --rm --network none --entrypoint prisma ghcr.io/berriai/litellm:main-stable --version -``` - -This command should succeed (showing engine versions) even with `--network none`, confirming that Prisma binaries are available without network access. - ## Troubleshooting -- **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project. -- **`Master key is not initialized`**: This error means the `LITELLM_MASTER_KEY` environment variable is not set. Make sure you have created a `.env` file in the project root with the `LITELLM_MASTER_KEY` defined. +- **`required variable LITELLM_MASTER_KEY is missing a value`**: Compose did not find a `.env` file with `LITELLM_MASTER_KEY` in the directory you ran it from. Generate one as shown above. +- **`password authentication failed for user "llmproxy"`**: the `litellm_postgres_data` volume was initialised by an older compose file with different credentials. `docker compose down -v` drops it and the next `up` recreates the database. +- **`build_admin_ui.sh: not found`**: the build context is wrong. Run `docker compose` from the root of the repository. diff --git a/docker/build_from_pip/Dockerfile.build_from_pip b/docker/build_from_pip/Dockerfile.build_from_pip deleted file mode 100644 index 25a7ed2fa97..00000000000 --- a/docker/build_from_pip/Dockerfile.build_from_pip +++ /dev/null @@ -1,64 +0,0 @@ -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9@sha256:10902f58a1606787602f303954cea099626a4adb02acbac4c69920fe9d278f82 -FROM $UV_IMAGE AS uvbin - -FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d - -ARG LITELLM_VERSION=1.83.0 - -WORKDIR /app - -COPY --from=uvbin /uv /usr/local/bin/uv -COPY --from=uvbin /uvx /usr/local/bin/uvx - -RUN apt-get update && \ - apt-get install -y --no-install-recommends gcc libffi-dev nodejs npm && \ - rm -rf /var/lib/apt/lists/* - -ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - PATH="/app/.venv/bin:${PATH}" - -COPY schema.prisma . - -# This image is specifically for validating/installing the published PyPI -# artifact, not the checked-out source tree. -# Keep the moved proxy-runtime packages explicit until the published PyPI -# artifact includes that extra; newer releases will simply dedupe these. -RUN uv venv --python python && \ - uv pip install --python /app/.venv/bin/python \ - "litellm[proxy,proxy-runtime]==${LITELLM_VERSION}" \ - "google-cloud-aiplatform==1.133.0" \ - "google-genai==1.37.0" \ - "anthropic[vertex]==0.84.0" \ - "grpcio==1.84.1" \ - "prometheus-client==0.20.0" \ - "langfuse==2.59.7" \ - "opentelemetry-api==1.28.0" \ - "opentelemetry-sdk==1.28.0" \ - "opentelemetry-exporter-otlp==1.28.0" \ - "ddtrace==4.11.0" \ - "sentry-sdk==2.21.0" \ - "mangum==0.17.0" \ - "azure-ai-contentsafety==1.0.0" \ - "azure-storage-file-datalake==12.20.0" \ - "pypdf==6.7.5" \ - "llm-sandbox==0.3.31" \ - "detect-secrets==1.5.0" \ - "prisma==0.11.0" \ - "openai==2.24.0" - -RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - npm_config_cache=/root/.npm \ - prisma generate --schema=./schema.prisma && \ - chmod -R a+rX /opt/prisma && \ - python -c "import sys; from prisma.client import BINARY_PATHS; bad = sorted(p for group in BINARY_PATHS.model_dump().values() for p in group.values() if not p.startswith('/opt/prisma/')); sys.exit('prisma engines baked outside /opt/prisma: %r' % bad) if bad else None" - -ENV PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries - -COPY docker/prod_entrypoint.sh /app/docker/prod_entrypoint.sh -RUN sed -i 's/\r$//' /app/docker/prod_entrypoint.sh && chmod +x /app/docker/prod_entrypoint.sh - -EXPOSE 4000/tcp - -ENTRYPOINT ["/app/docker/prod_entrypoint.sh"] -CMD ["--port", "4000"] diff --git a/docker/build_from_pip/Readme.md b/docker/build_from_pip/Readme.md deleted file mode 100644 index ad043588f33..00000000000 --- a/docker/build_from_pip/Readme.md +++ /dev/null @@ -1,9 +0,0 @@ -# Docker to build LiteLLM Proxy from litellm pip package - -### When to use this ? - -If you need to build LiteLLM Proxy from litellm pip package, you can use this Dockerfile as a reference. - -### Why build from pip package ? - -- If your company has a strict requirement around security / building images you can follow steps outlined here \ No newline at end of file diff --git a/docker/component_entrypoint.sh b/docker/component_entrypoint.sh deleted file mode 100755 index 3cd530e3009..00000000000 --- a/docker/component_entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -# stale samples from a previous container incarnation would be summed into the aggregate -if [ -n "$PROMETHEUS_MULTIPROC_DIR" ]; then - mkdir -p "$PROMETHEUS_MULTIPROC_DIR" - rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db "$PROMETHEUS_MULTIPROC_DIR"/litellm_admitted_series_* -fi - -case "$USE_DDTRACE" in - [Tt][Rr][Uu][Ee]) - export DD_TRACE_OPENAI_ENABLED="False" - exec ddtrace-run "$@" - ;; -esac - -exec "$@" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100755 index 003d9b21db8..00000000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -euo pipefail - -REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -VENV_PYTHON="$REPO_ROOT/.venv/bin/python" -MIGRATION_SCRIPT="$REPO_ROOT/litellm/proxy/prisma_migration.py" - -if [ -x "$VENV_PYTHON" ]; then - "$VENV_PYTHON" "$MIGRATION_SCRIPT" -elif command -v uv >/dev/null 2>&1; then - (cd "$REPO_ROOT" && uv run --no-sync python "$MIGRATION_SCRIPT") -else - python3 "$MIGRATION_SCRIPT" -fi - -echo "Migration script ran successfully!" diff --git a/docker/install_auto_router.sh b/docker/install_auto_router.sh deleted file mode 100755 index 4fedf201b41..00000000000 --- a/docker/install_auto_router.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# semantic-router dependencies are installed via `uv sync`. diff --git a/docker/prod_entrypoint.sh b/docker/prod_entrypoint.sh deleted file mode 100644 index 4386be65a32..00000000000 --- a/docker/prod_entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -if [ "$1" = "--admin-agent" ]; then - shift - export CONNECTION_AUTH_MODE=native - exec /opt/liteadmin/bin/litellm-admin-agent --web "$@" -fi - -case "$USE_DDTRACE" in - [Tt][Rr][Uu][Ee]) - export DD_TRACE_OPENAI_ENABLED="False" - exec ddtrace-run litellm "$@" - ;; -esac - -exec litellm "$@" diff --git a/docker/tests/nonroot.yaml b/docker/tests/nonroot.yaml deleted file mode 100644 index 36118ca8c59..00000000000 --- a/docker/tests/nonroot.yaml +++ /dev/null @@ -1,18 +0,0 @@ -schemaVersion: 2.0.0 - -metadataTest: - entrypoint: ["docker/prod_entrypoint.sh"] - user: "65534" - workdir: "/app" - -fileExistenceTests: - - name: "Prisma Folder" - path: "/usr/local/lib/python3.13/site-packages/prisma/" - shouldExist: true - uid: 65534 - gid: 65534 - - name: "Prisma Schema" - path: "/usr/local/lib/python3.13/site-packages/prisma/schema.prisma" - shouldExist: true - uid: 65534 - gid: 65534 diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py b/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py index f920aa7ac13..a0c896b7e05 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py @@ -17,6 +17,7 @@ from typing import List, Optional, Union from litellm._logging import verbose_logger from litellm.caching import DualCache +from litellm.exceptions import MissingMessageRoleError from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -86,6 +87,8 @@ class PagerDutyAlerting(SlackAlerting): Record a failure event. Only send an alert to PagerDuty if the configured *failure* threshold is exceeded in the specified window. """ + if isinstance(kwargs.get("exception"), MissingMessageRoleError): + return now = datetime.now(timezone.utc) standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get( "standard_logging_object" diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index a31e926256a..abc2487a4a5 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -8,7 +8,6 @@ import re from collections.abc import Callable, Mapping, Sequence from dataclasses import replace as dataclasses_replace from datetime import datetime, timedelta -from types import MappingProxyType from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast from pydantic import TypeAdapter, ValidationError @@ -21,6 +20,7 @@ from litellm_enterprise.proxy.common_utils.managed_object_attribution import ( import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs from litellm.constants import ( BATCH_LINE_ITEMS_MAX_PART_BYTES, MANAGED_OBJECT_STALENESS_CUTOFF_DAYS, @@ -766,8 +766,7 @@ class CheckBatchCost: credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {} _file_content = await afile_content( file_id=cast(str, raw_output_file_id), - _litellm_internal_model_credentials=MappingProxyType(dict(credentials)), - **cast(FileContentCallOptions, credentials), + **cast(FileContentCallOptions, {**credentials, **stored_s3_bucket_kwargs(cast(str, raw_output_file_id))}), ) content_bytes: Final = await _file_content_bytes(_file_content) @@ -867,10 +866,7 @@ class CheckBatchCost: error_file_failed_requests: Final = await count_error_file_failed_requests( response, custom_llm_provider=batch_file_provider, - litellm_params={ - **credentials, - "_litellm_internal_model_credentials": MappingProxyType(dict(credentials)), - }, + litellm_params=credentials, ) batch_result: Final = ( output_file_result @@ -1029,11 +1025,7 @@ class CheckBatchCost: custom_llm_provider=deployment_info.litellm_params.custom_llm_provider, ) cost_provider: Final = cast(BatchFileProvider, llm_provider) - credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {} - file_access: Final = { - **credentials, - "_litellm_internal_model_credentials": MappingProxyType(dict(credentials)), - } + file_access: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {} output_content: Final = ( await fetch_batch_managed_file_content( response.output_file_id, custom_llm_provider=cost_provider, litellm_params=file_access diff --git a/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py b/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py index a4a40a7a3d7..1c433d04819 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py @@ -182,7 +182,7 @@ class ManagedObjectCreatorAttribution: team_id: Final = getattr(job, "team_id", None) rows: Final = _CreatorRows( key=await self._live_key_row(api_key, job.created_by), - team=await self._live_team_row(team_id), + team=await self._live_team_row(team_id, job.created_at), user=await self._user_row(object_id, _creator_user_id(job.created_by)), ) return ResponseCreator( @@ -269,10 +269,15 @@ class ManagedObjectCreatorAttribution: verbose_proxy_logger.error(f"{self._poller_name}: could not look up team {team_id} for {object_id}: {e}") return None - async def _live_team_row(self, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None: + async def _live_team_row( + self, team_id: str | None, created_before: datetime | None = None + ) -> prisma_models.LiteLLM_TeamTable | None: if not team_id: return None - return await _team_table(self._prisma_client).find_unique(where={"team_id": team_id}) + team: Final = await _team_table(self._prisma_client).find_unique(where={"team_id": team_id}) + if team is not None and created_before is not None and team.created_at > created_before: + return None + return team def _response_creator_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str, object]: diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 91c5e2c5f39..bbb09e75b47 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -49,6 +49,7 @@ from litellm.constants import ( ) from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.cloud_storage_security import stored_s3_bucket_kwargs from litellm.litellm_core_utils.core_helpers import require_attributes from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.litellm_core_utils.internal_call_metadata import requests_background @@ -78,6 +79,7 @@ from litellm.proxy.litellm_pre_call_utils import ( from litellm.proxy.openai_files_endpoints.common_utils import ( BATCH_CREATE_HIDDEN_PARAM, FILE_LIST_CONTINUATION_CHUNK_SIZE, + TeamProviderDeployment, _is_base64_encoded_unified_file_id, apply_unified_file_ids, decode_model_from_file_id, @@ -127,7 +129,6 @@ class _ManagedFileRetrieve(Protocol): self, *, file_id: str, - _litellm_internal_model_credentials: Mapping[str, object] | None = None, **kwargs: Unpack[FileRetrieveCallOptions], ) -> OpenAIFileObject: ... @@ -396,11 +397,12 @@ class _CursorPageArgs(TypedDict, total=False): class _RouterFileCallKwargs(TypedDict, total=False): client: ReadOnly[AsyncOpenAI | None] custom_llm_provider: ReadOnly[str | None] + stored_s3_bucket_name: ReadOnly[str] _MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str]) _FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"}) -_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"}) +_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id"}) @dataclass(frozen=True, slots=True) @@ -459,19 +461,11 @@ async def _delete_provider_file( provider_file_id: str, request_data: Mapping[str, object], ) -> None: - credentials: Final = llm_router.registry().credentials(model_id, form="with_provider") - delete_data: Final = { - **{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS}, - **( - {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))} - if credentials is not None - else {} - ), - } + delete_data: Final = {k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS} await llm_router.afile_delete( model=model_id, file_id=provider_file_id, - **cast(_RouterFileCallKwargs, delete_data), + **cast(_RouterFileCallKwargs, {**delete_data, **stored_s3_bucket_kwargs(provider_file_id)}), ) @@ -1160,26 +1154,19 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): *, provider_file_id: str, call_options: Mapping[str, object], - internal_model_credentials: Mapping[str, object] | None = None, ) -> OpenAIFileObject: """Retrieve provider file details with transient retries and SDK retries disabled.""" retrieve_file: Final = cast(_ManagedFileRetrieve, litellm.afile_retrieve) retrieve_options: Final = cast( FileRetrieveCallOptions, - {**call_options, "max_retries": 0}, + {**call_options, **stored_s3_bucket_kwargs(provider_file_id), "max_retries": 0}, ) for attempt in range(len(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS) + 1): if attempt > 0: await self._sleep(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS[attempt - 1]) try: - if internal_model_credentials is None: - return await retrieve_file( - file_id=provider_file_id, - **retrieve_options, - ) return await retrieve_file( file_id=provider_file_id, - _litellm_internal_model_credentials=MappingProxyType(dict(internal_model_credentials)), **retrieve_options, ) except Exception as error: @@ -1221,7 +1208,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): self._afile_retrieve_with_retries( provider_file_id=provider_file_id, call_options=router_credentials, - internal_model_credentials=router_credentials, ), timeout=BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS, ) @@ -1443,6 +1429,43 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): user_api_key_dict=owner, ) + async def register_provider_file( + self, + *, + provider_file_id: str, + content_type: str, + deployment: TeamProviderDeployment, + owner: UserAPIKeyAuth, + llm_router: Router | None, + ) -> str | None: + unified_file_id: Final = self.get_unified_output_file_id( + provider_file_id, deployment.deployment_id, deployment.public_model_name, file_type=content_type + ) + provider_file_object, _ = await self._fetch_provider_file_object( + unified_file_id=unified_file_id, + provider_file_id=provider_file_id, + model_id=deployment.deployment_id, + model_name=None, + llm_router=llm_router, + ) + if provider_file_object is None: + return None + try: + await self.store_unified_file_id( + file_id=unified_file_id, + file_object=provider_file_object, + litellm_parent_otel_span=None, + model_mappings={deployment.deployment_id: provider_file_id}, + user_api_key_dict=owner, + ) + except Exception as error: # noqa: BLE001 # the upload succeeded, so the caller keeps the provider id + verbose_logger.warning( + f"Failed to register provider_file_id={sanitize_for_log(provider_file_id)} as a managed file: " + f"{type(error).__name__} {sanitize_for_log(error)}" + ) + return None + return unified_file_id + async def list_user_batches( self, user_api_key_dict: UserAPIKeyAuth, @@ -1774,8 +1797,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if potential_file_id and "llm_output_file_id," in potential_file_id: model_id = self.get_model_id_from_unified_file_id(potential_file_id) if model_id: + provider_file_id: Final = self.get_output_file_id_from_unified_file_id(potential_file_id) data["model"] = model_id - data["file_id"] = self.get_output_file_id_from_unified_file_id(potential_file_id) + data["file_id"] = provider_file_id + data.update(stored_s3_bucket_kwargs(provider_file_id)) elif call_type == CallTypes.acreate_batch.value: input_file_id = cast(Optional[str], data.get("input_file_id")) if input_file_id: @@ -2270,10 +2295,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): unified_batch_id = SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format(model_id, batch_id) return base64.urlsafe_b64encode(unified_batch_id.encode()).decode().rstrip("=") - def get_unified_output_file_id(self, output_file_id: str, model_id: str, model_name: Optional[str]) -> str: + def get_unified_output_file_id( + self, output_file_id: str, model_id: str, model_name: Optional[str], file_type: str = "application/json" + ) -> str: deterministic_uuid: Final = uuid5(uuid5(NAMESPACE_URL, model_id), output_file_id) unified_output_file_id = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format( - "application/json", + file_type, str(deterministic_uuid), model_name or "", output_file_id, @@ -2866,21 +2893,22 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): exception_dict = {} for model_id, provider_file_id in specific_model_file_id_mapping.items(): try: - # Cloud-storage providers (e.g. Bedrock S3) validate file ids - # against the deployment's configured bucket, which they only - # trust from this immutable server-side snapshot, never from - # request params. - credentials = llm_router.registry().credentials(model_id, form="with_provider") - if credentials is not None: - data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials))) - else: - data.pop("_litellm_internal_model_credentials", None) return cast( HttpxBinaryResponseContent, await llm_router.afile_content( model=model_id, file_id=provider_file_id, - **cast(_RouterFileCallKwargs, data), + **cast( + _RouterFileCallKwargs, + { + **data, + **( + stored_s3_bucket_kwargs(provider_file_id) + if isinstance(provider_file_id, str) + else {} + ), + }, + ), ), ) except Exception as e: diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py b/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py index fcaf58099b9..6e16ed8f187 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_vector_stores.py @@ -123,7 +123,7 @@ class _PROXY_LiteLLMManagedVectorStores( """ # Use the router to create the vector store response: Final = await llm_router.avector_store_create(model=model, **request_data) - return cast(VectorStoreCreateResponse, response) + return response # ============================================================================ # VECTOR STORE CRUD OPERATIONS diff --git a/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py b/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py index a2322d7f650..bb3b712d50b 100644 --- a/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py +++ b/enterprise/litellm_enterprise/types/enterprise_callbacks/send_emails.py @@ -1,9 +1,9 @@ import enum -from typing import Dict, List, Optional from pydantic import Field from litellm.proxy._types import WebhookEvent +from litellm.types.integrations.model_deprecation_email import MODEL_DEPRECATION_EMAIL_EVENT from litellm.types.llms.base import LiteLLMBaseModel @@ -26,7 +26,7 @@ class SendKeyCreatedEmailEvent(WebhookEvent): class SendKeyRotatedEmailEvent(WebhookEvent): virtual_key: str - key_alias: Optional[str] = None + key_alias: str | None = None """ The virtual key that was rotated this will be sk-123xxx, since we will be emailing this to the user to start using the new key @@ -34,23 +34,33 @@ class SendKeyRotatedEmailEvent(WebhookEvent): class EmailEvent(str, enum.Enum): + model_deprecations = MODEL_DEPRECATION_EMAIL_EVENT virtual_key_created = "Virtual Key Created" new_user_invitation = "New User Invitation" virtual_key_rotated = "Virtual Key Rotated" soft_budget_crossed = "Soft Budget Crossed" max_budget_alert = "Max Budget Alert" + class EmailEventSettings(LiteLLMBaseModel): event: EmailEvent enabled: bool + + class EmailEventSettingsUpdateRequest(LiteLLMBaseModel): - settings: List[EmailEventSettings] + settings: list[EmailEventSettings] + + class EmailEventSettingsResponse(LiteLLMBaseModel): - settings: List[EmailEventSettings] + settings: list[EmailEventSettings] + + class DefaultEmailSettings(LiteLLMBaseModel): """Default settings for email events""" - settings: Dict[EmailEvent, bool] = Field( + + settings: dict[EmailEvent, bool] = Field( default_factory=lambda: { + EmailEvent.model_deprecations: False, # Explicit opt-in for owner retirement reminders. EmailEvent.virtual_key_created: True, # On by default EmailEvent.new_user_invitation: True, # On by default EmailEvent.virtual_key_rotated: True, # On by default @@ -58,10 +68,12 @@ class DefaultEmailSettings(LiteLLMBaseModel): EmailEvent.max_budget_alert: True, # On by default } ) - def to_dict(self) -> Dict[str, bool]: + + def to_dict(self) -> dict[str, bool]: """Convert to dictionary with string keys for storage""" return {event.value: enabled for event, enabled in self.settings.items()} + @classmethod - def get_defaults(cls) -> Dict[str, bool]: + def get_defaults(cls) -> dict[str, bool]: """Get the default settings as a dictionary with string keys""" - return cls().to_dict() \ No newline at end of file + return cls().to_dict() diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 7db2de1f874..64fe03769b8 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.81" +version = "0.1.83" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.81" +version = "0.1.83" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/examples/bigmodel.md b/examples/bigmodel.md deleted file mode 100644 index 88a69796ebe..00000000000 --- a/examples/bigmodel.md +++ /dev/null @@ -1,63 +0,0 @@ -# BigModel (Zhipu AI) - -Use `bigmodel/` for the domestic BigModel platform and `zai/` for international Z.AI. Each platform uses its own credentials and default URL - -```yaml -model_list: - - model_name: glm - litellm_params: - model: bigmodel/glm-5.3 - api_key: os.environ/BIGMODEL_API_KEY - api_base: https://open.bigmodel.cn -``` - -One model configuration supports all three native APIs. The provider selects the upstream endpoint from the requested API - -| Proxy endpoint | BigModel endpoint | -| --- | --- | -| `/v1/chat/completions` | `/api/paas/v4/chat/completions` | -| `/v1/responses` | `/api/v1/responses` | -| `/v1/messages` | `/api/anthropic/v1/messages` | - -`api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix - -A GLM Coding Plan key uses `https://open.bigmodel.cn/api/coding/paas/v4` as its `api_base`, since Coding Plan and standard keys do not work on each other's endpoints. Chat Completions then calls the Coding Plan endpoint, and Responses and Messages keep the endpoints in the table above - -The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY` - -```python -import litellm - -chat = litellm.completion( - model="bigmodel/glm-5.3", - messages=[{"role": "user", "content": "Hello"}], - thinking={"type": "enabled"}, - reasoning_effort="low", -) - -response = litellm.responses( - model="bigmodel/glm-5.3", - input="Hello", - reasoning={"effort": "low"}, - store=True, -) - -async def messages(): - return await litellm.anthropic.messages.acreate( - model="bigmodel/glm-5.3", - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - thinking={"type": "adaptive"}, - output_config={"effort": "low"}, - ) -``` - -Each API retains its native response format and streaming events. The native Messages SDK entry point is asynchronous, as is the proxy's `/v1/messages` route - -GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 supports `low`, `high`, and `max` reasoning effort. Messages preserves native `thinking`, `output_config`, and `reasoning_effort` parameters - -Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search - -The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices - -API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27 diff --git a/examples/lar1_ollama_config.yaml b/examples/lar1_ollama_config.yaml deleted file mode 100644 index 998cbf169b3..00000000000 --- a/examples/lar1_ollama_config.yaml +++ /dev/null @@ -1,45 +0,0 @@ -model_list: - - model_name: agent-router - litellm_params: - model: ollama/qwen3.5:9b - api_base: http://127.0.0.1:11434 - model_info: - id: cloud-smart - type: cloud-smart - - - model_name: agent-router - litellm_params: - model: ollama/phi4-mini:latest - api_base: http://127.0.0.1:11434 - model_info: - id: cloud-fast - type: cloud-fast - - - model_name: agent-router - litellm_params: - model: ollama/llama3.2:3b - api_base: http://127.0.0.1:11434 - model_info: - id: local - type: local - - - model_name: agent-router - litellm_params: - model: ollama/lfm2.5-thinking:latest - api_base: http://127.0.0.1:11434 - model_info: - id: deep - type: deep - -router_settings: - routing_strategy: lar1 - routing_strategy_args: - confidence_threshold_low: 0.3 - confidence_threshold_medium: 0.5 - confidence_threshold_high: 0.7 - -general_settings: - master_key: sk-lar1-demo - -litellm_settings: - set_verbose: true diff --git a/gateway/AGENTS.md b/gateway/AGENTS.md new file mode 100644 index 00000000000..1bb4013179f --- /dev/null +++ b/gateway/AGENTS.md @@ -0,0 +1,16 @@ +# Gateway image entry point + +`python -m gateway.launch` is how the gateway image starts. `launch.py` hands its arguments to uvicorn's own CLI and serves `main.py`, which trims the proxy app to the LLM data plane + +- Importing `launch.py` stays cheap. Every uvicorn worker imports it again before it can answer its first health ping, so LiteLLM imports wait until `main` or `worker_app` runs +- PgBouncer starts exactly once per pod, before the workers start, and the workers inherit the loopback URL through `DATABASE_URL`. `configure_database` is the only place that does this + +## Rust forwarding + +`launch.py` is an entry point of Rust forwarding, and every forwarding decision lives in `litellm/proxy/rust_forward/`, whose `AGENTS.md` states the contract + +- `main` dispatches on `rust_forward_mode` and does nothing else about forwarding. With forwarding off or refused it serves exactly as before +- `python_server` reads the flags and environment through uvicorn's own CLI parser, so Rust gets the values uvicorn would run with. Never reparse a uvicorn flag by hand +- Under forwarding the supervisor process runs `configure_database`, as `before_start`, once forwarding is known to be possible. The supervised child never runs it and inherits the exported URL +- The supervised child appends `mode.uvicorn_flags` after the operator's flags, so the private listener and `--no-proxy-headers` win +- `worker_app` wraps the app with `peer.wrap`, which is the identity when forwarding is off diff --git a/gateway/Dockerfile b/gateway/Dockerfile deleted file mode 100644 index 658bed831ba..00000000000 --- a/gateway/Dockerfile +++ /dev/null @@ -1,165 +0,0 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7 -ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a -# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) -ARG PGBOUNCER_VERSION=1.26.0 -ARG PGBOUNCER_SHA256=afd25dd61ee6775d37b40629b87ce08736b3e6955f3057bb212e410fbf21c71d - -FROM $UV_IMAGE AS uvbin - -FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder -ARG PGBOUNCER_VERSION -ARG PGBOUNCER_SHA256 -USER root -RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl -WORKDIR /build -RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \ - echo "${PGBOUNCER_SHA256} pgbouncer.tar.gz" | sha256sum -c - && \ - tar xzf pgbouncer.tar.gz --strip-components=1 && \ - ./configure --prefix=/usr/local --with-openssl=/usr && \ - make -j"$(nproc)" pgbouncer && \ - install -m 0755 pgbouncer /usr/local/bin/pgbouncer - -# ---------- Rust extension ---------- -# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes -FROM $LITELLM_BUILD_IMAGE AS rust-native - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /uvx /usr/local/bin/ - -RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -ENV UV_PYTHON_DOWNLOADS=0 - -COPY pyproject.toml README.md rust-toolchain.toml ./ -COPY .cargo ./.cargo -COPY litellm-rust ./litellm-rust -COPY model_prices_and_context_window.json ./ - -RUN --mount=type=cache,target=/root/.cache/uv \ - mkdir -p litellm/rust_bridge /wheels && \ - touch litellm/__init__.py litellm/rust_bridge/__init__.py && \ - uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \ - python -c "import maturin; maturin.build_wheel('/wheels')" - -# ---------- Builder ---------- -FROM $LITELLM_BUILD_IMAGE AS builder - -WORKDIR /app -USER root - -COPY --from=uvbin /uv /uvx /usr/local/bin/ - -# nodejs/npm so `prisma generate` uses Wolfi's Node via PRISMA_USE_GLOBAL_NODE -# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi -# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes. -RUN for i in 1 2 3; do \ - apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -# UV_COMPILE_BYTECODE=1 precompiles .pyc at install time → faster cold start. -# UV_LINK_MODE=copy avoids hardlink warnings when uv installs from a -# BuildKit cache mount (different filesystem). -# UV_PYTHON_DOWNLOADS=0 force uv to use the apk-installed CPython instead of -# silently pulling a managed interpreter. -# PRISMA_USE_GLOBAL_NODE explicit (matches default) so an env override can't -# silently re-enable nodeenv's Node download. -ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ - UV_LINK_MODE=copy \ - UV_COMPILE_BYTECODE=1 \ - UV_PYTHON_DOWNLOADS=0 \ - PRISMA_USE_GLOBAL_NODE=true \ - PATH="/app/.venv/bin:${PATH}" - -# Stage 1 — install dependencies only. -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=bind,source=pyproject.toml,target=pyproject.toml \ - --mount=type=bind,source=uv.lock,target=uv.lock \ - --mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \ - --mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \ - uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra bedrock-realtime \ - --python python3.13 - -# Stage 2 — copy source and install the workspace members. -COPY . . - -RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-install-project --no-default-groups --no-editable \ - --extra proxy \ - --extra proxy-runtime \ - --group admin-mcp \ - --extra extra_proxy \ - --extra semantic-router \ - --extra bedrock-realtime \ - --python python3.13 - -RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \ - uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl - -# PYTHONPATH=/app makes the source tree shadow the installed package, so the -# compiled Rust extension must live next to the source or it is never imported. -RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/ - -RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ - npm_config_cache=/root/.npm \ - prisma generate --schema=./schema.prisma - -RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh - -# ---------- Runtime ---------- -FROM $LITELLM_RUNTIME_IMAGE AS runtime - -USER root - -RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \ - [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ - sleep 5; \ - done - -# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with -# /home/nonroot. We run the proxy as that user. -WORKDIR /app -ENV HOME=/home/nonroot \ - PATH="/app/.venv/bin:${PATH}" \ - PYTHONPATH="/app" \ - PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries - -COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv -COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm -COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise -COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway -COPY --from=builder --chown=nonroot:nonroot /app/litellm_cli /app/litellm_cli -COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh -COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma -COPY --from=builder /opt/prisma /opt/prisma -COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer - -RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \ - find /app/.venv -type d -path "*/tornado/test" -delete && \ - chmod -R a+rX /opt/prisma && \ - python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \ - python -c "import litellm, litellm.rust_bridge._native; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__" - -USER nonroot - -EXPOSE 4000/tcp - -ENTRYPOINT ["sh", "-c", "exec /app/docker/component_entrypoint.sh python -m gateway.launch --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"] -CMD ["--host", "0.0.0.0", "--port", "4000"] diff --git a/gateway/launch.py b/gateway/launch.py index 59f7121e513..c8b5b53af92 100644 --- a/gateway/launch.py +++ b/gateway/launch.py @@ -22,15 +22,16 @@ import sys from collections.abc import Callable, Mapping, Sequence from typing import TYPE_CHECKING, Final +from typing_extensions import assert_never from uvicorn.main import main as uvicorn_main from litellm_cli.main import without_gc_pauses if TYPE_CHECKING: - from fastapi import FastAPI - from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings + from litellm.proxy.rust_forward.peer import ASGIApp + from litellm.proxy.rust_forward.server import PythonServer GATEWAY_APP: Final = "gateway.launch:worker_app" KEEPALIVE_FLAG: Final = "--timeout-keep-alive" @@ -65,17 +66,40 @@ def pool_database_url( return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth()) -def worker_app() -> "FastAPI": +def worker_app() -> "ASGIApp": with without_gc_pauses(): from gateway.main import app - return app + from litellm.proxy.rust_forward.mode import rust_forward_mode + from litellm.proxy.rust_forward.peer import wrap + return wrap(app, rust_forward_mode(os.environ)) def _serve(argv: Sequence[str]) -> None: uvicorn_main(tuple(argv), prog_name="uvicorn") -def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: +def python_server(argv: Sequence[str]) -> "PythonServer": + """What uvicorn would serve the public port with, read from the same flags and environment it reads.""" + from litellm.proxy.rust_forward.server import PythonServer, Server, UvicornCliParams + from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager + + params: Final = UvicornCliParams.model_validate( + uvicorn_main.make_context("uvicorn", list(uvicorn_argv(argv, os.environ))).params + ) + return PythonServer( + server=Server.UVICORN, + listener=params.listener, + host=params.host, + port=params.port, + keepalive_seconds=params.timeout_keep_alive, + graceful_timeout_seconds=GracefulShutdownManager.get_timeout() + if params.timeout_graceful_shutdown is None + else params.timeout_graceful_shutdown, + trusted_proxies=params.trusted_proxies, + ) + + +def configure_database() -> None: from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PgBouncerError, PgBouncerSettings, export_pooled_database_url @@ -86,8 +110,59 @@ def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) - sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}") if pooled_url is not None: export_pooled_database_url(pooled_url) + + +def _serve_python_alone(argv: Sequence[str], serve: Callable[[Sequence[str]], None]) -> None: + configure_database() serve(uvicorn_argv(argv, os.environ)) +def _python_server_or_exit(argv: Sequence[str]) -> "PythonServer": + """Report ``--help``, ``--version`` and a bad flag the way uvicorn's own CLI does, since it never gets to.""" + import click + + try: + return python_server(argv) + except click.exceptions.Exit as stopped: + sys.exit(stopped.exit_code) + except click.ClickException as error: + error.show() + sys.exit(error.exit_code) + + +def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None: + from pydantic import ValidationError + + from litellm.proxy.rust_forward.mode import ( + Mode, + Supervised, + environment_error, + rust_forward_mode, + stop_with_supervisor, + uvicorn_flags, + ) + from litellm.proxy.rust_forward.supervisor import supervise + from litellm.proxy.rust_forward.supervisor.preflight import StartupError + + try: + mode: Final = rust_forward_mode(os.environ) + except ValidationError as error: + sys.exit(environment_error(error)) + match mode: + case Mode.DISABLED: + _serve_python_alone(argv, serve) + case Mode.SUPERVISOR: + supervised: Final = supervise(_python_server_or_exit(argv), before_start=configure_database) + if isinstance(supervised, StartupError): + _serve_python_alone(argv, serve) + else: + sys.exit(supervised) + case Supervised(): + stop_with_supervisor(mode) + serve((*uvicorn_argv(argv, os.environ), *uvicorn_flags(mode))) + case _: + assert_never(mode) + + if __name__ == "__main__": main(sys.argv[1:]) diff --git a/helm/litellm-helm/.helmignore b/helm/litellm-helm/.helmignore deleted file mode 100644 index 0e8a0eb36f4..00000000000 --- a/helm/litellm-helm/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/helm/litellm-helm/Chart.lock b/helm/litellm-helm/Chart.lock deleted file mode 100644 index 1e63f4f9eca..00000000000 --- a/helm/litellm-helm/Chart.lock +++ /dev/null @@ -1,9 +0,0 @@ -dependencies: -- name: postgresql - repository: oci://registry-1.docker.io/bitnamicharts - version: 14.3.1 -- name: redis - repository: oci://registry-1.docker.io/bitnamicharts - version: 18.19.1 -digest: sha256:38962e231f6596b93f82a8412bbe4cf5de696caecf5775dfbbd163383eb1c009 -generated: '2026-10-30T02:13:11.505726000Z' diff --git a/helm/litellm-helm/Chart.yaml b/helm/litellm-helm/Chart.yaml deleted file mode 100644 index a3cb388ffc6..00000000000 --- a/helm/litellm-helm/Chart.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: v2 - -# We can't call ourselves just "litellm" because then we couldn't publish to the -# same OCI repository as the "litellm" OCI image -name: litellm-helm -description: Call all LLM APIs using the OpenAI format - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.1.3 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: v1.85.1 - -annotations: - org.opencontainers.image.source: "https://github.com/BerriAI/litellm" - org.opencontainers.image.url: "https://docs.litellm.ai/" - -dependencies: - - name: "postgresql" - version: "14.3.1" - repository: oci://registry-1.docker.io/bitnamicharts - condition: db.deployStandalone - - name: redis - version: "18.19.1" - repository: oci://registry-1.docker.io/bitnamicharts - condition: redis.enabled diff --git a/helm/litellm-helm/README.md b/helm/litellm-helm/README.md deleted file mode 100644 index 694cca05b52..00000000000 --- a/helm/litellm-helm/README.md +++ /dev/null @@ -1,265 +0,0 @@ -# Helm Chart for LiteLLM - -> [!IMPORTANT] -> This is community maintained, Please make an issue if you run into a bug -> We recommend using [Docker or Kubernetes for production deployments](https://docs.litellm.ai/docs/proxy/prod) - -## Prerequisites - -- Kubernetes 1.21+ -- Helm 3.8.0+ - -If `db.deployStandalone` is used: - -- PV provisioner support in the underlying infrastructure - -If `db.useStackgresOperator` is used (not yet implemented): - -- The Stackgres Operator must already be installed in the Kubernetes Cluster. This chart will **not** install the operator if it is missing. - -## Parameters - -### LiteLLM Proxy Deployment Settings - -| Name | Description | Value | -| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | -| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` | -| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A | -| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A | -| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated on first install and reused on upgrades. | N/A | -| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | -| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | -| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` | -| `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` | -| `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` | -| `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` | -| `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` | -| `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` | -| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` | -| `livenessProbe.*` | Liveness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` | -| `readinessProbe.*` | Readiness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` | -| `startupProbe.*` | Startup probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay). | See `values.yaml` | -| `resources.*` | CPU/memory requests and limits for the LiteLLM container. Unset by default; production deployments should set 1 CPU and 4Gi of memory per worker. | `{}` | -| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` | -| `ingress.labels` | Additional labels for the Ingress resource | `{}` | -| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A | -| `proxyConfigMap.create` | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it. | `true` | -| `proxyConfigMap.name` | When `create=false`, name of the existing ConfigMap to mount. | `""` | -| `proxyConfigMap.key` | Key in the ConfigMap that contains the proxy config file. | `"config.yaml"` | -| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` | -| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. | -| `pdb.enabled` | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment | `false` | -| `pdb.minAvailable` | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` | -| `pdb.maxUnavailable` | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` | -| `pdb.annotations` | Extra metadata annotations to add to the PDB | `{}` | -| `pdb.labels` | Extra metadata labels to add to the PDB | `{}` | - -| `billingMetrics.enabled` | Enable enterprise billable-request metering. Requires an enterprise license. | `false` | -| `billingMetrics.endpoint` | Collector that the billable-request counter is pushed to. | `https://telemetry.litellm.ai` | -| `billingMetrics.secretName` | Name of an existing Secret holding the mTLS client certificate, under the keys `tls.crt` and `tls.key`. | `litellm-billing-metrics-mtls` | -| `billingMetrics.caSecretName` | Name of an existing Secret holding a CA bundle under the key `ca.crt`. Only needed for a private or test collector whose server certificate is not on the public web PKI. | `""` | -| `billingMetrics.exportIntervalMs` | How often the counter is pushed, in milliseconds. The proxy defaults to `60000` when unset. | `""` | - -#### Example `proxy_config` ConfigMap from values (default): - -``` -proxyConfigMap: - create: true - key: "config.yaml" - -proxy_config: - general_settings: - master_key: os.environ/PROXY_MASTER_KEY - model_list: [] -``` - -The chart ships no models, so add them under `proxy_config.model_list` or from the Admin UI with `store_model_in_db: true` - -``` -proxy_config: - model_list: - - model_name: gpt-5.6 - litellm_params: - model: openai/gpt-5.6 - api_key: os.environ/OPENAI_API_KEY -``` - -#### Example using existing `proxyConfigMap` instead of creating it: - -``` -proxyConfigMap: - create: false - name: my-litellm-config - key: config.yaml - -# proxy_config is ignored in this mode -``` - -#### Example `environmentSecrets` Secret - -``` -apiVersion: v1 -kind: Secret -metadata: - name: litellm-envsecrets -data: - AZURE_OPENAI_API_KEY: TXlTZWN1cmVLM3k= -type: Opaque -``` - -#### Enterprise billable-request metering - -Enterprise licenses meter billable requests by pushing a counter to LiteLLM's collector over mutual TLS. The chart does not create the client certificate; it mounts one you already hold, read-only, so the private key is never exposed through the environment. Create the Secret under the name the chart expects, then turn the block on: - -``` -kubectl create secret tls litellm-billing-metrics-mtls --cert=client.crt --key=client.key -``` - -``` -billingMetrics: - enabled: true -``` - -Set `billingMetrics.caSecretName` only when the collector is a private or test one whose server certificate is not on the public web PKI; the production collector needs no CA override. The chart fails the render rather than deploying a proxy that silently never exports, so a missing `secretName` or an emptied `endpoint` surfaces at `helm install` time. - -### Database Settings - -| Name | Description | Value | -| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | -| `db.useExisting` | Use an existing Postgres database. A Kubernetes Secret object must exist that contains credentials for connecting to the database. An example secret object definition is provided below. | `false` | -| `db.endpoint` | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to. | `localhost` | -| `db.database` | If `db.useExisting` is `true`, the name of the existing database to connect to. | `litellm` | -| `db.url` | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value. | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)` | -| `db.secret.name` | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials. | `postgres` | -| `db.secret.usernameKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance. | `username` | -| `db.secret.passwordKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user. | `password` | -| `db.username` | If `db.useExisting` is `true`, the database user as a plain value. When set, `db.secret.usernameKey` is not read. | `""` | -| `db.port` | If `db.useExisting` is `true` with token auth, the Postgres port used to build the connection URL. Password auth takes the port from `db.url`. | `5432` | -| `db.schema` | With token auth, the Postgres schema holding the LiteLLM tables, rendered as `DATABASE_SCHEMA` when the tables are not in `public`. Ignored with password auth, where the schema stays in `db.url` as `?schema=`. | `""` | -| `db.useIAMAuth` | If `db.useExisting` is `true` and `db.deployStandalone` is `false`, authenticate with an AWS RDS IAM token instead of a password. See [Token auth](#token-auth-to-an-existing-postgres-rds-iam-or-microsoft-entra-id). | `false` | -| `db.useAzureEntraAuth` | If `db.useExisting` is `true` and `db.deployStandalone` is `false`, authenticate with a Microsoft Entra ID token instead of a password. Mutually exclusive with `db.useIAMAuth`. | `false` | -| `db.useStackgresOperator` | Not yet implemented. | `false` | -| `db.deployStandalone` | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups. | `true` | -| `postgresql.*` | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml) | -| `postgresql.auth.*` | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used. | `NoTaGrEaTpAsSwOrD` | -| `postgresql.image.*` | If `db.deployStandalone` is `true`, the image for the bundled Postgres. Pinned to a `docker.io/bitnamilegacy` build because Bitnami retired the versioned tags under `docker.io/bitnami`. | `bitnamilegacy/postgresql:16.2.0-debian-12-r6` | -| `redis.image.*` | If `redis.enabled` is `true`, the image for the bundled Redis. Pinned to a `docker.io/bitnamilegacy` build for the same reason. | `bitnamilegacy/redis:7.2.4-debian-12-r9` | - -#### Bundled Postgres image - -Bitnami removed the versioned tags from `docker.io/bitnami` and republished the archived builds under `docker.io/bitnamilegacy`, so the image defaults that ship inside the `postgresql` and `redis` subcharts no longer pull. The chart pins both to the `bitnamilegacy` copies of the exact builds those subchart versions were released with, which keeps the on-disk data directory layout unchanged for existing installs. - -Keep `postgresql.image.tag` pinned. `docker.io/bitnami/postgresql` still publishes a floating `latest`, and pointing the bundled Postgres at a different major version starts the server against a data directory it cannot read (`database files are incompatible with server`). There is no in-place way back, so crossing a major version means dumping the database with the old image and restoring it into the new one. The chart refuses to render when the tag is empty or `latest`. - -Those images no longer receive updates. For anything beyond getting started, run Postgres outside the chart and point at it with `db.useExisting`. - -#### Example Postgres `db.useExisting` Secret - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: postgres -data: - # Password for the "postgres" user - postgres-password: - username: litellm - password: -type: Opaque -``` - -#### Token auth to an existing Postgres (RDS IAM or Microsoft Entra ID) - -When the database accepts short-lived tokens instead of a password (AWS RDS or Aurora with IAM database authentication, or Azure Database for PostgreSQL with Microsoft Entra ID), there is no password to put in a Secret. Turn on `db.useIAMAuth` or `db.useAzureEntraAuth` and the chart stops referencing `db.secret.passwordKey` and `db.url`. It sets `IAM_TOKEN_DB_AUTH` (or `AZURE_POSTGRESQL_AUTH`) and emits `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_USERNAME` and `DATABASE_NAME` for the proxy, the migrations job and the collector sidecar. Each of them mints its own token at startup and builds `DATABASE_URL` from those parts, and the in-container pgbouncer (`db.connectionPool`) refreshes the token before it expires. - -```yaml -db: - useExisting: true - deployStandalone: false - useIAMAuth: true - endpoint: litellm.cluster-abc123.us-west-1.rds.amazonaws.com - database: litellm - username: litellm_iam # the role granted rds_iam; no db.secret needed -``` - -The pod needs cloud credentials for the token: on EKS use IRSA or Pod Identity on the chart's `serviceAccount` (and `migrationJob.serviceAccountName` when the Helm hook runs before the ServiceAccount exists), on AKS use workload identity. Set `AWS_RDS_REGION` under `envVars` when the RDS hostname is not a standard regional endpoint (the token must be signed for the region the database lives in). TLS options that would otherwise go in `db.url` are set as `DATABASE_SSLMODE` and `DATABASE_SSLROOTCERT` under `envVars`. Leaving `db.username` empty reads the user from `db.secret.usernameKey` as before. - -#### Examples for `environmentSecrets` and `environemntConfigMaps` - -```yaml -# Use config map for not-secret configuration data -apiVersion: v1 -kind: ConfigMap -metadata: - name: litellm-env-configmap -data: - SOME_KEY: someValue - ANOTHER_KEY: anotherValue -``` - -```yaml -# Use secrets for things which are actually secret like API keys, credentials, etc -# Base64 encode the values stored in a Kubernetes Secret: $ pbpaste | base64 | pbcopy -# The --decode flag is convenient: $ pbpaste | base64 --decode - -apiVersion: v1 -kind: Secret -metadata: - name: litellm-env-secret -type: Opaque -data: - SOME_PASSWORD: cDZbUGVXeU5e0ZW # base64 encoded - ANOTHER_PASSWORD: AAZbUGVXeU5e0ZB # base64 encoded -``` - -Source: [GitHub Gist from troyharvey](https://gist.github.com/troyharvey/4506472732157221e04c6b15e3b3f094) - -### Migration Job Settings - -The migration job supports both ArgoCD and Helm hooks to ensure database migrations run at the appropriate time during deployments. - -| Name | Description | Value | -| -------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------- | -| `migrationJob.enabled` | Enable or disable the schema migration Job | `true` | -| `migrationJob.backoffLimit` | Backoff limit for Job restarts | `4` | -| `migrationJob.ttlSecondsAfterFinished` | TTL for completed migration jobs | `120` | -| `migrationJob.annotations` | Additional annotations for the migration job pod | `{}` | -| `migrationJob.extraContainers` | Additional containers to run alongside the migration job | `[]` | -| `migrationJob.hooks.argocd.enabled` | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy) | `true` | -| `migrationJob.hooks.helm.enabled` | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy) | `false` | -| `migrationJob.hooks.helm.weight` | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified. | N/A | - -## Accessing the Admin UI - -When browsing to the URL published per the settings in `ingress.*`, you will -be prompted for **Admin Configuration**. The **Proxy Endpoint** is the internal -(from the `litellm` pod's perspective) URL published by the `-litellm` -Kubernetes Service. If the deployment uses the default settings for this -service, the **Proxy Endpoint** should be set to `http://-litellm:4000`. - -The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey` -was not provided to the helm command line, the `masterkey` is a randomly -generated string in the `sk-...` format stored in the `-litellm-masterkey` Kubernetes Secret. -The key is generated once on the first install; later `helm upgrade` runs reuse the -value already in that Secret, so upgrading never rotates the master key. - -```bash -kubectl -n litellm get secret -litellm-masterkey -o jsonpath="{.data.masterkey}" -``` - -## Admin UI Limitations - -At the time of writing, the Admin UI is unable to add models. This is because -it would need to update the `config.yaml` file which is a exposed ConfigMap, and -therefore, read-only. This is a limitation of this helm chart, not the Admin UI -itself. - -## Connect Lens - -This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render - -## Shutdown accounting - -`GRACEFUL_SHUTDOWN_TIMEOUT` defaults to 30 seconds. Pending success callbacks share the time remaining since SIGTERM or the first `/health/drain` call. Callback flushing reserves the existing 10-second teardown headroom for cancellation, tracked accounting tasks and persistence. Exhausting this budget can discard callbacks, which is logged - -Set `terminationGracePeriodSeconds` to at least `GRACEFUL_SHUTDOWN_TIMEOUT + 10`, allowing additional time for preStop hooks and database writes. This is not a hard shutdown bound or a guarantee of persistence. Database retries retain their own limits, and cancellation-resistant callbacks can remain alive. The existing preStop hook can also be followed by Uvicorn's separate connection drain, as described in `values.yaml` diff --git a/helm/litellm-helm/charts/postgresql-14.3.1.tgz b/helm/litellm-helm/charts/postgresql-14.3.1.tgz deleted file mode 100644 index e8e2fac0fda..00000000000 Binary files a/helm/litellm-helm/charts/postgresql-14.3.1.tgz and /dev/null differ diff --git a/helm/litellm-helm/charts/redis-18.19.1.tgz b/helm/litellm-helm/charts/redis-18.19.1.tgz deleted file mode 100644 index 4a55a980088..00000000000 Binary files a/helm/litellm-helm/charts/redis-18.19.1.tgz and /dev/null differ diff --git a/helm/litellm-helm/ci/test-values.yaml b/helm/litellm-helm/ci/test-values.yaml deleted file mode 100644 index 33a4df942ee..00000000000 --- a/helm/litellm-helm/ci/test-values.yaml +++ /dev/null @@ -1,15 +0,0 @@ -fullnameOverride: "" -# Disable database deployment and configuration -db: - deployStandalone: false - useExisting: false - -# Test environment variables -envVars: - DD_ENV: "dev_helm" - DD_SERVICE: "litellm" - USE_DDTRACE: "true" - -# Disable migration job since we're not using a database -migrationJob: - enabled: false \ No newline at end of file diff --git a/helm/litellm-helm/templates/NOTES.txt b/helm/litellm-helm/templates/NOTES.txt deleted file mode 100644 index 017bbfa78bd..00000000000 --- a/helm/litellm-helm/templates/NOTES.txt +++ /dev/null @@ -1,23 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "litellm.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "litellm.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "litellm.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "litellm.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} -PDB: {{ if .Values.pdb.enabled }}enabled{{ else }}disabled{{ end }}. Configure via .Values.pdb.* \ No newline at end of file diff --git a/helm/litellm-helm/templates/_helpers.tpl b/helm/litellm-helm/templates/_helpers.tpl deleted file mode 100644 index 929fc415163..00000000000 --- a/helm/litellm-helm/templates/_helpers.tpl +++ /dev/null @@ -1,425 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "litellm.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "litellm.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "litellm.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "litellm.labels" -}} -helm.sh/chart: {{ include "litellm.chart" . }} -{{ include "litellm.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "litellm.selectorLabels" -}} -app.kubernetes.io/name: {{ include "litellm.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Enterprise billable-request metering. The client certificate identifies the -deployment to LiteLLM's collector, so it is mounted read-only from an existing -Secret rather than passed through the environment. -*/}} -{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}} -{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}} - -{{- define "litellm.billingMetricsEnv" -}} -- name: LITELLM_BILLING_METRICS_ENDPOINT - value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }} -- name: LITELLM_BILLING_METRICS_CLIENT_CERT - value: {{ printf "%s/tls.crt" (include "litellm.billingMetrics.certDir" .) | quote }} -- name: LITELLM_BILLING_METRICS_CLIENT_KEY - value: {{ printf "%s/tls.key" (include "litellm.billingMetrics.certDir" .) | quote }} -{{- if .Values.billingMetrics.caSecretName }} -- name: LITELLM_BILLING_METRICS_CA_CERT - value: {{ printf "%s/ca.crt" (include "litellm.billingMetrics.caDir" .) | quote }} -{{- end }} -{{- with .Values.billingMetrics.exportIntervalMs }} -- name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS - value: {{ . | quote }} -{{- end }} -{{- end -}} - -{{- define "litellm.billingMetricsVolumes" -}} -- name: billing-metrics-mtls - secret: - secretName: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)" .Values.billingMetrics.secretName }} -{{- if .Values.billingMetrics.caSecretName }} -- name: billing-metrics-mtls-ca - secret: - secretName: {{ .Values.billingMetrics.caSecretName }} -{{- end }} -{{- end -}} - -{{- define "litellm.billingMetricsVolumeMounts" -}} -- name: billing-metrics-mtls - mountPath: {{ include "litellm.billingMetrics.certDir" . }} - readOnly: true -{{- if .Values.billingMetrics.caSecretName }} -- name: billing-metrics-mtls-ca - mountPath: {{ include "litellm.billingMetrics.caDir" . }} - readOnly: true -{{- end }} -{{- end -}} - -{{/* -Create the name of the service account to use -*/}} -{{- define "litellm.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "litellm.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - -{{/* -Create the service account name used by migration jobs. -When Helm hooks are enabled, pre-install/pre-upgrade hooks run before normal resources. -If this chart is creating the ServiceAccount, it is not yet available for the hook job, -so fall back to "default" (or an explicit override) to avoid a cyclic dependency. -*/}} -{{- define "litellm.migrationServiceAccountName" -}} -{{- if and .Values.migrationJob.hooks.helm.enabled .Values.serviceAccount.create }} -{{- default "default" .Values.migrationJob.serviceAccountName }} -{{- else }} -{{- include "litellm.serviceAccountName" . }} -{{- end }} -{{- end }} - -{{/* -Get redis service name. -The bundled Redis subchart only serves sentinel in "replication" architecture -(it rejects standalone + sentinel outright), and in that mode the sentinel -Service is named "-redis", not "-redis-master". -*/}} -{{- define "litellm.redis.serviceName" -}} -{{- if .Values.redis.sentinel.enabled -}} -{{- printf "%s-%s" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}} -{{- else -}} -{{- printf "%s-%s-master" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}} -{{- end -}} -{{- end -}} - -{{/* -Get redis service port -*/}} -{{- define "litellm.redis.port" -}} -{{- if .Values.redis.sentinel.enabled -}} -{{ .Values.redis.sentinel.service.ports.sentinel }} -{{- else -}} -{{ .Values.redis.master.service.ports.redis }} -{{- end -}} -{{- end -}} - -{{/* -Reject an unpinned image tag for the bundled PostgreSQL. -A floating tag lets a chart upgrade start a newer PostgreSQL major against the -existing PersistentVolumeClaim. The server then refuses to start on a data -directory written by another major version, and the only way back is a dump -taken before the change, which by that point no longer exists. -*/}} -{{- define "litellm.validateBundledPostgresImageTag" -}} -{{- $tag := .Values.postgresql.image.tag | default "" | toString -}} -{{- $digest := .Values.postgresql.image.digest | default "" | toString -}} -{{- if and (eq $digest "") (or (eq $tag "") (eq $tag "latest")) -}} -{{- fail (printf "postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got %q). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore." $tag) -}} -{{- end -}} -{{- end -}} - -{{/* -Whether the existing Postgres is reached with a short-lived token (AWS RDS IAM -or Microsoft Entra ID) instead of a password. -*/}} -{{- define "litellm.db.tokenAuth" -}} -{{- if or .Values.db.useIAMAuth .Values.db.useAzureEntraAuth -}}true{{- end -}} -{{- end -}} - -{{- define "litellm.validateDbTokenAuth" -}} -{{- if and .Values.db.useIAMAuth .Values.db.useAzureEntraAuth -}} -{{- fail "db.useIAMAuth and db.useAzureEntraAuth are mutually exclusive: the database password can only come from one token source" -}} -{{- end -}} -{{- if and (include "litellm.db.tokenAuth" .) (or (not .Values.db.useExisting) .Values.db.deployStandalone) -}} -{{- fail "db.useIAMAuth and db.useAzureEntraAuth need db.useExisting: true and db.deployStandalone: false (token auth targets an existing RDS or Azure Database for PostgreSQL; the bundled postgresql subchart only supports password auth)" -}} -{{- end -}} -{{- end -}} - -{{/* -Database env for an existing Postgres, shared by the proxy container, the -collector sidecar and the migrations Job. Under token auth every entrypoint -mints its own token and assembles DATABASE_URL itself, so db.url is left out. -*/}} -{{- define "litellm.existingDbEnv" -}} -{{- $tokenAuth := include "litellm.db.tokenAuth" . -}} -- name: DATABASE_USERNAME - {{- if .Values.db.username }} - value: {{ .Values.db.username | quote }} - {{- else }} - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.usernameKey }} - {{- end }} -{{- if .Values.db.useIAMAuth }} -- name: IAM_TOKEN_DB_AUTH - value: "true" -{{- else if .Values.db.useAzureEntraAuth }} -- name: AZURE_POSTGRESQL_AUTH - value: "true" -{{- else }} -- name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.passwordKey }} -{{- end }} -- name: DATABASE_HOST - {{- if .Values.db.secret.endpointKey }} - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.endpointKey }} - {{- else }} - value: {{ .Values.db.endpoint }} - {{- end }} -{{- if $tokenAuth }} -- name: DATABASE_PORT - value: {{ .Values.db.port | quote }} -{{- end }} -- name: DATABASE_NAME - value: {{ .Values.db.database }} -{{- if and $tokenAuth .Values.db.schema }} -- name: DATABASE_SCHEMA - value: {{ .Values.db.schema | quote }} -{{- end }} -{{- if not $tokenAuth }} -- name: DATABASE_URL - value: {{ .Values.db.url | quote }} -{{- end }} -{{- end -}} - -{{/* -Environment shared by the proxy container and the opt-in collector sidecar: -database, pgbouncer, master key, redis, user envVars. Both containers must see -the same DATABASE_URL and REDIS_* so the sidecar reaches the pod's pgbouncer -and the same spend transaction buffer. -*/}} -{{- define "litellm.proxyEnv" -}} -{{- include "litellm.validateDbTokenAuth" . -}} -- name: HOST - value: "{{ .Values.listen | default "0.0.0.0" }}" -- name: PORT - value: {{ .Values.service.port | quote}} -{{- if .Values.db.deployStandalone }} -- name: DATABASE_USERNAME - valueFrom: - secretKeyRef: - name: {{ include "litellm.fullname" . }}-dbcredentials - key: username -- name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "litellm.fullname" . }}-dbcredentials - key: password -- name: DATABASE_HOST - value: {{ .Release.Name }}-postgresql -- name: DATABASE_NAME - value: litellm -{{- else if .Values.db.useExisting }} -{{ include "litellm.existingDbEnv" . }} -{{- end }} -{{- if and .Values.db.useExisting .Values.db.readReplicaUrl .Values.db.secret.readReplicaEndpointKey (not .Values.db.secret.readReplicaUrlKey) }} -- name: DATABASE_READER_HOST - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.readReplicaEndpointKey }} -{{- end }} -{{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }} -- name: DATABASE_URL_READ_REPLICA - valueFrom: - secretKeyRef: - name: {{ .Values.db.secret.name }} - key: {{ .Values.db.secret.readReplicaUrlKey }} -{{- else if .Values.db.readReplicaUrl }} -- name: DATABASE_URL_READ_REPLICA - value: {{ .Values.db.readReplicaUrl | quote }} -{{- end }} -{{- if .Values.db.connectionPool.enabled }} -- name: LITELLM_PGBOUNCER_ENABLED - value: "true" -- name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - value: {{ .Values.db.connectionPool.maxDbConnections | quote }} -- name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN - value: {{ .Values.db.connectionPool.maxClientConn | quote }} -{{- end }} -- name: PROXY_MASTER_KEY - valueFrom: - secretKeyRef: - name: {{ .Values.masterkeySecretName | default (printf "%s-masterkey" (include "litellm.fullname" .)) }} - key: {{ .Values.masterkeySecretKey | default "masterkey" }} -{{- if .Values.redis.enabled }} -- name: REDIS_HOST - value: {{ include "litellm.redis.serviceName" . }} -- name: REDIS_PORT - value: {{ include "litellm.redis.port" . | quote }} -- name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "redis.secretName" .Subcharts.redis }} - key: {{include "redis.secretPasswordKey" .Subcharts.redis }} -{{- end }} -{{- /* - Inject LITELLM_LOG only when envVars does not already define it. -*/}} -{{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }} -- name: LITELLM_LOG - value: {{ .Values.logLevel | quote }} -{{- end }} -{{- if .Values.envVars }} -{{- range $key, $val := .Values.envVars }} -- name: {{ $key }} - value: {{ $val | quote }} -{{- end }} -{{- end }} -{{- with .Values.extraEnvVars }} -{{ toYaml . }} -{{- end }} -{{- if .Values.migrationJob.enabled }} -# Schema updates are owned by the dedicated migrations Job; skip -# the proxy's startup `prisma db push` so N replicas don't race -# one DB on every rollout. Placed last (after envVars and -# extraEnvVars) so this override can't be silently shadowed by a -# user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env -# semantics — same pattern the migrations Job uses. -- name: DISABLE_SCHEMA_UPDATE - value: "true" -{{- end }} -{{- end -}} - -{{/* -Proxy-only metering and metrics env. The collector sidecar serves no HTTP -traffic, so it gets neither. -*/}} -{{- define "litellm.proxyMetricsEnv" -}} -{{- if .Values.billingMetrics.enabled }} -{{ include "litellm.billingMetricsEnv" . }} -{{- end }} -{{- if .Values.metricsServer.enabled }} -{{- if eq (int .Values.metricsServer.port) (int .Values.service.port) }} -{{- fail "metricsServer.port must differ from service.port" }} -{{- end }} -- name: PROMETHEUS_METRICS_PORT - value: {{ .Values.metricsServer.port | quote }} -{{- end }} -{{- end -}} - -{{/* -Directory of the collector's unix socket, shared between the two containers -through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP. -*/}} -{{- define "litellm.collector.socketDir" -}} -{{- if and .Values.collector.enabled (hasPrefix "unix://" .Values.collector.address) -}} -{{- dir (trimPrefix "unix://" .Values.collector.address) -}} -{{- end -}} -{{- end -}} - -{{- define "litellm.collectorEnv" -}} -- name: LITELLM_COLLECTOR_ENABLED - value: "true" -- name: LITELLM_COLLECTOR_ADDRESS - value: {{ .Values.collector.address | quote }} -- name: LITELLM_COLLECTOR_BUFFER_SIZE - value: {{ .Values.collector.bufferSize | quote }} -- name: LITELLM_COLLECTOR_ON_UNAVAILABLE - value: {{ .Values.collector.onUnavailable | quote }} -- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS - value: {{ .Values.collector.drainTimeoutSeconds | quote }} -{{- end -}} - -{{- define "litellm.gateway.collectorSocketDir" -}} -{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}} -{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}} -{{- end -}} -{{- end -}} - -{{/* -LITELLM_COLLECTOR_* env shared by the producer (gateway container) and the -consumer (collector container), so both agree on the transport and the -shutdown drain window. -*/}} -{{- define "litellm.gateway.collectorEnv" -}} -{{- with .Values.gateway.collector }} -- name: LITELLM_COLLECTOR_ENABLED - value: "true" -- name: LITELLM_COLLECTOR_ADDRESS - value: {{ .address | quote }} -- name: LITELLM_COLLECTOR_BUFFER_SIZE - value: {{ .bufferSize | quote }} -- name: LITELLM_COLLECTOR_ON_UNAVAILABLE - value: {{ .onUnavailable | quote }} -- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS - value: {{ .drainTimeoutSeconds | quote }} -{{- end }} -{{- end -}} - -{{- define "litellm.lensConnectionEnv" -}} -{{- with .Values.lensWorker }} -{{- if or .enabled (and .mode (ne .mode "disabled")) }} -{{- fail "lensWorker was removed; deploy Lens from its own chart and set lens.* instead, see the chart README" }} -{{- end }} -{{- end }} -{{- with .Values.lens }} -{{- if .enabled }} -- name: LITELLM_LENS_URL - value: {{ required "lens.url is required when lens.enabled is true" .url | quote }} -- name: LITELLM_LENS_PUBLIC_URL - value: {{ required "lens.publicUrl is required when lens.enabled is true" .publicUrl | quote }} -- name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: - name: {{ required "lens.gatewaySecret.name is required when lens.enabled is true" .gatewaySecret.name | quote }} - key: {{ .gatewaySecret.key | quote }} -- name: LITELLM_LENS_SERVICE_TOKEN - valueFrom: - secretKeyRef: - name: {{ required "lens.serviceTokenSecret.name is required when lens.enabled is true" .serviceTokenSecret.name | quote }} - key: {{ .serviceTokenSecret.key | quote }} -{{- end }} -{{- end }} -{{- end -}} diff --git a/helm/litellm-helm/templates/configmap-litellm.yaml b/helm/litellm-helm/templates/configmap-litellm.yaml deleted file mode 100644 index 03e4f620206..00000000000 --- a/helm/litellm-helm/templates/configmap-litellm.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- if .Values.proxyConfigMap.create }} -{{- $config := deepCopy .Values.proxy_config }} -{{- if and .Values.redis.enabled (dig "coordination" "enabled" true .Values.redis) }} -{{- $generalSettings := (get $config "general_settings") | default dict }} -{{- if not (hasKey $generalSettings "coordination_redis") }} -{{- $coordinationRedis := dict "host" "os.environ/REDIS_HOST" "port" "os.environ/REDIS_PORT" "password" "os.environ/REDIS_PASSWORD" }} -{{- if .Values.redis.sentinel.enabled }} -{{- $sentinelNode := list (include "litellm.redis.serviceName" .) (include "litellm.redis.port" . | int) }} -{{- $coordinationRedis = dict "sentinel_nodes" (list $sentinelNode) "service_name" (default "mymaster" .Values.redis.sentinel.masterSet) "password" "os.environ/REDIS_PASSWORD" }} -{{- end }} -{{- $_ := set $generalSettings "coordination_redis" $coordinationRedis }} -{{- $_ := set $config "general_settings" $generalSettings }} -{{- end }} -{{- end }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "litellm.fullname" . }}-config -data: - config.yaml: | -{{ $config | toYaml | indent 6 }} -{{- end }} diff --git a/helm/litellm-helm/templates/deployment.yaml b/helm/litellm-helm/templates/deployment.yaml deleted file mode 100644 index 37e7eaec9e9..00000000000 --- a/helm/litellm-helm/templates/deployment.yaml +++ /dev/null @@ -1,264 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - {{- toYaml .Values.deploymentAnnotations | nindent 4 }} - name: {{ include "litellm.fullname" . }} - labels: - {{- include "litellm.labels" . | nindent 4 }} - {{- if .Values.deploymentLabels }} - {{- toYaml .Values.deploymentLabels | nindent 4 }} - {{- end }} -spec: - {{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }} - replicas: {{ .Values.replicaCount }} - {{- end }} - {{- with .Values.strategy }} - strategy: - {{- toYaml . | nindent 4 }} - {{- end }} - selector: - matchLabels: - {{- include "litellm.selectorLabels" . | nindent 6 }} - {{- if .Values.deploymentMinReadySeconds }} - minReadySeconds: {{ .Values.deploymentMinReadySeconds }} - {{- end }} - template: - metadata: - annotations: - {{- if .Values.proxyConfigMap.create }} - checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }} - {{- end }} - {{- with .Values.podAnnotations }} - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - labels: - {{- include "litellm.labels" . | nindent 8 }} - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "litellm.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - {{- with .Values.extraInitContainers }} - initContainers: - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - containers: - - name: {{ include "litellm.name" . }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - {{- include "litellm.lensConnectionEnv" . | nindent 12 }} - {{- include "litellm.proxyEnv" . | nindent 12 }} - {{- if .Values.liteadmin.enabled }} - - name: LITELLM_ADMIN_AGENT_URL - value: {{ printf "http://%s-liteadmin:10000" (include "litellm.fullname" . | trunc 53 | trimSuffix "-") | quote }} - - name: ADMIN_AGENT_SERVICE_TOKEN - valueFrom: - secretKeyRef: - name: {{ required "liteadmin.existingSecret is required" .Values.liteadmin.existingSecret }} - key: ADMIN_AGENT_SERVICE_TOKEN - {{- if not (hasKey (default dict .Values.envVars) "PROXY_BASE_URL") }} - - name: PROXY_BASE_URL - value: {{ required "liteadmin.gatewayUrl is required" .Values.liteadmin.gatewayUrl | quote }} - {{- end }} - {{- end }} - {{- include "litellm.proxyMetricsEnv" . | nindent 12 }} - {{- if .Values.collector.enabled }} - {{- include "litellm.collectorEnv" . | nindent 12 }} - {{- end }} - envFrom: - {{- range .Values.environmentSecrets }} - - secretRef: - name: {{ . }} - {{- end }} - {{- range .Values.environmentConfigMaps }} - - configMapRef: - name: {{ . }} - {{- end }} - {{- if .Values.command }} - command: {{ toYaml .Values.command | nindent 12 }} - {{- end }} - {{- if .Values.args }} - args: {{ toYaml .Values.args | nindent 12 }} - {{- else }} - args: - - --config - - /etc/litellm/config.yaml - {{ if .Values.numWorkers }} - - --num_workers - - {{ .Values.numWorkers | quote }} - {{- end }} - {{- end }} - ports: - - name: http - containerPort: {{ .Values.service.port }} - protocol: TCP - {{- if .Values.metricsServer.enabled }} - - name: metrics - containerPort: {{ .Values.metricsServer.port }} - protocol: TCP - {{- end }} - livenessProbe: - httpGet: - path: {{ .Values.livenessProbe.path | quote }} - port: "http" - initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.livenessProbe.periodSeconds }} - timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} - successThreshold: {{ .Values.livenessProbe.successThreshold }} - failureThreshold: {{ .Values.livenessProbe.failureThreshold }} - readinessProbe: - httpGet: - path: {{ .Values.readinessProbe.path | quote }} - port: "http" - initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.readinessProbe.periodSeconds }} - timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} - successThreshold: {{ .Values.readinessProbe.successThreshold }} - failureThreshold: {{ .Values.readinessProbe.failureThreshold }} - startupProbe: - httpGet: - path: {{ .Values.startupProbe.path | quote }} - port: "http" - initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.startupProbe.periodSeconds }} - timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }} - successThreshold: {{ .Values.startupProbe.successThreshold }} - failureThreshold: {{ .Values.startupProbe.failureThreshold }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - name: litellm-config - mountPath: /etc/litellm/config.yaml - subPath: config.yaml - {{ if .Values.securityContext.readOnlyRootFilesystem }} - - name: tmp - mountPath: /tmp - - name: cache - mountPath: /.cache - - name: npm - mountPath: /.npm - {{- end }} - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }} - {{- end }} - {{- if include "litellm.collector.socketDir" . }} - - name: collector-socket - mountPath: {{ include "litellm.collector.socketDir" . }} - {{- end }} - {{- with .Values.volumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.lifecycle }} - lifecycle: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- if .Values.collector.enabled }} - - name: {{ include "litellm.name" . }}-collector - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: {{ toYaml .Values.collector.command | nindent 12 }} - env: - {{- include "litellm.proxyEnv" . | nindent 12 }} - {{- include "litellm.collectorEnv" . | nindent 12 }} - - name: LITELLM_JOB_ROLE - value: collector - {{- if not (hasKey (default dict .Values.envVars) "CONFIG_FILE_PATH") }} - - name: CONFIG_FILE_PATH - value: /etc/litellm/config.yaml - {{- end }} - envFrom: - {{- range .Values.environmentSecrets }} - - secretRef: - name: {{ . }} - {{- end }} - {{- range .Values.environmentConfigMaps }} - - configMapRef: - name: {{ . }} - {{- end }} - resources: - {{- toYaml .Values.collector.resources | nindent 12 }} - volumeMounts: - - name: litellm-config - mountPath: /etc/litellm/config.yaml - subPath: config.yaml - {{- if include "litellm.collector.socketDir" . }} - - name: collector-socket - mountPath: {{ include "litellm.collector.socketDir" . }} - {{- end }} - {{ if .Values.securityContext.readOnlyRootFilesystem }} - - name: tmp - mountPath: /tmp - - name: cache - mountPath: /.cache - - name: npm - mountPath: /.npm - {{- end }} - {{- with .Values.volumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- end }} - {{- with .Values.extraContainers }} - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - volumes: - {{ if .Values.securityContext.readOnlyRootFilesystem }} - - name: tmp - emptyDir: - sizeLimit: 500Mi - - name: cache - emptyDir: - sizeLimit: 500Mi - - name: npm - emptyDir: - sizeLimit: 500Mi - {{- end }} - - name: litellm-config - configMap: - {{- if .Values.proxyConfigMap.create }} - name: {{ include "litellm.fullname" . }}-config - {{- else }} - name: {{ .Values.proxyConfigMap.name }} - {{- end }} - items: - - key: {{ .Values.proxyConfigMap.key | default "config.yaml" }} - path: "config.yaml" - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsVolumes" . | nindent 8 }} - {{- end }} - {{- if include "litellm.collector.socketDir" . }} - - name: collector-socket - emptyDir: - sizeLimit: 1Mi - {{- end }} - {{- with .Values.volumes }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds | default 90 }} - {{- if .Values.topologySpreadConstraints }} - topologySpreadConstraints: - {{- toYaml .Values.topologySpreadConstraints | nindent 8 }} - {{- end }} diff --git a/helm/litellm-helm/templates/extra-resources.yaml b/helm/litellm-helm/templates/extra-resources.yaml deleted file mode 100644 index 33190d96fc0..00000000000 --- a/helm/litellm-helm/templates/extra-resources.yaml +++ /dev/null @@ -1,6 +0,0 @@ -{{- if .Values.extraResources }} -{{- range .Values.extraResources }} ---- -{{ toYaml . | nindent 0 }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/helm/litellm-helm/templates/hpa.yaml b/helm/litellm-helm/templates/hpa.yaml deleted file mode 100644 index a651f916d21..00000000000 --- a/helm/litellm-helm/templates/hpa.yaml +++ /dev/null @@ -1,64 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "litellm.fullname" . }} - labels: - {{- include "litellm.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "litellm.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - {{- if .Values.autoscaling.behavior }} - behavior: - {{- toYaml .Values.autoscaling.behavior | nindent 4 }} - {{- end }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- if and .Values.collector.enabled .Values.collector.scaleOnProxyContainerCpu }} - - type: ContainerResource - containerResource: - name: cpu - container: {{ include "litellm.name" . }} - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- else }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} - {{- with .Values.autoscaling.targetRequestsPerSecond }} - - type: Pods - pods: - metric: - name: litellm_requests_per_second - target: - type: AverageValue - averageValue: {{ toJson . | trimAll "\"" | quote }} - {{- end }} - {{- with .Values.autoscaling.targetTokensPerSecond }} - - type: Pods - pods: - metric: - name: litellm_tokens_per_second - target: - type: AverageValue - averageValue: {{ toJson . | trimAll "\"" | quote }} - {{- end }} -{{- end }} diff --git a/helm/litellm-helm/templates/ingress.yaml b/helm/litellm-helm/templates/ingress.yaml deleted file mode 100644 index 3d6e3f4af7a..00000000000 --- a/helm/litellm-helm/templates/ingress.yaml +++ /dev/null @@ -1,78 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "litellm.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "litellm.labels" . | nindent 4 }} - {{- with .Values.ingress.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- if and $.Values.lens.enabled $.Values.lens.service.name }} - - path: /lens-ingest - pathType: Prefix - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $.Values.lens.service.name }} - port: - number: {{ $.Values.lens.service.port }} - {{- else }} - serviceName: {{ $.Values.lens.service.name }} - servicePort: {{ $.Values.lens.service.port }} - {{- end }} - {{- end }} - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/litellm-helm/templates/keda.yaml b/helm/litellm-helm/templates/keda.yaml deleted file mode 100644 index bf585d0d4be..00000000000 --- a/helm/litellm-helm/templates/keda.yaml +++ /dev/null @@ -1,58 +0,0 @@ -{{- if and .Values.keda.enabled (not .Values.autoscaling.enabled) }} -apiVersion: keda.sh/v1alpha1 -kind: ScaledObject -metadata: - name: {{ include "litellm.fullname" . }} - labels: - {{- include "litellm.labels" . | nindent 4 }} - {{- if .Values.keda.scaledObject.annotations }} - annotations: {{ toYaml .Values.keda.scaledObject.annotations | nindent 4 }} - {{- end }} -spec: - scaleTargetRef: - name: {{ include "litellm.fullname" . }} - pollingInterval: {{ .Values.keda.pollingInterval }} - cooldownPeriod: {{ .Values.keda.cooldownPeriod }} - minReplicaCount: {{ .Values.keda.minReplicas }} - maxReplicaCount: {{ .Values.keda.maxReplicas }} -{{- with .Values.keda.fallback }} - fallback: - failureThreshold: {{ .failureThreshold | default 3 }} - replicas: {{ .replicas | default $.Values.keda.maxReplicas }} -{{- end }} - triggers: -{{- with .Values.keda.triggers }} - {{- toYaml . | nindent 2 }} -{{- end }} -{{- $prom := .Values.keda.prometheus }} -{{- if or $prom.requestsPerSecond $prom.tokensPerSecond }} -{{- if not $prom.serverAddress }} -{{- fail "keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set" }} -{{- end }} -{{- $selector := printf "namespace=%q,job=%q" .Release.Namespace (printf "%s%s" (include "litellm.fullname" .) (ternary "-metrics" "" .Values.metricsServer.enabled)) }} -{{- with $prom.requestsPerSecond }} - - type: prometheus - metadata: - serverAddress: {{ $prom.serverAddress | quote }} - threshold: {{ toJson . | trimAll "\"" | quote }} - query: {{ printf "sum(rate(litellm_proxy_total_requests_metric_total{%s}[1m]))" $selector | quote }} -{{- end }} -{{- with $prom.tokensPerSecond }} - - type: prometheus - metadata: - serverAddress: {{ $prom.serverAddress | quote }} - threshold: {{ toJson . | trimAll "\"" | quote }} - query: {{ printf "sum(rate(litellm_total_tokens_metric_total{%s}[1m]))" $selector | quote }} -{{- end }} -{{- end }} - advanced: - restoreToOriginalReplicaCount: {{ .Values.keda.restoreToOriginalReplicaCount }} -{{- if .Values.keda.behavior }} - horizontalPodAutoscalerConfig: - behavior: -{{- with .Values.keda.behavior }} -{{- toYaml . | nindent 8 }} -{{- end }} - -{{- end }} -{{- end }} diff --git a/helm/litellm-helm/templates/migrations-job.yaml b/helm/litellm-helm/templates/migrations-job.yaml deleted file mode 100644 index 7aa96a59102..00000000000 --- a/helm/litellm-helm/templates/migrations-job.yaml +++ /dev/null @@ -1,122 +0,0 @@ -{{- if .Values.migrationJob.enabled }} -# This job runs the Prisma migrations for the LiteLLM DB. -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "litellm.fullname" . }}-migrations - labels: - {{- include "litellm.labels" . | nindent 4 }} - {{- with .Values.migrationJob.jobLabels }} - {{- toYaml . | nindent 4 }} - {{- end }} - annotations: - {{- if .Values.migrationJob.hooks.argocd.enabled }} - argocd.argoproj.io/hook: PreSync - argocd.argoproj.io/hook-delete-policy: BeforeHookCreation - {{- end }} - {{- if .Values.migrationJob.hooks.helm.enabled }} - helm.sh/hook: "pre-install,pre-upgrade" - helm.sh/hook-delete-policy: "before-hook-creation" - helm.sh/hook-weight: {{ .Values.migrationJob.hooks.helm.weight | default "1" | quote }} - {{- end }} - checksum/config: {{ toYaml .Values | sha256sum }} - {{- with .Values.migrationJob.jobAnnotations }} - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - template: - metadata: - labels: - {{- include "litellm.labels" . | nindent 8 }} - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.migrationJob.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - annotations: - {{- with .Values.migrationJob.annotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - {{- with .Values.migrationJob.extraInitContainers }} - initContainers: - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - containers: - - name: prisma-migrations - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - {{- if .Values.migrationJob.command }} - command: {{ toYaml .Values.migrationJob.command | nindent 12 }} - {{- else }} - command: ["python", "litellm/proxy/prisma_migration.py"] - {{- end }} - {{- if .Values.migrationJob.args }} - args: {{ toYaml .Values.migrationJob.args | nindent 12 }} - {{- else if .Values.migrationJob.command }} - args: [] - {{- end }} - workingDir: "/app" - env: - {{- include "litellm.validateDbTokenAuth" . -}} - {{- if .Values.db.useExisting }} - {{- include "litellm.existingDbEnv" . | nindent 12 }} - {{- else if .Values.db.deployStandalone }} - - name: DATABASE_URL - value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }} - {{- end }} - {{- if .Values.envVars }} - {{- range $key, $val := .Values.envVars }} - - name: {{ $key }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with .Values.extraEnvVars }} - {{- toYaml . | nindent 12 }} - {{- end }} - - name: DISABLE_SCHEMA_UPDATE - value: "false" # always run the migration from the Helm PreSync hook, override the value set - {{- with .Values.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.migrationJob.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.migrationJob.extraContainers }} - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - {{- with .Values.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} - restartPolicy: OnFailure - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }} - backoffLimit: {{ .Values.migrationJob.backoffLimit }} - {{- with .Values.migrationJob.activeDeadlineSeconds }} - activeDeadlineSeconds: {{ . }} - {{- end }} -{{- end }} diff --git a/helm/litellm-helm/templates/poddisruptionbudget.yaml b/helm/litellm-helm/templates/poddisruptionbudget.yaml deleted file mode 100644 index 1715b94c1f6..00000000000 --- a/helm/litellm-helm/templates/poddisruptionbudget.yaml +++ /dev/null @@ -1,33 +0,0 @@ -{{- /* -PodDisruptionBudget for LiteLLM proxy -Controlled via .Values.pdb.enabled and .Values.pdb.{minAvailable|maxUnavailable} -Only one of minAvailable / maxUnavailable should be set. If both are set, minAvailable wins. -*/ -}} -{{- if .Values.pdb.enabled }} -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: {{ include "litellm.fullname" . }} - labels: - {{- include "litellm.labels" . | nindent 4 }} - {{- with .Values.pdb.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .Values.pdb.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - selector: - matchLabels: - {{- /* Match the Deployment selector to target the same pod set */ -}} - {{- include "litellm.selectorLabels" . | nindent 6 }} - {{- if .Values.pdb.minAvailable }} - minAvailable: {{ .Values.pdb.minAvailable }} - {{- else if .Values.pdb.maxUnavailable }} - maxUnavailable: {{ .Values.pdb.maxUnavailable }} - {{- else }} - # Safe default if enabled but not configured - maxUnavailable: 1 - {{- end }} -{{- end }} diff --git a/helm/litellm-helm/templates/secret-dbcredentials.yaml b/helm/litellm-helm/templates/secret-dbcredentials.yaml deleted file mode 100644 index 8ab89a4579e..00000000000 --- a/helm/litellm-helm/templates/secret-dbcredentials.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if .Values.db.deployStandalone -}} -{{- include "litellm.validateBundledPostgresImageTag" . -}} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "litellm.fullname" . }}-dbcredentials -data: - # Password for the "postgres" user - postgres-password: {{ ( index .Values.postgresql.auth "postgres-password") | default "litellm" | b64enc }} - username: {{ .Values.postgresql.auth.username | default "litellm" | b64enc }} - password: {{ .Values.postgresql.auth.password | default "litellm" | b64enc }} -type: Opaque -{{- end -}} \ No newline at end of file diff --git a/helm/litellm-helm/templates/secret-masterkey.yaml b/helm/litellm-helm/templates/secret-masterkey.yaml deleted file mode 100644 index 60ab4e74c6b..00000000000 --- a/helm/litellm-helm/templates/secret-masterkey.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if not .Values.masterkeySecretName }} -{{- $secretName := printf "%s-masterkey" (include "litellm.fullname" .) }} -{{- $existing := lookup "v1" "Secret" .Release.Namespace $secretName }} -{{- $masterkey := .Values.masterkey | default (dig "data" "masterkey" "" $existing | b64dec) | default (printf "sk-%s" (randAlphaNum 18)) }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ $secretName }} -data: - masterkey: {{ $masterkey | b64enc }} -type: Opaque -{{- end }} diff --git a/helm/litellm-helm/templates/service-metrics.yaml b/helm/litellm-helm/templates/service-metrics.yaml deleted file mode 100644 index 1d23fe39606..00000000000 --- a/helm/litellm-helm/templates/service-metrics.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.metricsServer.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "litellm.fullname" . }}-metrics - labels: - {{- include "litellm.labels" . | nindent 4 }} -spec: - type: ClusterIP - ports: - - port: {{ .Values.metricsServer.port }} - targetPort: metrics - protocol: TCP - name: metrics - selector: - {{- include "litellm.selectorLabels" . | nindent 4 }} -{{- end }} diff --git a/helm/litellm-helm/templates/service.yaml b/helm/litellm-helm/templates/service.yaml deleted file mode 100644 index 11812208929..00000000000 --- a/helm/litellm-helm/templates/service.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "litellm.fullname" . }} - {{- with .Values.service.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} - labels: - {{- include "litellm.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - {{- if and (eq .Values.service.type "LoadBalancer") .Values.service.loadBalancerClass }} - loadBalancerClass: {{ .Values.service.loadBalancerClass }} - {{- end }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "litellm.selectorLabels" . | nindent 4 }} diff --git a/helm/litellm-helm/templates/serviceaccount.yaml b/helm/litellm-helm/templates/serviceaccount.yaml deleted file mode 100644 index 7655470fa42..00000000000 --- a/helm/litellm-helm/templates/serviceaccount.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "litellm.serviceAccountName" . }} - labels: - {{- include "litellm.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -automountServiceAccountToken: {{ .Values.serviceAccount.automount }} -{{- end }} diff --git a/helm/litellm-helm/templates/servicemonitor.yaml b/helm/litellm-helm/templates/servicemonitor.yaml deleted file mode 100644 index 68083d0da61..00000000000 --- a/helm/litellm-helm/templates/servicemonitor.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{- with .Values.serviceMonitor }} -{{- if and (eq .enabled true) }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ include "litellm.fullname" $ }} - labels: - {{- include "litellm.labels" $ | nindent 4 }} - {{- if .labels }} - {{- toYaml .labels | nindent 4 }} - {{- end }} - {{- if .annotations }} - annotations: - {{- toYaml .annotations | nindent 4 }} - {{- end }} -spec: - selector: - matchLabels: - {{- include "litellm.selectorLabels" $ | nindent 6 }} - namespaceSelector: - matchNames: - # if not set, use the release namespace - {{- if not .namespaceSelector.matchNames }} - - {{ $.Release.Namespace | quote }} - {{- else }} - {{- toYaml .namespaceSelector.matchNames | nindent 4 }} - {{- end }} - endpoints: - - port: {{ ternary "metrics" "http" $.Values.metricsServer.enabled }} - path: /metrics/ - interval: {{ .interval }} - scrapeTimeout: {{ .scrapeTimeout }} - scheme: http - {{- if .relabelings }} - relabelings: -{{- toYaml .relabelings | nindent 4 }} - {{- end }} -{{- end }} -{{- end }} diff --git a/helm/litellm-helm/templates/tests/test-connection.yaml b/helm/litellm-helm/templates/tests/test-connection.yaml deleted file mode 100644 index 86a8f66b10b..00000000000 --- a/helm/litellm-helm/templates/tests/test-connection.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "litellm.fullname" . }}-test-connection" - labels: - {{- include "litellm.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['sh', '-c'] - args: - - | - # Wait for a bit to allow the service to be ready - sleep 10 - # Try multiple times with a delay between attempts - for i in $(seq 1 30); do - wget -T 5 "{{ include "litellm.fullname" . }}:{{ .Values.service.port }}/health/readiness" && exit 0 - echo "Attempt $i failed, waiting..." - sleep 2 - done - exit 1 - restartPolicy: Never \ No newline at end of file diff --git a/helm/litellm-helm/templates/tests/test-env-vars.yaml b/helm/litellm-helm/templates/tests/test-env-vars.yaml deleted file mode 100644 index 9f0277557a4..00000000000 --- a/helm/litellm-helm/templates/tests/test-env-vars.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "litellm.fullname" . }}-env-test" - labels: - {{- include "litellm.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: test - image: busybox - command: ['sh', '-c'] - args: - - | - # Test DD_ENV - if [ "$DD_ENV" != "dev_helm" ]; then - echo "❌ Environment variable DD_ENV mismatch. Expected: dev_helm, Got: $DD_ENV" - exit 1 - fi - echo "✅ Environment variable DD_ENV matches expected value: $DD_ENV" - - # Test DD_SERVICE - if [ "$DD_SERVICE" != "litellm" ]; then - echo "❌ Environment variable DD_SERVICE mismatch. Expected: litellm, Got: $DD_SERVICE" - exit 1 - fi - echo "✅ Environment variable DD_SERVICE matches expected value: $DD_SERVICE" - - # Test USE_DDTRACE - if [ "$USE_DDTRACE" != "true" ]; then - echo "❌ Environment variable USE_DDTRACE mismatch. Expected: true, Got: $USE_DDTRACE" - exit 1 - fi - echo "✅ Environment variable USE_DDTRACE matches expected value: $USE_DDTRACE" - env: - - name: DD_ENV - value: {{ .Values.envVars.DD_ENV | quote }} - - name: DD_SERVICE - value: {{ .Values.envVars.DD_SERVICE | quote }} - - name: USE_DDTRACE - value: {{ .Values.envVars.USE_DDTRACE | quote }} - restartPolicy: Never \ No newline at end of file diff --git a/helm/litellm-helm/templates/tests/test-servicemonitor.yaml b/helm/litellm-helm/templates/tests/test-servicemonitor.yaml deleted file mode 100644 index ef8475339c3..00000000000 --- a/helm/litellm-helm/templates/tests/test-servicemonitor.yaml +++ /dev/null @@ -1,152 +0,0 @@ -{{- if .Values.serviceMonitor.enabled }} -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "litellm.fullname" . }}-test-servicemonitor" - labels: - {{- include "litellm.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: test - image: docker.io/bitnamilegacy/kubectl:1.29.2-debian-12-r3 - command: ['sh', '-c'] - args: - - | - set -e - echo "🔍 Testing ServiceMonitor configuration..." - - # Check if ServiceMonitor exists - if ! kubectl get servicemonitor {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} &>/dev/null; then - echo "❌ ServiceMonitor not found" - exit 1 - fi - echo "✅ ServiceMonitor exists" - - # Get ServiceMonitor YAML - SM=$(kubectl get servicemonitor {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} -o yaml) - - # Test endpoint configuration - ENDPOINT_PORT=$(echo "$SM" | grep -A 5 "endpoints:" | grep "port:" | awk '{print $2}') - if [ "$ENDPOINT_PORT" != "http" ]; then - echo "❌ Endpoint port mismatch. Expected: http, Got: $ENDPOINT_PORT" - exit 1 - fi - echo "✅ Endpoint port is correctly set to: $ENDPOINT_PORT" - - # Test endpoint path - ENDPOINT_PATH=$(echo "$SM" | grep -A 5 "endpoints:" | grep "path:" | awk '{print $2}') - if [ "$ENDPOINT_PATH" != "/metrics/" ]; then - echo "❌ Endpoint path mismatch. Expected: /metrics/, Got: $ENDPOINT_PATH" - exit 1 - fi - echo "✅ Endpoint path is correctly set to: $ENDPOINT_PATH" - - # Test interval - INTERVAL=$(echo "$SM" | grep "interval:" | awk '{print $2}') - if [ "$INTERVAL" != "{{ .Values.serviceMonitor.interval }}" ]; then - echo "❌ Interval mismatch. Expected: {{ .Values.serviceMonitor.interval }}, Got: $INTERVAL" - exit 1 - fi - echo "✅ Interval is correctly set to: $INTERVAL" - - # Test scrapeTimeout - TIMEOUT=$(echo "$SM" | grep "scrapeTimeout:" | awk '{print $2}') - if [ "$TIMEOUT" != "{{ .Values.serviceMonitor.scrapeTimeout }}" ]; then - echo "❌ ScrapeTimeout mismatch. Expected: {{ .Values.serviceMonitor.scrapeTimeout }}, Got: $TIMEOUT" - exit 1 - fi - echo "✅ ScrapeTimeout is correctly set to: $TIMEOUT" - - # Test scheme - SCHEME=$(echo "$SM" | grep "scheme:" | awk '{print $2}') - if [ "$SCHEME" != "http" ]; then - echo "❌ Scheme mismatch. Expected: http, Got: $SCHEME" - exit 1 - fi - echo "✅ Scheme is correctly set to: $SCHEME" - - {{- if .Values.serviceMonitor.labels }} - # Test custom labels - echo "🔍 Checking custom labels..." - {{- range $key, $value := .Values.serviceMonitor.labels }} - LABEL_VALUE=$(echo "$SM" | grep -A 20 "metadata:" | grep "{{ $key }}:" | awk '{print $2}') - if [ "$LABEL_VALUE" != "{{ $value }}" ]; then - echo "❌ Label {{ $key }} mismatch. Expected: {{ $value }}, Got: $LABEL_VALUE" - exit 1 - fi - echo "✅ Label {{ $key }} is correctly set to: {{ $value }}" - {{- end }} - {{- end }} - - {{- if .Values.serviceMonitor.annotations }} - # Test annotations - echo "🔍 Checking annotations..." - {{- range $key, $value := .Values.serviceMonitor.annotations }} - ANNOTATION_VALUE=$(echo "$SM" | grep -A 10 "annotations:" | grep "{{ $key }}:" | awk '{print $2}') - if [ "$ANNOTATION_VALUE" != "{{ $value }}" ]; then - echo "❌ Annotation {{ $key }} mismatch. Expected: {{ $value }}, Got: $ANNOTATION_VALUE" - exit 1 - fi - echo "✅ Annotation {{ $key }} is correctly set to: {{ $value }}" - {{- end }} - {{- end }} - - {{- if .Values.serviceMonitor.namespaceSelector.matchNames }} - # Test namespace selector - echo "🔍 Checking namespace selector..." - {{- range .Values.serviceMonitor.namespaceSelector.matchNames }} - if ! echo "$SM" | grep -A 5 "namespaceSelector:" | grep -q "{{ . }}"; then - echo "❌ Namespace {{ . }} not found in namespaceSelector" - exit 1 - fi - echo "✅ Namespace {{ . }} found in namespaceSelector" - {{- end }} - {{- else }} - # Test default namespace selector (should be release namespace) - if ! echo "$SM" | grep -A 5 "namespaceSelector:" | grep -q "{{ .Release.Namespace }}"; then - echo "❌ Release namespace {{ .Release.Namespace }} not found in namespaceSelector" - exit 1 - fi - echo "✅ Default namespace selector set to release namespace: {{ .Release.Namespace }}" - {{- end }} - - {{- if .Values.serviceMonitor.relabelings }} - # Test relabelings - echo "🔍 Checking relabelings configuration..." - if ! echo "$SM" | grep -q "relabelings:"; then - echo "❌ Relabelings section not found" - exit 1 - fi - echo "✅ Relabelings section exists" - {{- range .Values.serviceMonitor.relabelings }} - {{- if .targetLabel }} - if ! echo "$SM" | grep -A 50 "relabelings:" | grep -q "targetLabel: {{ .targetLabel }}"; then - echo "❌ Relabeling targetLabel {{ .targetLabel }} not found" - exit 1 - fi - echo "✅ Relabeling targetLabel {{ .targetLabel }} found" - {{- end }} - {{- if .action }} - if ! echo "$SM" | grep -A 50 "relabelings:" | grep -q "action: {{ .action }}"; then - echo "❌ Relabeling action {{ .action }} not found" - exit 1 - fi - echo "✅ Relabeling action {{ .action }} found" - {{- end }} - {{- end }} - {{- end }} - - # Test selector labels match the service - echo "🔍 Checking selector labels match service..." - SVC_LABELS=$(kubectl get svc {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} -o jsonpath='{.metadata.labels}') - echo "Service labels: $SVC_LABELS" - echo "✅ Selector labels validation passed" - - echo "" - echo "🎉 All ServiceMonitor tests passed successfully!" - serviceAccountName: {{ include "litellm.serviceAccountName" . }} - restartPolicy: Never -{{- end }} - diff --git a/helm/litellm-helm/tests/billing_metrics_tests.yaml b/helm/litellm-helm/tests/billing_metrics_tests.yaml deleted file mode 100644 index 71803df378c..00000000000 --- a/helm/litellm-helm/tests/billing_metrics_tests.yaml +++ /dev/null @@ -1,297 +0,0 @@ -suite: test billingMetrics wiring on the proxy deployment -templates: - - deployment.yaml - - configmap-litellm.yaml - - migrations-job.yaml -tests: - - it: is off by default, adding no env, volume, or mount - template: deployment.yaml - asserts: - - notContains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: litellm-billing-metrics-mtls - - notContains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls - mountPath: /etc/litellm/billing-mtls - readOnly: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai - - - it: renders the endpoint and the mounted cert paths when enabled - template: deployment.yaml - set: - billingMetrics: - enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_CLIENT_CERT - value: /etc/litellm/billing-mtls/tls.crt - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_CLIENT_KEY - value: /etc/litellm/billing-mtls/tls.key - - # The conventional Secret name is the default, so enabling the block is enough. - - it: mounts the default cert secret read-only alongside the config volume - template: deployment.yaml - set: - billingMetrics: - enabled: true - asserts: - - contains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: litellm-billing-metrics-mtls - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls - mountPath: /etc/litellm/billing-mtls - readOnly: true - - - it: honours a secretName override - template: deployment.yaml - set: - billingMetrics: - enabled: true - secretName: my-billing-mtls - asserts: - - contains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: my-billing-mtls - - notContains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: litellm-billing-metrics-mtls - - - it: honours an endpoint override - template: deployment.yaml - set: - billingMetrics: - enabled: true - endpoint: https://collector.internal:4318 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://collector.internal:4318 - - # The production collector presents a public web-PKI certificate, so the CA - # override must stay absent unless a private collector is configured. - - it: omits the CA env, volume, and mount when no caSecretName is set - template: deployment.yaml - set: - billingMetrics: - enabled: true - asserts: - - notContains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls-ca - secret: - secretName: billing-ca - - notContains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls-ca - mountPath: /etc/litellm/billing-mtls-ca - readOnly: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_CA_CERT - value: /etc/litellm/billing-mtls-ca/ca.crt - - - it: mounts the CA secret when caSecretName is set - template: deployment.yaml - set: - billingMetrics: - enabled: true - caSecretName: billing-ca - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_CA_CERT - value: /etc/litellm/billing-mtls-ca/ca.crt - - contains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls-ca - secret: - secretName: billing-ca - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls-ca - mountPath: /etc/litellm/billing-mtls-ca - readOnly: true - - - it: passes the export interval through only when set - template: deployment.yaml - set: - billingMetrics: - enabled: true - exportIntervalMs: 5000 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS - value: "5000" - - - it: omits the export interval when unset - template: deployment.yaml - set: - billingMetrics: - enabled: true - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS - value: "60000" - - # Kubernetes resolves duplicate env names last-wins, so the chart-owned billing - # entries must render after .Values.envVars or a user could silently redirect - # the metering export. The three billing entries are the last ones emitted here - # (migrationJob, which appends DISABLE_SCHEMA_UPDATE, is off for this case). - - it: renders the billing endpoint after envVars so it cannot be shadowed - template: deployment.yaml - set: - migrationJob: - enabled: false - billingMetrics: - enabled: true - envVars: - LITELLM_BILLING_METRICS_ENDPOINT: https://shadowed.example - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://shadowed.example - - equal: - path: spec.template.spec.containers[0].env[-3] - value: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai - - equal: - path: spec.template.spec.containers[0].env[-2].name - value: LITELLM_BILLING_METRICS_CLIENT_CERT - - equal: - path: spec.template.spec.containers[0].env[-1].name - value: LITELLM_BILLING_METRICS_CLIENT_KEY - - - it: keeps user-supplied volumes and mounts alongside the billing secret - template: deployment.yaml - set: - billingMetrics: - enabled: true - volumes: - - name: custom-callbacks - configMap: - name: my-callbacks - volumeMounts: - - name: custom-callbacks - mountPath: /app/callbacks - asserts: - - contains: - path: spec.template.spec.volumes - content: - name: custom-callbacks - configMap: - name: my-callbacks - - contains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: litellm-billing-metrics-mtls - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: custom-callbacks - mountPath: /app/callbacks - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls - mountPath: /etc/litellm/billing-mtls - readOnly: true - - - it: still mounts the proxy config when enabled - template: deployment.yaml - set: - billingMetrics: - enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: litellm-config - mountPath: /etc/litellm/config.yaml - subPath: config.yaml - - # Only the proxy serves billable traffic. The migrations Job must never mount - # the client certificate, and it renders its own env and volumes, so nothing - # stops a future edit from wiring the billing include into it by mistake. - - it: does not touch the migrations job when enabled - template: migrations-job.yaml - set: - billingMetrics: - enabled: true - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai - - notExists: - path: spec.template.spec.containers[0].volumeMounts - - notExists: - path: spec.template.spec.volumes - - - it: fails loudly when enabled with an emptied secretName - template: deployment.yaml - set: - billingMetrics: - enabled: true - secretName: "" - asserts: - - failedTemplate: - errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key) - - - it: fails loudly when enabled without an endpoint - template: deployment.yaml - set: - billingMetrics: - enabled: true - endpoint: "" - asserts: - - failedTemplate: - errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true diff --git a/helm/litellm-helm/tests/bundled_db_images_tests.yaml b/helm/litellm-helm/tests/bundled_db_images_tests.yaml deleted file mode 100644 index 8f0860c2721..00000000000 --- a/helm/litellm-helm/tests/bundled_db_images_tests.yaml +++ /dev/null @@ -1,94 +0,0 @@ -suite: test bundled database images -templates: - - charts/postgresql/templates/primary/statefulset.yaml - - charts/redis/templates/master/application.yaml - - charts/redis/templates/configmap.yaml - - charts/redis/templates/health-configmap.yaml - - charts/redis/templates/scripts-configmap.yaml - - charts/redis/templates/secret.yaml - - secret-dbcredentials.yaml - - templates/tests/test-servicemonitor.yaml -tests: - - it: should pull the bundled postgres from a repository that still publishes the pinned tag - template: charts/postgresql/templates/primary/statefulset.yaml - set: - db.deployStandalone: true - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: docker.io/bitnamilegacy/postgresql:16.2.0-debian-12-r6 - - - it: should pull the bundled postgres metrics exporter from the same repository - template: charts/postgresql/templates/primary/statefulset.yaml - set: - db.deployStandalone: true - postgresql.metrics.enabled: true - asserts: - - equal: - path: spec.template.spec.containers[1].image - value: docker.io/bitnamilegacy/postgres-exporter:0.15.0-debian-12-r14 - - - it: should run the bundled postgres init container from the same repository - template: charts/postgresql/templates/primary/statefulset.yaml - set: - db.deployStandalone: true - postgresql.volumePermissions.enabled: true - asserts: - - equal: - path: spec.template.spec.initContainers[0].image - value: docker.io/bitnamilegacy/os-shell:12-debian-12-r16 - - - it: should pull the bundled redis from a repository that still publishes the pinned tag - template: charts/redis/templates/master/application.yaml - set: - redis.enabled: true - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: docker.io/bitnamilegacy/redis:7.2.4-debian-12-r9 - - - it: should reject a floating postgres tag that could cross a major version on an existing volume - template: secret-dbcredentials.yaml - set: - db.deployStandalone: true - postgresql.image.tag: latest - asserts: - - failedTemplate: - errorMessage: 'postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got "latest"). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore.' - - - it: should reject an empty postgres tag - template: secret-dbcredentials.yaml - set: - db.deployStandalone: true - postgresql.image.tag: "" - asserts: - - failedTemplate: - errorMessage: 'postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got ""). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore.' - - - it: should accept an empty postgres tag when the image is pinned by digest - template: secret-dbcredentials.yaml - set: - db.deployStandalone: true - postgresql.image.tag: "" - postgresql.image.digest: sha256:0d0e2f1a5b3c4d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6 - asserts: - - hasDocuments: - count: 1 - - - it: should run the servicemonitor test pod from a pinned image - template: templates/tests/test-servicemonitor.yaml - set: - serviceMonitor.enabled: true - asserts: - - equal: - path: spec.containers[0].image - value: docker.io/bitnamilegacy/kubectl:1.29.2-debian-12-r3 - - - it: should not constrain the postgres tag when the bundled database is not deployed - template: secret-dbcredentials.yaml - set: - db.deployStandalone: false - postgresql.image.tag: latest - asserts: - - hasDocuments: - count: 0 diff --git a/helm/litellm-helm/tests/collector_tests.yaml b/helm/litellm-helm/tests/collector_tests.yaml deleted file mode 100644 index 0340b1161b7..00000000000 --- a/helm/litellm-helm/tests/collector_tests.yaml +++ /dev/null @@ -1,272 +0,0 @@ -suite: test collector sidecar -templates: - - deployment.yaml - - hpa.yaml - - configmap-litellm.yaml -tests: - - it: should run the proxy alone with no collector env by default - template: deployment.yaml - asserts: - - lengthEqual: - path: spec.template.spec.containers - count: 1 - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_ENABLED - value: "true" - - notContains: - path: spec.template.spec.volumes - content: - name: collector-socket - any: true - - - it: should add the sidecar on the same image and point both containers at the unix socket - template: deployment.yaml - set: - image.tag: test - db.connectionPool.enabled: true - collector.enabled: true - collector.resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: "1" - memory: 2Gi - asserts: - - lengthEqual: - path: spec.template.spec.containers - count: 2 - - equal: - path: spec.template.spec.containers[1].name - value: litellm-collector - - equal: - path: spec.template.spec.containers[1].image - value: ghcr.io/berriai/litellm:test - - equal: - path: spec.template.spec.containers[1].command - value: [python, -m, litellm.proxy.collector] - - equal: - path: spec.template.spec.containers[1].resources.requests.cpu - value: 500m - - equal: - path: spec.template.spec.containers[1].resources.limits.memory - value: 2Gi - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_ENABLED - value: "true" - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_ADDRESS - value: unix:///var/run/litellm/collector.sock - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_BUFFER_SIZE - value: "1000" - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_ON_UNAVAILABLE - value: fallback - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_JOB_ROLE - value: collector - - contains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_JOB_ROLE - value: collector - - contains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_COLLECTOR_ADDRESS - value: unix:///var/run/litellm/collector.sock - - contains: - path: spec.template.spec.containers[1].env - content: - name: CONFIG_FILE_PATH - value: /etc/litellm/config.yaml - - contains: - path: spec.template.spec.containers[1].env - content: - name: DATABASE_HOST - value: RELEASE-NAME-postgresql - - contains: - path: spec.template.spec.containers[1].env - content: - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: RELEASE-NAME-litellm-dbcredentials - key: password - - contains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_ENABLED - value: "true" - - contains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - value: "20" - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: collector-socket - mountPath: /var/run/litellm - - contains: - path: spec.template.spec.containers[1].volumeMounts - content: - name: collector-socket - mountPath: /var/run/litellm - - contains: - path: spec.template.spec.containers[1].volumeMounts - content: - name: litellm-config - mountPath: /etc/litellm/config.yaml - subPath: config.yaml - - contains: - path: spec.template.spec.volumes - content: - name: collector-socket - emptyDir: - sizeLimit: 1Mi - - - it: should skip the socket volume and pass the policy through on tcp transport - template: deployment.yaml - set: - collector.enabled: true - collector.address: tcp://127.0.0.1:4100 - collector.onUnavailable: drop - collector.bufferSize: 50 - envVars: - CONFIG_FILE_PATH: /custom/config.yaml - asserts: - - lengthEqual: - path: spec.template.spec.containers - count: 2 - - notContains: - path: spec.template.spec.containers[1].env - content: - name: CONFIG_FILE_PATH - value: /etc/litellm/config.yaml - - contains: - path: spec.template.spec.containers[1].env - content: - name: CONFIG_FILE_PATH - value: /custom/config.yaml - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_ADDRESS - value: tcp://127.0.0.1:4100 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_ON_UNAVAILABLE - value: drop - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_COLLECTOR_BUFFER_SIZE - value: "50" - - notContains: - path: spec.template.spec.volumes - content: - name: collector-socket - any: true - - - it: should keep metrics and billing env on the proxy container only - template: deployment.yaml - set: - collector.enabled: true - metricsServer.enabled: true - metricsServer.port: 9090 - billingMetrics.enabled: true - billingMetrics.endpoint: https://metering.example.com - billingMetrics.secretName: billing-mtls - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: PROMETHEUS_METRICS_PORT - value: "9090" - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://metering.example.com - - notContains: - path: spec.template.spec.containers[1].env - content: - name: PROMETHEUS_METRICS_PORT - any: true - - notContains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - any: true - - notContains: - path: spec.template.spec.containers[1].volumeMounts - content: - name: billing-metrics-mtls - any: true - - - it: should give the sidecar the same scratch mounts as the proxy on a read-only root - template: deployment.yaml - set: - collector.enabled: true - securityContext.readOnlyRootFilesystem: true - asserts: - - contains: - path: spec.template.spec.containers[1].volumeMounts - content: - name: npm - mountPath: /.npm - - contains: - path: spec.template.spec.containers[1].volumeMounts - content: - name: cache - mountPath: /.cache - - contains: - path: spec.template.spec.containers[1].volumeMounts - content: - name: tmp - mountPath: /tmp - - - it: should keep the pod-wide cpu metric unless asked to scale on the proxy container - template: hpa.yaml - set: - autoscaling.enabled: true - collector.enabled: true - asserts: - - equal: { path: "spec.metrics[0].type", value: Resource } - - equal: { path: "spec.metrics[0].resource.name", value: cpu } - - - it: should scale on the proxy container's cpu only when opted in - template: hpa.yaml - set: - autoscaling.enabled: true - collector.enabled: true - collector.scaleOnProxyContainerCpu: true - asserts: - - equal: { path: "spec.metrics[0].type", value: ContainerResource } - - equal: { path: "spec.metrics[0].containerResource.name", value: cpu } - - equal: { path: "spec.metrics[0].containerResource.container", value: litellm } - - equal: { path: "spec.metrics[0].containerResource.target.averageUtilization", value: 60 } - - isNull: { path: "spec.metrics[0].resource" } - - - it: should not switch to the container metric while the sidecar is off - template: hpa.yaml - set: - autoscaling.enabled: true - collector.scaleOnProxyContainerCpu: true - asserts: - - equal: { path: "spec.metrics[0].type", value: Resource } diff --git a/helm/litellm-helm/tests/connection_pool_tests.yaml b/helm/litellm-helm/tests/connection_pool_tests.yaml deleted file mode 100644 index 203082f27ba..00000000000 --- a/helm/litellm-helm/tests/connection_pool_tests.yaml +++ /dev/null @@ -1,112 +0,0 @@ -suite: test in-container connection pool -templates: - - deployment.yaml - - configmap-litellm.yaml -tests: - - it: should not emit pgbouncer env vars by default - template: deployment.yaml - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_PGBOUNCER_ENABLED - value: "true" - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - value: "20" - - - it: should enable the pool with the default sizing when connectionPool.enabled is set - template: deployment.yaml - set: - db.connectionPool.enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_PGBOUNCER_ENABLED - value: "true" - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - value: "20" - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN - value: "1000" - - - it: should pass custom sizing through as strings next to the worker count - template: deployment.yaml - set: - numWorkers: 4 - db.connectionPool.enabled: true - db.connectionPool.maxDbConnections: 8 - db.connectionPool.maxClientConn: 400 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - value: "8" - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN - value: "400" - - contains: - path: spec.template.spec.containers[0].args - content: "4" - - - it: should give the collector sidecar the same pool env as the proxy container - template: deployment.yaml - set: - collector.enabled: true - db.connectionPool.enabled: true - db.connectionPool.maxDbConnections: 8 - db.connectionPool.maxClientConn: 400 - asserts: - - equal: - path: spec.template.spec.containers[1].name - value: litellm-collector - - contains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_ENABLED - value: "true" - - contains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - value: "8" - - contains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN - value: "400" - - - it: should give the collector sidecar no pool env when the pool is off - template: deployment.yaml - set: - collector.enabled: true - asserts: - - equal: - path: spec.template.spec.containers[1].name - value: litellm-collector - - notContains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_ENABLED - any: true - - notContains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS - any: true - - notContains: - path: spec.template.spec.containers[1].env - content: - name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN - any: true diff --git a/helm/litellm-helm/tests/coordination_redis_tests.yaml b/helm/litellm-helm/tests/coordination_redis_tests.yaml deleted file mode 100644 index 0b58b1e6bc8..00000000000 --- a/helm/litellm-helm/tests/coordination_redis_tests.yaml +++ /dev/null @@ -1,143 +0,0 @@ -suite: test coordination redis -templates: - - configmap-litellm.yaml - - deployment.yaml -tests: - - it: should not render coordination_redis when redis is disabled - template: configmap-litellm.yaml - set: - redis.enabled: false - asserts: - - notMatchRegex: - path: data["config.yaml"] - pattern: coordination_redis - - - it: should not emit redis env vars when redis is disabled - template: deployment.yaml - set: - redis.enabled: false - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: REDIS_HOST - value: RELEASE-NAME-redis-master - any: true - - - it: should render coordination_redis pointing at the bundled redis when enabled - template: configmap-litellm.yaml - set: - redis.enabled: true - asserts: - - matchRegex: - path: data["config.yaml"] - pattern: "coordination_redis:\n host: os.environ/REDIS_HOST\n password: os.environ/REDIS_PASSWORD\n port: os.environ/REDIS_PORT\n" - - matchRegex: - path: data["config.yaml"] - pattern: "master_key: os.environ/PROXY_MASTER_KEY" - - - it: should emit redis env vars backing the coordination_redis os.environ refs - template: deployment.yaml - set: - redis.enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: REDIS_HOST - value: RELEASE-NAME-redis-master - - contains: - path: spec.template.spec.containers[0].env - content: - name: REDIS_PORT - value: "6379" - - contains: - path: spec.template.spec.containers[0].env - content: - name: REDIS_PASSWORD - valueFrom: - secretKeyRef: - name: RELEASE-NAME-redis - key: redis-password - - - it: should not render coordination_redis when coordination is opted out - template: configmap-litellm.yaml - set: - redis.enabled: true - redis.coordination.enabled: false - asserts: - - notMatchRegex: - path: data["config.yaml"] - pattern: coordination_redis - - - it: should keep emitting redis env vars when coordination is opted out - template: deployment.yaml - set: - redis.enabled: true - redis.coordination.enabled: false - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: REDIS_HOST - value: RELEASE-NAME-redis-master - - - it: should not clobber a user supplied coordination_redis block - template: configmap-litellm.yaml - set: - redis.enabled: true - proxy_config.general_settings.coordination_redis: - url: os.environ/COORDINATION_REDIS_URL - asserts: - - matchRegex: - path: data["config.yaml"] - pattern: "coordination_redis:\n url: os.environ/COORDINATION_REDIS_URL\n" - - notMatchRegex: - path: data["config.yaml"] - pattern: "host: os.environ/REDIS_HOST" - - - it: should render sentinel_nodes and service_name in sentinel mode - template: configmap-litellm.yaml - set: - redis.enabled: true - redis.architecture: replication - redis.sentinel.enabled: true - asserts: - # The sentinel Service the redis subchart renders is "-redis", and a - # plain client cannot speak the sentinel protocol, so host/port must not appear - - matchRegex: - path: data["config.yaml"] - pattern: "coordination_redis:\n password: os.environ/REDIS_PASSWORD\n sentinel_nodes:\n - - RELEASE-NAME-redis\n - 26379\n service_name: mymaster\n" - - notMatchRegex: - path: data["config.yaml"] - pattern: "host: os.environ/REDIS_HOST" - - - it: should carry a custom sentinel masterSet into service_name - template: configmap-litellm.yaml - set: - redis.enabled: true - redis.architecture: replication - redis.sentinel.enabled: true - redis.sentinel.masterSet: litellm-master - asserts: - - matchRegex: - path: data["config.yaml"] - pattern: "service_name: litellm-master" - - - it: should point REDIS_HOST at the sentinel service in sentinel mode - template: deployment.yaml - set: - redis.enabled: true - redis.architecture: replication - redis.sentinel.enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: REDIS_HOST - value: RELEASE-NAME-redis - - contains: - path: spec.template.spec.containers[0].env - content: - name: REDIS_PORT - value: "26379" diff --git a/helm/litellm-helm/tests/database_auth_tests.yaml b/helm/litellm-helm/tests/database_auth_tests.yaml deleted file mode 100644 index 2c480196dfb..00000000000 --- a/helm/litellm-helm/tests/database_auth_tests.yaml +++ /dev/null @@ -1,288 +0,0 @@ -suite: test existing database token auth -templates: - - deployment.yaml - - migrations-job.yaml - - configmap-litellm.yaml -tests: - - it: should keep password auth from the secret when no token auth is set - template: deployment.yaml - set: - db.useExisting: true - db.deployStandalone: false - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_USERNAME - valueFrom: - secretKeyRef: - name: postgres - key: username - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: postgres - key: password - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL - value: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME) - - notContains: - path: spec.template.spec.containers[0].env - content: - name: IAM_TOKEN_DB_AUTH - any: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: AZURE_POSTGRESQL_AUTH - any: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PORT - any: true - - - it: should render RDS IAM auth without a password secret on the proxy and the collector sidecar - template: deployment.yaml - set: - db.useExisting: true - db.deployStandalone: false - db.useIAMAuth: true - db.username: litellm_iam - db.endpoint: writer.cluster-abc.us-west-1.rds.amazonaws.com - db.port: 5433 - collector.enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: IAM_TOKEN_DB_AUTH - value: "true" - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_USERNAME - value: litellm_iam - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_HOST - value: writer.cluster-abc.us-west-1.rds.amazonaws.com - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PORT - value: "5433" - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_NAME - value: litellm - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PASSWORD - any: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL - any: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: AZURE_POSTGRESQL_AUTH - any: true - - contains: - path: spec.template.spec.containers[1].env - content: - name: IAM_TOKEN_DB_AUTH - value: "true" - - notContains: - path: spec.template.spec.containers[1].env - content: - name: DATABASE_PASSWORD - any: true - - - it: should give the migrations job the same token auth env as the proxy - template: migrations-job.yaml - set: - migrationJob.enabled: true - db.useExisting: true - db.deployStandalone: false - db.useIAMAuth: true - db.username: litellm_iam - db.endpoint: writer.cluster-abc.us-west-1.rds.amazonaws.com - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: IAM_TOKEN_DB_AUTH - value: "true" - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_USERNAME - value: litellm_iam - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_HOST - value: writer.cluster-abc.us-west-1.rds.amazonaws.com - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PORT - value: "5432" - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PASSWORD - any: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL - any: true - - - it: should carry db.schema as DATABASE_SCHEMA under token auth and omit it when empty - template: deployment.yaml - set: - db.useExisting: true - db.deployStandalone: false - db.useIAMAuth: true - db.username: litellm_iam - db.schema: tenant1 - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_SCHEMA - value: tenant1 - - - it: should leave the schema to db.url under password auth even when db.schema is set - template: deployment.yaml - set: - db.useExisting: true - db.deployStandalone: false - db.schema: tenant1 - db.url: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)?schema=tenant1 - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_SCHEMA - any: true - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL - value: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)?schema=tenant1 - - - it: should not emit DATABASE_SCHEMA when db.schema is empty - template: deployment.yaml - set: - db.useExisting: true - db.deployStandalone: false - db.useIAMAuth: true - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_SCHEMA - any: true - - - it: should read the user from the secret when db.username is empty under token auth - template: deployment.yaml - set: - db.useExisting: true - db.deployStandalone: false - db.useIAMAuth: true - db.secret.name: rds-identity - db.secret.usernameKey: user - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_USERNAME - valueFrom: - secretKeyRef: - name: rds-identity - key: user - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PASSWORD - any: true - - - it: should render Entra auth with AZURE_POSTGRESQL_AUTH and no password - template: deployment.yaml - set: - db.useExisting: true - db.deployStandalone: false - db.useAzureEntraAuth: true - db.username: litellm@example.onmicrosoft.com - db.endpoint: litellm.postgres.database.azure.com - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: AZURE_POSTGRESQL_AUTH - value: "true" - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_USERNAME - value: litellm@example.onmicrosoft.com - - notContains: - path: spec.template.spec.containers[0].env - content: - name: IAM_TOKEN_DB_AUTH - any: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_PASSWORD - any: true - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL - any: true - - - it: should fail when both token sources are enabled - template: deployment.yaml - set: - migrationJob.enabled: false - db.useExisting: true - db.deployStandalone: false - db.useIAMAuth: true - db.useAzureEntraAuth: true - asserts: - - failedTemplate: - errorMessage: "db.useIAMAuth and db.useAzureEntraAuth are mutually exclusive: the database password can only come from one token source" - - - it: should fail when token auth is set while the bundled postgres is still deployed - template: deployment.yaml - set: - migrationJob.enabled: false - db.useExisting: true - db.useIAMAuth: true - asserts: - - failedTemplate: - errorPattern: "need db.useExisting: true and db.deployStandalone: false" - - - it: should fail when token auth is set without useExisting - template: migrations-job.yaml - set: - migrationJob.enabled: true - db.deployStandalone: false - db.useAzureEntraAuth: true - asserts: - - failedTemplate: - errorPattern: "need db.useExisting: true and db.deployStandalone: false" diff --git a/helm/litellm-helm/tests/deployment_command_args_labels_tests.yaml b/helm/litellm-helm/tests/deployment_command_args_labels_tests.yaml deleted file mode 100644 index 6b0d45ebf48..00000000000 --- a/helm/litellm-helm/tests/deployment_command_args_labels_tests.yaml +++ /dev/null @@ -1,68 +0,0 @@ -suite: test deployment command, args, and deploymentLabels -templates: - - deployment.yaml - - configmap-litellm.yaml -tests: - - it: should override args when custom args specified - template: deployment.yaml - set: - args: - - --custom-arg1 - - value1 - - --custom-arg2 - asserts: - - equal: - path: spec.template.spec.containers[0].args - value: - - --custom-arg1 - - value1 - - --custom-arg2 - - it: should set custom command when specified - template: deployment.yaml - set: - command: - - /bin/sh - - -c - asserts: - - equal: - path: spec.template.spec.containers[0].command - value: - - /bin/sh - - -c - - it: should set custom command and args together - template: deployment.yaml - set: - command: - - python - - -u - args: - - my_script.py - - --verbose - asserts: - - equal: - path: spec.template.spec.containers[0].command - value: - - python - - -u - - equal: - path: spec.template.spec.containers[0].args - value: - - my_script.py - - --verbose - - it: should add deploymentLabels to deployment metadata - template: deployment.yaml - set: - deploymentLabels: - environment: production - team: platform - version: v1.2.3 - asserts: - - equal: - path: metadata.labels.environment - value: production - - equal: - path: metadata.labels.team - value: platform - - equal: - path: metadata.labels.version - value: v1.2.3 diff --git a/helm/litellm-helm/tests/deployment_tests.yaml b/helm/litellm-helm/tests/deployment_tests.yaml deleted file mode 100644 index 292eeadfd13..00000000000 --- a/helm/litellm-helm/tests/deployment_tests.yaml +++ /dev/null @@ -1,520 +0,0 @@ -suite: test deployment -templates: - - deployment.yaml - - configmap-litellm.yaml -tests: - - it: should work - template: deployment.yaml - set: - image.tag: test - asserts: - - isKind: - of: Deployment - - matchRegex: - path: metadata.name - pattern: -litellm$ - - equal: - path: spec.template.spec.containers[0].image - value: ghcr.io/berriai/litellm:test - - it: should work with tolerations - template: deployment.yaml - set: - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - asserts: - - equal: - path: spec.template.spec.tolerations[0].key - value: node-role.kubernetes.io/master - - equal: - path: spec.template.spec.tolerations[0].operator - value: Exists - - it: should work with affinity - template: deployment.yaml - set: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: topology.kubernetes.io/zone - operator: In - values: - - antarctica-east1 - asserts: - - equal: - path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key - value: topology.kubernetes.io/zone - - equal: - path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator - value: In - - equal: - path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[0] - value: antarctica-east1 - - it: should work without masterkeySecretName or masterkeySecretKey - template: deployment.yaml - set: - masterkeySecretName: "" - masterkeySecretKey: "" - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: PROXY_MASTER_KEY - valueFrom: - secretKeyRef: - name: RELEASE-NAME-litellm-masterkey - key: masterkey - - it: should work with masterkeySecretName and masterkeySecretKey - template: deployment.yaml - set: - masterkeySecretName: my-secret - masterkeySecretKey: my-key - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: PROXY_MASTER_KEY - valueFrom: - secretKeyRef: - name: my-secret - key: my-key - - it: should inject DATABASE_READER_HOST from readReplicaEndpointKey before DATABASE_URL_READ_REPLICA - template: deployment.yaml - set: - db: - deployStandalone: false - useExisting: true - secret: - name: postgres - usernameKey: username - passwordKey: password - readReplicaEndpointKey: reader-host - readReplicaUrl: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME)?sslmode=require - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_READER_HOST - valueFrom: - secretKeyRef: - name: postgres - key: reader-host - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL_READ_REPLICA - value: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME)?sslmode=require - # $(VAR) interpolation only resolves vars defined EARLIER in the env - # array, so the reader host must precede the composed URL - - equal: - path: spec.template.spec.containers[0].env[7].name - value: DATABASE_READER_HOST - - equal: - path: spec.template.spec.containers[0].env[8].name - value: DATABASE_URL_READ_REPLICA - - it: should omit reader host when readReplicaUrl is unset - template: deployment.yaml - set: - db: - deployStandalone: false - useExisting: true - secret: - name: postgres - usernameKey: username - passwordKey: password - readReplicaEndpointKey: reader-host - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_READER_HOST - valueFrom: - secretKeyRef: - name: postgres - key: reader-host - - it: should prefer readReplicaUrlKey over readReplicaEndpointKey composition - template: deployment.yaml - set: - db: - useExisting: true - secret: - name: postgres - usernameKey: username - passwordKey: password - readReplicaUrlKey: reader-url - readReplicaEndpointKey: reader-host - readReplicaUrl: postgresql://ignored - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL_READ_REPLICA - valueFrom: - secretKeyRef: - name: postgres - key: reader-url - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL_READ_REPLICA - value: postgresql://ignored - # the unused reader-host secret ref must be suppressed so a missing - # key can't fail pod creation - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_READER_HOST - valueFrom: - secretKeyRef: - name: postgres - key: reader-host - - it: should work with extraEnvVars - template: deployment.yaml - set: - extraEnvVars: - - name: EXTRA_ENV_VAR - valueFrom: - fieldRef: - fieldPath: metadata.labels['env'] - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: EXTRA_ENV_VAR - valueFrom: - fieldRef: - fieldPath: metadata.labels['env'] - - it: should work with both extraEnvVars and envVars - template: deployment.yaml - set: - envVars: - ENV_VAR: ENV_VAR_VALUE - extraEnvVars: - - name: EXTRA_ENV_VAR - value: EXTRA_ENV_VAR_VALUE - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: ENV_VAR - value: ENV_VAR_VALUE - - contains: - path: spec.template.spec.containers[0].env - content: - name: EXTRA_ENV_VAR - value: EXTRA_ENV_VAR_VALUE - - it: should mount existing configmap when create=false - template: deployment.yaml - set: - proxyConfigMap: - create: false - name: my-litellm-config - key: custom.yaml - asserts: - - contains: - path: spec.template.spec.volumes - content: - name: litellm-config - configMap: - name: my-litellm-config - items: - - key: custom.yaml - path: config.yaml - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: litellm-config - mountPath: /etc/litellm/config.yaml - subPath: config.yaml - - it: should work with lifecycle hooks - template: deployment.yaml - set: - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - echo "Container stopping" - asserts: - - exists: - path: spec.template.spec.containers[0].lifecycle - - equal: - path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[0] - value: /bin/sh - - equal: - path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[1] - value: -c - - equal: - path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2] - value: echo "Container stopping" - - it: should render background health check settings from proxy_config.general_settings - template: configmap-litellm.yaml - set: - proxy_config.general_settings.background_health_checks: true - proxy_config.general_settings.health_check_interval: 240 - proxy_config.general_settings.health_check_concurrency: 16 - proxy_config.general_settings.health_check_details: false - asserts: - - matchRegex: - path: data["config.yaml"] - pattern: '(?m)^\s*background_health_checks:\s*true$' - - matchRegex: - path: data["config.yaml"] - pattern: '(?m)^\s*health_check_interval:\s*240$' - - matchRegex: - path: data["config.yaml"] - pattern: '(?m)^\s*health_check_concurrency:\s*16$' - - matchRegex: - path: data["config.yaml"] - pattern: '(?m)^\s*health_check_details:\s*false$' - - it: should not render any placeholder models when the user sets proxy_config without model_list - template: configmap-litellm.yaml - set: - proxy_config.general_settings.store_model_in_db: true - asserts: - - notMatchRegex: - path: data["config.yaml"] - pattern: 'model_name:' - - matchRegex: - path: data["config.yaml"] - pattern: 'store_model_in_db: true' - - matchRegex: - path: data["config.yaml"] - pattern: 'master_key: os.environ/PROXY_MASTER_KEY' - - it: should render a user-supplied model_list as the only models - template: configmap-litellm.yaml - set: - proxy_config.model_list: - - model_name: my-model - litellm_params: - model: openai/my-model - api_key: os.environ/OPENAI_API_KEY - asserts: - - matchRegex: - path: data["config.yaml"] - pattern: 'model_list:\n- litellm_params:\n api_key: os.environ/OPENAI_API_KEY\n model: openai/my-model\n model_name: my-model\n$' - - it: should allow overriding liveness, readiness, and startup probes - template: deployment.yaml - set: - livenessProbe: - path: /custom/livez - initialDelaySeconds: 5 - periodSeconds: 15 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - readinessProbe: - path: /custom/readyz - initialDelaySeconds: 10 - periodSeconds: 20 - timeoutSeconds: 6 - successThreshold: 1 - failureThreshold: 6 - startupProbe: - path: /custom/startupz - initialDelaySeconds: 15 - periodSeconds: 25 - timeoutSeconds: 7 - successThreshold: 1 - failureThreshold: 40 - asserts: - - equal: - path: spec.template.spec.containers[0].livenessProbe.httpGet.path - value: /custom/livez - - equal: - path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds - value: 5 - - equal: - path: spec.template.spec.containers[0].readinessProbe.httpGet.path - value: /custom/readyz - - equal: - path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds - value: 6 - - equal: - path: spec.template.spec.containers[0].startupProbe.httpGet.path - value: /custom/startupz - - equal: - path: spec.template.spec.containers[0].startupProbe.failureThreshold - value: 40 - - it: should render container resources from values - template: deployment.yaml - set: - resources: - limits: - cpu: 500m - memory: 2Gi - requests: - cpu: 250m - memory: 1Gi - asserts: - - equal: - path: spec.template.spec.containers[0].resources.limits.cpu - value: 500m - - equal: - path: spec.template.spec.containers[0].resources.limits.memory - value: 2Gi - - equal: - path: spec.template.spec.containers[0].resources.requests.cpu - value: 250m - - equal: - path: spec.template.spec.containers[0].resources.requests.memory - value: 1Gi - - it: should keep default probes and empty resources unchanged - template: deployment.yaml - asserts: - - equal: - path: spec.template.spec.containers[0].livenessProbe.httpGet.path - value: /health/liveliness - - equal: - path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds - value: 0 - - equal: - path: spec.template.spec.containers[0].livenessProbe.periodSeconds - value: 15 - - equal: - path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds - value: 5 - - equal: - path: spec.template.spec.containers[0].livenessProbe.successThreshold - value: 1 - - equal: - path: spec.template.spec.containers[0].livenessProbe.failureThreshold - value: 5 - - equal: - path: spec.template.spec.containers[0].readinessProbe.httpGet.path - value: /health/readiness - - equal: - path: spec.template.spec.containers[0].readinessProbe.initialDelaySeconds - value: 0 - - equal: - path: spec.template.spec.containers[0].readinessProbe.periodSeconds - value: 10 - - equal: - path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds - value: 5 - - equal: - path: spec.template.spec.containers[0].readinessProbe.successThreshold - value: 1 - - equal: - path: spec.template.spec.containers[0].readinessProbe.failureThreshold - value: 3 - - equal: - path: spec.template.spec.containers[0].startupProbe.httpGet.path - value: /health/readiness - - equal: - path: spec.template.spec.containers[0].startupProbe.initialDelaySeconds - value: 0 - - equal: - path: spec.template.spec.containers[0].startupProbe.periodSeconds - value: 10 - - equal: - path: spec.template.spec.containers[0].startupProbe.timeoutSeconds - value: 5 - - equal: - path: spec.template.spec.containers[0].startupProbe.successThreshold - value: 1 - - equal: - path: spec.template.spec.containers[0].startupProbe.failureThreshold - value: 30 - - equal: - path: spec.template.spec.containers[0].resources - value: {} - - it: should be able to set minReadySeconds - template: deployment.yaml - set: - deploymentMinReadySeconds: 5 - asserts: - - equal: - path: spec.minReadySeconds - value: 5 - - it: should have minReadySeconds absent when deploymentMinReadySeconds is not set - template: deployment.yaml - asserts: - - notExists: - path: spec.minReadySeconds - - it: should work with extraInitContainers - template: deployment.yaml - set: - extraInitContainers: - - name: init-test - image: busybox:latest - command: ["echo", "hello"] - asserts: - - contains: - path: spec.template.spec.initContainers - content: - name: init-test - image: busybox:latest - command: ["echo", "hello"] - - it: should support tpl in extraInitContainers - template: deployment.yaml - set: - image: - repository: ghcr.io/berriai/litellm - tag: test - extraInitContainers: - - name: init-tpl - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - command: ["echo", "hello"] - asserts: - - contains: - path: spec.template.spec.initContainers - content: - name: init-tpl - image: "ghcr.io/berriai/litellm:test" - command: ["echo", "hello"] - - it: should work with extraContainers - template: deployment.yaml - set: - extraContainers: - - name: sidecar - image: busybox:latest - asserts: - - contains: - path: spec.template.spec.containers - content: - name: sidecar - image: busybox:latest - - it: should support tpl in extraContainers - template: deployment.yaml - set: - image: - repository: ghcr.io/berriai/litellm - tag: test - extraContainers: - - name: sidecar-tpl - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - asserts: - - contains: - path: spec.template.spec.containers - content: - name: sidecar-tpl - image: "ghcr.io/berriai/litellm:test" - - it: should support tpl in podAnnotations - template: deployment.yaml - set: - image: - repository: ghcr.io/berriai/litellm - tag: test - # Mirrors the real-world scenario this feature unblocks: - # user disables the built-in ConfigMap (and its built-in checksum/config - # annotation) and re-implements checksum/config themselves via tpl. - proxyConfigMap: - create: false - podAnnotations: - checksum/config: "{{ .Values.image.tag }}" - example.com/some-key: "{{ .Values.image.repository }}" - example.com/literal: "plain-string-value" - asserts: - - equal: - path: spec.template.metadata.annotations["checksum/config"] - value: "test" - - equal: - path: spec.template.metadata.annotations["example.com/some-key"] - value: "ghcr.io/berriai/litellm" - - equal: - path: spec.template.metadata.annotations["example.com/literal"] - value: "plain-string-value" diff --git a/helm/litellm-helm/tests/hpa_tests.yaml b/helm/litellm-helm/tests/hpa_tests.yaml deleted file mode 100644 index e446f58c8fe..00000000000 --- a/helm/litellm-helm/tests/hpa_tests.yaml +++ /dev/null @@ -1,144 +0,0 @@ -suite: "hpa" -templates: - - hpa.yaml -tests: - - it: "renders behavior when set" - set: - autoscaling.enabled: true - autoscaling.behavior: - scaleUp: - stabilizationWindowSeconds: 60 - policies: - - type: Pods - value: 2 - periodSeconds: 60 - scaleDown: - stabilizationWindowSeconds: 90 - policies: - - type: Pods - value: 1 - periodSeconds: 60 - asserts: - - isKind: { of: HorizontalPodAutoscaler } - - equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 } - - equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 } - - - it: "does not render behavior when not set" - set: - autoscaling.enabled: true - asserts: - - isKind: { of: HorizontalPodAutoscaler } - - isNull: { path: spec.behavior } - - - it: "scales on cpu at the documented 60 percent by default" - set: - autoscaling.enabled: true - asserts: - - isKind: { of: HorizontalPodAutoscaler } - - equal: { path: "spec.metrics[0].resource.name", value: cpu } - - equal: { path: "spec.metrics[0].resource.target.type", value: Utilization } - - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 60 } - - - it: "does not scale on memory by default" - set: - autoscaling.enabled: true - asserts: - - lengthEqual: { path: spec.metrics, count: 1 } - - - it: "honours an explicit cpu target override" - set: - autoscaling.enabled: true - autoscaling.targetCPUUtilizationPercentage: 75 - asserts: - - equal: { path: "spec.metrics[0].resource.target.averageUtilization", value: 75 } - - - it: "renders a memory metric only when a memory target is set" - set: - autoscaling.enabled: true - autoscaling.targetMemoryUtilizationPercentage: 80 - asserts: - - lengthEqual: { path: spec.metrics, count: 2 } - - equal: { path: "spec.metrics[1].resource.name", value: memory } - - equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 } - - - it: "renders no workload metrics by default" - set: - autoscaling.enabled: true - autoscaling.targetMemoryUtilizationPercentage: 80 - asserts: - - lengthEqual: { path: spec.metrics, count: 2 } - - notContains: { path: spec.metrics, content: { type: Pods }, any: true } - - - it: "adds a requests-per-second Pods metric after the cpu metric" - set: - autoscaling.enabled: true - autoscaling.targetRequestsPerSecond: 90 - asserts: - - lengthEqual: { path: spec.metrics, count: 2 } - - equal: { path: "spec.metrics[0].resource.name", value: cpu } - - equal: - path: "spec.metrics[1]" - value: - type: Pods - pods: - metric: { name: litellm_requests_per_second } - target: { type: AverageValue, averageValue: "90" } - - - it: "adds a tokens-per-second Pods metric on its own" - set: - autoscaling.enabled: true - autoscaling.targetTokensPerSecond: 6M - asserts: - - lengthEqual: { path: spec.metrics, count: 2 } - - equal: - path: "spec.metrics[1]" - value: - type: Pods - pods: - metric: { name: litellm_tokens_per_second } - target: { type: AverageValue, averageValue: "6M" } - - notContains: - path: spec.metrics - content: { type: Pods, pods: { metric: { name: litellm_requests_per_second } } } - any: true - - - it: "renders requests, tokens, cpu and memory metrics together" - set: - autoscaling.enabled: true - autoscaling.targetMemoryUtilizationPercentage: 80 - autoscaling.targetRequestsPerSecond: 90 - autoscaling.targetTokensPerSecond: 6000000 - asserts: - - lengthEqual: { path: spec.metrics, count: 4 } - - equal: { path: "spec.metrics[0].resource.name", value: cpu } - - equal: { path: "spec.metrics[1].resource.name", value: memory } - - equal: { path: "spec.metrics[2].pods.metric.name", value: litellm_requests_per_second } - - equal: { path: "spec.metrics[2].pods.target.averageValue", value: "90" } - - equal: { path: "spec.metrics[3].pods.metric.name", value: litellm_tokens_per_second } - - equal: { path: "spec.metrics[3].pods.target.averageValue", value: "6000000" } - - - it: "scales on workload metrics alone when the cpu target is cleared" - set: - autoscaling.enabled: true - autoscaling.targetCPUUtilizationPercentage: null - autoscaling.targetRequestsPerSecond: 90 - autoscaling.targetTokensPerSecond: 6000000 - asserts: - - lengthEqual: { path: spec.metrics, count: 2 } - - notContains: { path: spec.metrics, content: { type: Resource }, any: true } - - equal: { path: "spec.metrics[0].pods.metric.name", value: litellm_requests_per_second } - - equal: { path: "spec.metrics[1].pods.metric.name", value: litellm_tokens_per_second } - - notMatchRegexRaw: { pattern: per_minute } - - - it: "ignores the per-minute keys, which the chart never shipped" - set: - autoscaling.enabled: true - autoscaling.targetRequestsPerMinute: 5400 - autoscaling.targetTokensPerMinute: 360000000 - asserts: - - lengthEqual: { path: spec.metrics, count: 1 } - - notContains: { path: spec.metrics, content: { type: Pods }, any: true } - - - it: "renders no hpa when autoscaling is disabled" - asserts: - - hasDocuments: { count: 0 } diff --git a/helm/litellm-helm/tests/ingress_tests.yaml b/helm/litellm-helm/tests/ingress_tests.yaml deleted file mode 100644 index aad6ecfcee8..00000000000 --- a/helm/litellm-helm/tests/ingress_tests.yaml +++ /dev/null @@ -1,45 +0,0 @@ -suite: Ingress Configuration Tests -templates: - - ingress.yaml -tests: - - it: should not create Ingress by default - asserts: - - hasDocuments: - count: 0 - - - it: should create Ingress when enabled - set: - ingress.enabled: true - asserts: - - hasDocuments: - count: 1 - - isKind: - of: Ingress - - - it: should add custom labels - set: - ingress.enabled: true - ingress.labels: - custom-label: "true" - another-label: "value" - asserts: - - isKind: - of: Ingress - - equal: - path: metadata.labels.custom-label - value: "true" - - equal: - path: metadata.labels.another-label - value: "value" - - - it: should add annotations - set: - ingress.enabled: true - ingress.annotations: - kubernetes.io/ingress.class: "nginx" - asserts: - - isKind: - of: Ingress - - equal: - path: metadata.annotations["kubernetes.io/ingress.class"] - value: "nginx" diff --git a/helm/litellm-helm/tests/keda_tests.yaml b/helm/litellm-helm/tests/keda_tests.yaml deleted file mode 100644 index c9598646223..00000000000 --- a/helm/litellm-helm/tests/keda_tests.yaml +++ /dev/null @@ -1,106 +0,0 @@ -suite: "keda" -templates: - - keda.yaml -release: - name: rel - namespace: llm -tests: - - it: "renders no scaled object by default" - asserts: - - hasDocuments: { count: 0 } - - - it: "passes user triggers through and adds no prometheus triggers by default" - set: - keda.enabled: true - keda.triggers: - - type: cpu - metricType: Utilization - metadata: { value: "60" } - asserts: - - isKind: { of: ScaledObject } - - equal: - path: spec.triggers - value: - - type: cpu - metricType: Utilization - metadata: { value: "60" } - - - it: "scales on release-wide requests per second divided by the per-replica target" - set: - keda.enabled: true - keda.prometheus.serverAddress: http://prometheus-operated.monitoring.svc:9090 - keda.prometheus.requestsPerSecond: 90 - asserts: - - lengthEqual: { path: spec.triggers, count: 1 } - - equal: - path: "spec.triggers[0]" - value: - type: prometheus - metadata: - serverAddress: http://prometheus-operated.monitoring.svc:9090 - threshold: "90" - query: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm"}[1m])) - - - it: "scales on tokens per second on its own" - set: - keda.enabled: true - keda.prometheus.serverAddress: http://prom:9090 - keda.prometheus.tokensPerSecond: 6000000 - asserts: - - lengthEqual: { path: spec.triggers, count: 1 } - - equal: { path: "spec.triggers[0].type", value: prometheus } - - equal: { path: "spec.triggers[0].metadata.threshold", value: "6000000" } - - equal: - path: "spec.triggers[0].metadata.query" - value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm"}[1m])) - - - it: "appends requests and tokens triggers after user triggers and selects the metrics service job" - set: - keda.enabled: true - metricsServer.enabled: true - keda.triggers: - - type: cpu - metricType: Utilization - metadata: { value: "60" } - keda.prometheus.serverAddress: http://prom:9090 - keda.prometheus.requestsPerSecond: 90 - keda.prometheus.tokensPerSecond: 6000000 - asserts: - - lengthEqual: { path: spec.triggers, count: 3 } - - equal: { path: "spec.triggers[0].type", value: cpu } - - equal: { path: "spec.triggers[1].metadata.threshold", value: "90" } - - equal: - path: "spec.triggers[1].metadata.query" - value: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m])) - - equal: { path: "spec.triggers[2].metadata.threshold", value: "6000000" } - - equal: - path: "spec.triggers[2].metadata.query" - value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m])) - - notMatchRegexRaw: { pattern: "\\* *60|per_minute|PerMinute" } - - - it: "ignores the per-minute keys, which the chart never shipped" - set: - keda.enabled: true - keda.prometheus.serverAddress: http://prom:9090 - keda.prometheus.requestsPerMinute: 5400 - keda.prometheus.tokensPerMinute: 360000000 - asserts: - - isKind: { of: ScaledObject } - - isNullOrEmpty: { path: spec.triggers } - - - it: "refuses a workload target without a prometheus server address" - set: - keda.enabled: true - keda.prometheus.requestsPerSecond: 90 - asserts: - - failedTemplate: - errorMessage: keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set - - - it: "yields to the hpa when both autoscalers are enabled" - set: - autoscaling.enabled: true - keda.enabled: true - keda.prometheus.serverAddress: http://prom:9090 - keda.prometheus.requestsPerSecond: 90 - asserts: - - hasDocuments: { count: 0 } diff --git a/helm/litellm-helm/tests/lens_connection_tests.yaml b/helm/litellm-helm/tests/lens_connection_tests.yaml deleted file mode 100644 index e01e49d0ffc..00000000000 --- a/helm/litellm-helm/tests/lens_connection_tests.yaml +++ /dev/null @@ -1,128 +0,0 @@ -suite: lens connection -templates: - - deployment.yaml - - configmap-litellm.yaml - - ingress.yaml -tests: - - it: connects the proxy to an external Lens - template: deployment.yaml - set: - lens.enabled: true - lens.url: http://lens.lens.svc:4318 - lens.publicUrl: https://lens.example.com/lens-ingest - lens.gatewaySecret.name: lens-gateway - lens.serviceTokenSecret.name: lens-service - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_URL - value: http://lens.lens.svc:4318 - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_PUBLIC_URL - value: https://lens.example.com/lens-ingest - - contains: - path: spec.template.spec.containers[0].env - content: - name: LENS_GATEWAY_SECRET - valueFrom: - secretKeyRef: - name: lens-gateway - key: gateway-secret - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_LENS_SERVICE_TOKEN - valueFrom: - secretKeyRef: - name: lens-service - key: service-token - - - it: requires lens.url when Lens is enabled - template: deployment.yaml - set: - lens.enabled: true - lens.publicUrl: https://lens.example.com/lens-ingest - lens.gatewaySecret.name: lens-gateway - lens.serviceTokenSecret.name: lens-service - asserts: - - failedTemplate: - errorMessage: lens.url is required when lens.enabled is true - - - it: requires lens.publicUrl when Lens is enabled - template: deployment.yaml - set: - lens.enabled: true - lens.url: http://lens.lens.svc:4318 - lens.gatewaySecret.name: lens-gateway - lens.serviceTokenSecret.name: lens-service - asserts: - - failedTemplate: - errorMessage: lens.publicUrl is required when lens.enabled is true - - - it: requires lens.gatewaySecret.name when Lens is enabled - template: deployment.yaml - set: - lens.enabled: true - lens.url: http://lens.lens.svc:4318 - lens.publicUrl: https://lens.example.com/lens-ingest - lens.serviceTokenSecret.name: lens-service - asserts: - - failedTemplate: - errorMessage: lens.gatewaySecret.name is required when lens.enabled is true - - - it: requires lens.serviceTokenSecret.name when Lens is enabled - template: deployment.yaml - set: - lens.enabled: true - lens.url: http://lens.lens.svc:4318 - lens.publicUrl: https://lens.example.com/lens-ingest - lens.gatewaySecret.name: lens-gateway - asserts: - - failedTemplate: - errorMessage: lens.serviceTokenSecret.name is required when lens.enabled is true - - - it: routes /lens-ingest to the named Lens service ahead of the host paths - template: ingress.yaml - set: - ingress.enabled: true - lens.enabled: true - lens.url: http://lens.lens.svc:4318 - lens.publicUrl: https://lens.example.com/lens-ingest - lens.gatewaySecret.name: lens-gateway - lens.serviceTokenSecret.name: lens-service - lens.service.name: existing-lens - asserts: - - equal: - path: spec.rules[0].http.paths[0].path - value: /lens-ingest - - contains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - pathType: Prefix - backend: - service: - name: existing-lens - port: - number: 4318 - - - it: leaves /lens-ingest off the ingress without a Lens service name - template: ingress.yaml - set: - ingress.enabled: true - lens.enabled: true - lens.url: http://lens.lens.svc:4318 - lens.publicUrl: https://lens.example.com/lens-ingest - lens.gatewaySecret.name: lens-gateway - lens.serviceTokenSecret.name: lens-service - asserts: - - exists: - path: spec.rules[0].http.paths - - notContains: - path: spec.rules[0].http.paths - content: - path: /lens-ingest - any: true diff --git a/helm/litellm-helm/tests/masterkey-secret_tests.yaml b/helm/litellm-helm/tests/masterkey-secret_tests.yaml deleted file mode 100644 index 296f26755b8..00000000000 --- a/helm/litellm-helm/tests/masterkey-secret_tests.yaml +++ /dev/null @@ -1,71 +0,0 @@ -suite: test masterkey secret -templates: - - secret-masterkey.yaml -tests: - - it: should create a secret if masterkeySecretName is not set. should start with sk-xxxx (base64 encoded as c2st*) - template: secret-masterkey.yaml - set: - masterkeySecretName: "" - asserts: - - isKind: - of: Secret - - matchRegex: - path: data.masterkey - pattern: ^c2st - # Note: The masterkey is generated as "sk-<18-random-chars>" in plain text, - # but stored as base64 encoded in Kubernetes secret (requirement). - # "sk-" base64 encodes to "c2st", so we check for "^c2st" pattern. - - it: should reuse the master key already stored in the cluster instead of generating a new one on upgrade - template: secret-masterkey.yaml - set: - masterkeySecretName: "" - kubernetesProvider: - scheme: - "v1/Secret": - gvr: - version: "v1" - resource: "secrets" - namespaced: true - objects: - - kind: Secret - apiVersion: v1 - metadata: - name: RELEASE-NAME-litellm-masterkey - namespace: NAMESPACE - data: - masterkey: c2stZXhpc3Rpbmcta2V5 - asserts: - - equal: - path: data.masterkey - value: c2stZXhpc3Rpbmcta2V5 - - it: should let an explicit masterkey value override the one already stored in the cluster - template: secret-masterkey.yaml - set: - masterkeySecretName: "" - masterkey: sk-explicit - kubernetesProvider: - scheme: - "v1/Secret": - gvr: - version: "v1" - resource: "secrets" - namespaced: true - objects: - - kind: Secret - apiVersion: v1 - metadata: - name: RELEASE-NAME-litellm-masterkey - namespace: NAMESPACE - data: - masterkey: c2stZXhpc3Rpbmcta2V5 - asserts: - - equal: - path: data.masterkey - value: c2stZXhwbGljaXQ= - - it: should not create a secret if masterkeySecretName is set - template: secret-masterkey.yaml - set: - masterkeySecretName: my-secret - asserts: - - hasDocuments: - count: 0 diff --git a/helm/litellm-helm/tests/metrics_server_tests.yaml b/helm/litellm-helm/tests/metrics_server_tests.yaml deleted file mode 100644 index 085d69ac640..00000000000 --- a/helm/litellm-helm/tests/metrics_server_tests.yaml +++ /dev/null @@ -1,106 +0,0 @@ -suite: separate metrics server -templates: - - configmap-litellm.yaml - - deployment.yaml - - service.yaml - - service-metrics.yaml - - servicemonitor.yaml -tests: - - it: should not expose a metrics port or PROMETHEUS_METRICS_PORT by default - asserts: - - notContains: - path: spec.template.spec.containers[0].ports - content: - name: metrics - any: true - template: deployment.yaml - - notContains: - path: spec.template.spec.containers[0].env - content: - name: PROMETHEUS_METRICS_PORT - any: true - template: deployment.yaml - - lengthEqual: - path: spec.ports - count: 1 - template: service.yaml - - hasDocuments: - count: 0 - template: service-metrics.yaml - - - it: should scrape the proxy port when the metrics server is disabled - template: servicemonitor.yaml - set: - serviceMonitor.enabled: true - asserts: - - equal: - path: spec.endpoints[0].port - value: http - - - it: should wire the separate metrics server through container, a ClusterIP metrics service and servicemonitor - set: - metricsServer.enabled: true - metricsServer.port: 4101 - serviceMonitor.enabled: true - service.type: LoadBalancer - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: PROMETHEUS_METRICS_PORT - value: "4101" - template: deployment.yaml - - contains: - path: spec.template.spec.containers[0].ports - content: - name: metrics - containerPort: 4101 - protocol: TCP - template: deployment.yaml - - lengthEqual: - path: spec.ports - count: 1 - template: service.yaml - - equal: - path: spec.type - value: LoadBalancer - template: service.yaml - - equal: - path: metadata.name - value: RELEASE-NAME-litellm-metrics - template: service-metrics.yaml - - equal: - path: spec.type - value: ClusterIP - template: service-metrics.yaml - - equal: - path: spec.ports - value: - - port: 4101 - targetPort: metrics - protocol: TCP - name: metrics - template: service-metrics.yaml - - equal: - path: spec.selector - value: - app.kubernetes.io/name: litellm - app.kubernetes.io/instance: RELEASE-NAME - template: service-metrics.yaml - - equal: - path: spec.endpoints[0].port - value: metrics - template: servicemonitor.yaml - - equal: - path: spec.endpoints[0].path - value: /metrics/ - template: servicemonitor.yaml - - - it: should reject a metrics port equal to the proxy port - template: deployment.yaml - set: - metricsServer.enabled: true - metricsServer.port: 4000 - asserts: - - failedTemplate: - errorMessage: metricsServer.port must differ from service.port diff --git a/helm/litellm-helm/tests/migrations-job_tests.yaml b/helm/litellm-helm/tests/migrations-job_tests.yaml deleted file mode 100644 index bd5dd457f95..00000000000 --- a/helm/litellm-helm/tests/migrations-job_tests.yaml +++ /dev/null @@ -1,432 +0,0 @@ -suite: test migrations job -templates: - - migrations-job.yaml -tests: - - it: should work with envVars - template: migrations-job.yaml - set: - envVars: - TEST_ENV_VAR: "test_value" - ANOTHER_VAR: "another_value" - migrationJob: - enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: TEST_ENV_VAR - value: "test_value" - - contains: - path: spec.template.spec.containers[0].env - content: - name: ANOTHER_VAR - value: "another_value" - - - it: should work with extraEnvVars - template: migrations-job.yaml - set: - extraEnvVars: - - name: EXTRA_ENV_VAR - valueFrom: - fieldRef: - fieldPath: metadata.labels['env'] - - name: SIMPLE_EXTRA_VAR - value: "simple_value" - migrationJob: - enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: EXTRA_ENV_VAR - valueFrom: - fieldRef: - fieldPath: metadata.labels['env'] - - contains: - path: spec.template.spec.containers[0].env - content: - name: SIMPLE_EXTRA_VAR - value: "simple_value" - - - it: should work with both envVars and extraEnvVars - template: migrations-job.yaml - set: - envVars: - ENV_VAR: "env_var_value" - extraEnvVars: - - name: EXTRA_ENV_VAR - value: "extra_env_var_value" - migrationJob: - enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: ENV_VAR - value: "env_var_value" - - contains: - path: spec.template.spec.containers[0].env - content: - name: EXTRA_ENV_VAR - value: "extra_env_var_value" - - - it: should not render when migrations job is disabled - template: migrations-job.yaml - set: - migrationJob: - enabled: false - asserts: - - hasDocuments: - count: 0 - - - it: should still include default env vars - template: migrations-job.yaml - set: - envVars: - CUSTOM_VAR: "custom_value" - migrationJob: - enabled: true - db: - useExisting: true - endpoint: "test-db" - database: "testdb" - url: "postgresql://user:pass@test-db:5432/testdb" - secret: - name: "test-secret" - usernameKey: "username" - passwordKey: "password" - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: DISABLE_SCHEMA_UPDATE - value: "false" - - contains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_HOST - value: "test-db" - - contains: - path: spec.template.spec.containers[0].env - content: - name: CUSTOM_VAR - value: "custom_value" - - - it: should override a user-supplied DISABLE_SCHEMA_UPDATE so the Job always migrates - template: migrations-job.yaml - set: - envVars: - DISABLE_SCHEMA_UPDATE: "true" - migrationJob: - enabled: true - asserts: - # The Job is what owns the schema, so it renders its own - # DISABLE_SCHEMA_UPDATE=false after envVars and extraEnvVars. Kubernetes - # takes the last value for a duplicated name, so the user's "true" cannot - # leave the schema unmigrated. Skipping migrations is migrationJob.enabled. - - equal: - path: spec.template.spec.containers[0].env[-1] - value: - name: DISABLE_SCHEMA_UPDATE - value: "false" - - - it: should not include DATABASE_URL when deployStandalone is false - template: migrations-job.yaml - set: - migrationJob: - enabled: true - db: - deployStandalone: false - useExisting: false - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: DATABASE_URL - - - it: should use default service account for helm hooks when serviceAccount.create is true - template: migrations-job.yaml - set: - migrationJob: - enabled: true - hooks: - helm: - enabled: true - serviceAccount: - create: true - asserts: - - equal: - path: spec.template.spec.serviceAccountName - value: default - - - it: should use migrationJob.serviceAccountName override for helm hooks when serviceAccount.create is true - template: migrations-job.yaml - set: - migrationJob: - enabled: true - serviceAccountName: migration-sa - hooks: - helm: - enabled: true - serviceAccount: - create: true - asserts: - - equal: - path: spec.template.spec.serviceAccountName - value: migration-sa - - - it: should use chart service account when helm hooks are disabled - template: migrations-job.yaml - set: - migrationJob: - enabled: true - hooks: - helm: - enabled: false - serviceAccount: - create: true - name: my-custom-sa - asserts: - - equal: - path: spec.template.spec.serviceAccountName - value: my-custom-sa - - - it: should use pre-existing service account when helm hooks are enabled but serviceAccount.create is false - template: migrations-job.yaml - set: - migrationJob: - enabled: true - hooks: - helm: - enabled: true - serviceAccount: - create: false - name: pre-existing-sa - asserts: - - equal: - path: spec.template.spec.serviceAccountName - value: pre-existing-sa - - it: should work with extraInitContainers - template: migrations-job.yaml - set: - migrationJob: - enabled: true - extraInitContainers: - - name: init-test - image: busybox:latest - command: ["echo", "hello"] - asserts: - - contains: - path: spec.template.spec.initContainers - content: - name: init-test - image: busybox:latest - command: ["echo", "hello"] - - it: should support tpl in extraInitContainers - template: migrations-job.yaml - set: - image: - repository: ghcr.io/berriai/litellm - tag: test - migrationJob: - enabled: true - extraInitContainers: - - name: init-tpl - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - command: ["echo", "hello"] - asserts: - - contains: - path: spec.template.spec.initContainers - content: - name: init-tpl - image: "ghcr.io/berriai/litellm:test" - command: ["echo", "hello"] - - it: should work with extraContainers - template: migrations-job.yaml - set: - migrationJob: - enabled: true - extraContainers: - - name: sidecar - image: busybox:latest - asserts: - - contains: - path: spec.template.spec.containers - content: - name: sidecar - image: busybox:latest - - it: should support tpl in extraContainers - template: migrations-job.yaml - set: - image: - repository: ghcr.io/berriai/litellm - tag: test - migrationJob: - enabled: true - extraContainers: - - name: sidecar-tpl - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - asserts: - - contains: - path: spec.template.spec.containers - content: - name: sidecar-tpl - image: "ghcr.io/berriai/litellm:test" - - it: should render the pod-level securityContext from podSecurityContext - template: migrations-job.yaml - set: - migrationJob: - enabled: true - podSecurityContext: - fsGroup: 10000 - runAsUser: 10000 - runAsNonRoot: true - asserts: - - equal: - path: spec.template.spec.securityContext - value: - fsGroup: 10000 - runAsUser: 10000 - runAsNonRoot: true - - it: should keep the pod-level and container-level securityContext separate - template: migrations-job.yaml - set: - migrationJob: - enabled: true - podSecurityContext: - fsGroup: 10000 - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - asserts: - - equal: - path: spec.template.spec.securityContext - value: - fsGroup: 10000 - - equal: - path: spec.template.spec.containers[0].securityContext - value: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - - it: should schedule onto the same nodes as the gateway - template: migrations-job.yaml - set: - migrationJob: - enabled: true - nodeSelector: - karpenter.sh/nodepool: litellm-e2e - tolerations: - - key: workload - operator: Equal - value: litellm-e2e - effect: NoSchedule - asserts: - - equal: - path: spec.template.spec.nodeSelector - value: - karpenter.sh/nodepool: litellm-e2e - - equal: - path: spec.template.spec.tolerations - value: - - key: workload - operator: Equal - value: litellm-e2e - effect: NoSchedule - - - it: bounds the Job with a deadline by default, so a blocked migration cannot stall the release forever - set: - migrationJob: - enabled: true - asserts: - - equal: - path: spec.activeDeadlineSeconds - value: 1800 - - - it: honours an operator-supplied deadline - set: - migrationJob: - enabled: true - activeDeadlineSeconds: 600 - asserts: - - equal: - path: spec.activeDeadlineSeconds - value: 600 - - - it: omits the deadline entirely when it is nulled out, restoring the unbounded behaviour - set: - migrationJob: - enabled: true - activeDeadlineSeconds: null - asserts: - - notExists: - path: spec.activeDeadlineSeconds - - - it: should set custom jobLabels and jobAnnotations on Job metadata - template: migrations-job.yaml - set: - migrationJob: - enabled: true - jobLabels: - environment: production - team: platform - jobAnnotations: - example.com/cost-center: "1234" - asserts: - - equal: - path: metadata.labels.environment - value: production - - equal: - path: metadata.labels.team - value: platform - - equal: - path: metadata.annotations['example.com/cost-center'] - value: "1234" - - - it: should set custom podLabels on Pod template - template: migrations-job.yaml - set: - migrationJob: - enabled: true - podLabels: - custom.io/pod-role: migration - asserts: - - equal: - path: spec.template.metadata.labels['custom.io/pod-role'] - value: migration - - - it: should override container command and args - template: migrations-job.yaml - set: - migrationJob: - enabled: true - command: - - sh - args: - - -c - - echo migrating - asserts: - - equal: - path: spec.template.spec.containers[0].command - value: - - sh - - equal: - path: spec.template.spec.containers[0].args - value: - - -c - - echo migrating - - - it: should clear container args when only command is specified - template: migrations-job.yaml - set: - migrationJob: - enabled: true - command: - - sh - asserts: - - equal: - path: spec.template.spec.containers[0].command - value: - - sh - - equal: - path: spec.template.spec.containers[0].args - value: [] diff --git a/helm/litellm-helm/tests/pdb_tests.yaml b/helm/litellm-helm/tests/pdb_tests.yaml deleted file mode 100644 index 5e042e80bd3..00000000000 --- a/helm/litellm-helm/tests/pdb_tests.yaml +++ /dev/null @@ -1,45 +0,0 @@ -suite: "pdb enabled" -templates: - - poddisruptionbudget.yaml -tests: - - it: "renders a PDB with maxUnavailable=1" - set: - pdb.enabled: true - pdb.maxUnavailable: 1 - asserts: - - hasDocuments: { count: 1 } - - isKind: { of: PodDisruptionBudget } - - equal: { path: apiVersion, value: policy/v1 } - - equal: { path: spec.maxUnavailable, value: 1 } - - equal: - path: spec.selector.matchLabels - value: - app.kubernetes.io/name: litellm - app.kubernetes.io/instance: RELEASE-NAME - ---- -suite: "pdb disabled" -templates: - - poddisruptionbudget.yaml -tests: - - it: "does not render when disabled" - set: - pdb.enabled: false - asserts: - - hasDocuments: { count: 0 } - ---- -suite: "pdb minAvailable precedence" -templates: - - poddisruptionbudget.yaml -tests: - - it: "uses minAvailable when both are set" - set: - pdb.enabled: true - pdb.minAvailable: "50%" - pdb.maxUnavailable: 1 - asserts: - - isKind: { of: PodDisruptionBudget } - - equal: { path: apiVersion, value: policy/v1 } - - equal: { path: spec.minAvailable, value: "50%" } - - isNull: { path: spec.maxUnavailable } diff --git a/helm/litellm-helm/tests/service_tests.yaml b/helm/litellm-helm/tests/service_tests.yaml deleted file mode 100644 index 43ed0180bc8..00000000000 --- a/helm/litellm-helm/tests/service_tests.yaml +++ /dev/null @@ -1,116 +0,0 @@ -suite: Service Configuration Tests -templates: - - service.yaml -tests: - - it: should create a default ClusterIP service - template: service.yaml - asserts: - - isKind: - of: Service - - equal: - path: spec.type - value: ClusterIP - - equal: - path: spec.ports[0].port - value: 4000 - - equal: - path: spec.ports[0].targetPort - value: http - - equal: - path: spec.ports[0].protocol - value: TCP - - equal: - path: spec.ports[0].name - value: http - - isNull: - path: spec.loadBalancerClass - - - it: should create a NodePort service when specified - template: service.yaml - set: - service.type: NodePort - asserts: - - isKind: - of: Service - - equal: - path: spec.type - value: NodePort - - isNull: - path: spec.loadBalancerClass - - - it: should create a LoadBalancer service when specified - template: service.yaml - set: - service.type: LoadBalancer - asserts: - - isKind: - of: Service - - equal: - path: spec.type - value: LoadBalancer - - isNull: - path: spec.loadBalancerClass - - - it: should add loadBalancerClass when specified with LoadBalancer type - template: service.yaml - set: - service.type: LoadBalancer - service.loadBalancerClass: tailscale - asserts: - - isKind: - of: Service - - equal: - path: spec.type - value: LoadBalancer - - equal: - path: spec.loadBalancerClass - value: tailscale - - - it: should not add loadBalancerClass when specified with ClusterIP type - template: service.yaml - set: - service.type: ClusterIP - service.loadBalancerClass: tailscale - asserts: - - isKind: - of: Service - - equal: - path: spec.type - value: ClusterIP - - isNull: - path: spec.loadBalancerClass - - - it: should use custom port when specified - template: service.yaml - set: - service.port: 8080 - asserts: - - equal: - path: spec.ports[0].port - value: 8080 - - - it: should add service annotations when specified - template: service.yaml - set: - service.annotations: - cloud.google.com/load-balancer-type: "Internal" - service.beta.kubernetes.io/aws-load-balancer-internal: "true" - asserts: - - isKind: - of: Service - - equal: - path: metadata.annotations - value: - cloud.google.com/load-balancer-type: "Internal" - service.beta.kubernetes.io/aws-load-balancer-internal: "true" - - - it: should use the correct selector labels - template: service.yaml - asserts: - - isNotNull: - path: spec.selector - - equal: - path: spec.selector - value: - app.kubernetes.io/name: litellm - app.kubernetes.io/instance: RELEASE-NAME diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml deleted file mode 100644 index f51f8bdd173..00000000000 --- a/helm/litellm-helm/values.yaml +++ /dev/null @@ -1,682 +0,0 @@ -# Default values for litellm. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 -liteadmin: - enabled: false - existingSecret: "" - gatewayUrl: "" - model: "" - readOnly: false - storageSize: 1Gi - storageClassName: "" - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - memory: 1Gi -# numWorkers: 2 - -image: - # Bundles the prisma CLI and engines, which is what lets the migrations job - # and the proxy's own schema check run without network access. - repository: ghcr.io/berriai/litellm - pullPolicy: Always - # Overrides the image tag whose default is the chart appVersion. - # tag: "latest" - tag: "" - -imagePullSecrets: [] -nameOverride: "litellm" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -# annotations for litellm deployment -deploymentAnnotations: {} -deploymentLabels: {} -deploymentMinReadySeconds: 0 - -# annotations for litellm pods -podAnnotations: {} -podLabels: {} - -# -- Deployment strategy configuration -# Example: -# type: RollingUpdate -# rollingUpdate: -# maxUnavailable: 0 -# maxSurge: 1 -strategy: {} - -terminationGracePeriodSeconds: 90 -topologySpreadConstraints: - [] - # - maxSkew: 1 - # topologyKey: kubernetes.io/hostname - # whenUnsatisfiable: DoNotSchedule - # labelSelector: - # matchLabels: - # app: litellm - -# At the time of writing, the litellm docker image requires write access to the -# filesystem on startup so that prisma can install some dependencies. -podSecurityContext: {} -securityContext: - {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: false - # runAsNonRoot: true - # runAsUser: 1000 - -# A list of Kubernetes Secret objects that will be exported to the LiteLLM proxy -# pod as environment variables. These secrets can then be referenced in the -# configuration file (or "litellm" ConfigMap) with `os.environ/` -environmentSecrets: - [] - # - litellm-env-secret - -# A list of Kubernetes ConfigMap objects that will be exported to the LiteLLM proxy -# pod as environment variables. The ConfigMap kv-pairs can then be referenced in the -# configuration file (or "litellm" ConfigMap) with `os.environ/` -environmentConfigMaps: - [] - # - litellm-env-configmap - -service: - type: ClusterIP - port: 4000 - # If service type is `LoadBalancer` you can - # optionally specify loadBalancerClass - # loadBalancerClass: tailscale - -# Probes for LiteLLM gateway container -livenessProbe: - path: /health/liveliness - initialDelaySeconds: 0 - periodSeconds: 15 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - -readinessProbe: - path: /health/readiness - initialDelaySeconds: 0 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - -startupProbe: - path: /health/readiness - initialDelaySeconds: 0 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 30 - -ingress: - enabled: false - className: "nginx" - labels: {} - annotations: - {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: api.example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -# masterkey: changeit - -# if set, use this secret for the master key; otherwise, autogenerate a new one -masterkeySecretName: "" - -# if set, use this secret key for the master key; otherwise, use the default key -masterkeySecretKey: "" - -# Optional: enterprise billable-request metering. When enabled, the proxy counts -# successful requests to inference, MCP, and A2A endpoints and pushes them to -# LiteLLM's collector over mutual TLS. Requires an enterprise license. -# The client certificate identifies the deployment, so it is mounted read-only -# from an existing Secret and never passed through the environment. -billingMetrics: - enabled: false - endpoint: https://telemetry.litellm.ai # collector to push the counter to - secretName: litellm-billing-metrics-mtls # existing Secret holding tls.crt and tls.key - # Only for private or test collectors whose server certificate is not on the - # public web PKI. The production collector needs no CA override. - caSecretName: "" # existing Secret holding ca.crt - exportIntervalMs: "" # push cadence; the proxy defaults to 60000 - -proxyConfigMap: - # when true, creates a new configmap - create: true - # if create is false and name is set, use existing ConfigMap - # create: false - # name: "" - # key: "config.yaml" - -# The elements within proxy_config are rendered as config.yaml for the proxy -# Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml -# Reference: https://docs.litellm.ai/docs/proxy/configs -proxy_config: - model_list: [] - general_settings: - master_key: os.environ/PROXY_MASTER_KEY - -# Serve Prometheus /metrics from a separate process (PROMETHEUS_METRICS_PORT) -# so a scrape never runs on an inference worker. Adds a `metrics` port to the -# container and a dedicated ClusterIP `-metrics` Service, and the -# ServiceMonitor scrapes it instead of the proxy port. The separate port has -# no virtual-key auth: keep it off public ingress. Needs the proxy image -# v1.101.0 or newer. -metricsServer: - enabled: false - port: 4001 - -# Opt-in sidecar that runs the post-response spend pipeline (cost calculation, -# spend logs, spend counters, budget reservation reconciliation) so the proxy's -# uvicorn workers only serialise a compact typed event and go back to serving -# inference. Same image and tag as the proxy, second container in the same pod, -# fed over loopback (a unix socket on a shared emptyDir, or 127.0.0.1 TCP). It -# reuses the pod's in-container pgbouncer (db.connectionPool) and the same Redis -# spend transaction buffer, so the per-pod DB connection budget is unchanged. -# Delivery is at-most-once inside the pod: events already handed to the sidecar -# are lost if it crashes before writing them; events the workers could not hand -# over follow onUnavailable. Both containers drain on SIGTERM within -# terminationGracePeriodSeconds -collector: - enabled: false - # unix:////.sock (the becomes a shared emptyDir) or tcp://127.0.0.1: - address: unix:///var/run/litellm/collector.sock - # Events each uvicorn worker holds in memory while the sidecar is slow or restarting - bufferSize: 1000 - # fallback: run the pipeline in the worker when the sidecar is unreachable or the - # buffer is full (spend stays exact, that request costs proxy CPU again) - # drop: count and discard the event instead (spend under-reports) - onUnavailable: fallback - # How long the workers keep pushing buffered events on shutdown, and how long the - # sidecar keeps serving its open connections after SIGTERM - drainTimeoutSeconds: 10 - command: - - python - - -m - - litellm.proxy.collector - # Sized independently of the proxy container; the pipeline is CPU bound - resources: {} - # requests: - # cpu: 500m - # memory: 1Gi - # limits: - # cpu: "1" - # memory: 2Gi - # When autoscaling.enabled, swap the pod-wide cpu Resource metric for an - # autoscaling/v2 ContainerResource metric on the proxy container only, so the - # sidecar's CPU never scales inference replicas. Needs Kubernetes 1.30+ (or the - # HPAContainerMetrics feature gate on 1.27 to 1.29) - scaleOnProxyContainerCpu: false - -resources: - {} - # Unset by default so the chart installs on small clusters such as Minikube, and so an - # upgrade never leaves a running pod Pending. Production deployments should set these. - # A proxy at DB-connected steady state needs about 1 CPU and 4Gi of memory per worker; - # sizing below that gets the pod OOMKilled once traffic and DB connections ramp up. - # Scale both figures with --num_workers, then uncomment the lines below and remove the - # curly braces after 'resources:'. See "Recommended Machine Specifications" in - # https://docs.litellm.ai/docs/proxy/prod. - # requests: - # cpu: "1" - # memory: 4Gi - # limits: - # cpu: "1" - # memory: 4Gi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - # 60 is the documented recommendation. See "Recommended Machine Specifications" - # in https://docs.litellm.ai/docs/proxy/prod. A new replica clears the startupProbe - # above only after up to failureThreshold x periodSeconds = 300 seconds, so a target - # high enough to trip near saturation adds capacity minutes after it was needed. - targetCPUUtilizationPercentage: 60 - # Deliberately left unset rather than given a value. The prisma query engine's - # resident memory is a high-water mark that ratchets to the pod's worst-ever write - # and is never returned, so a memory target reads the largest write a pod ever did - # rather than what it is doing now, and replicas ratchet up without scaling back in. - # Memory is a floor to provision under 'resources', not a signal to scale on. - # targetMemoryUtilizationPercentage: 80 - # behavior: {} - # Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics - # named `litellm_requests_per_second` and `litellm_tokens_per_second` with an - # AverageValue target, alongside whichever resource targets are set (the HPA - # follows the metric asking for the most replicas). A Prometheus Adapter must - # serve those two names on custom.metrics.k8s.io from the proxy's counters, - # grouped by the scrape target's `pod` label (enable serviceMonitor below so - # every pod is scraped on its own): - # litellm_requests_per_second: - # sum(rate(litellm_proxy_total_requests_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>) - # litellm_tokens_per_second: - # sum(rate(litellm_total_tokens_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>) - # rate() over [1m] is already per second, so no `* 60`. How fast the HPA - # reacts is set by that window, the scrape interval and the HPA sync period - # (15s by default), not by the unit: keep serviceMonitor.interval at 15s or - # faster so a 1m window holds at least 4 samples. averageValue takes SI - # suffixes, so "6M" is six million tokens per second per pod. Tokens are - # counted when a response completes, so TPS trails long streams. - targetRequestsPerSecond: "" - targetTokensPerSecond: "" - -# Autoscaling with keda is mutually exclusive with hpa -keda: - enabled: false - minReplicas: 1 - maxReplicas: 100 - pollingInterval: 30 - cooldownPeriod: 300 - # fallback: - # failureThreshold: 3 - # replicas: 11 - restoreToOriginalReplicaCount: false - scaledObject: - annotations: {} - triggers: [] - # - type: prometheus - # metadata: - # serverAddress: http://:9090 - # metricName: http_requests_total - # threshold: '100' - # query: sum(rate(http_requests_total{deployment="my-deployment"}[2m])) - # First-class Prometheus triggers on the proxy's own request and token - # counters, appended to `triggers`. Each target is the per-second load one - # replica should carry: KEDA divides the release-wide - # `sum(rate([1m]))` by it to pick the replica count. Thresholds - # are plain numbers (KEDA parses them as floats, no SI suffixes). The - # queries select samples by the release namespace and the `job` label the - # chart's ServiceMonitor produces (the metrics Service name), so enable - # serviceMonitor below together with metricsServer: the http port serves - # /metrics/ behind virtual-key auth and answers an unauthenticated scrape - # with 401. Reaction time comes from the [1m] window, the scrape interval - # and pollingInterval above, so keep both at 15s or faster. Tokens are - # counted at completion, so TPS trails long streams. serverAddress is - # required once either target is set. - prometheus: - serverAddress: "" - requestsPerSecond: "" - tokensPerSecond: "" - behavior: {} - # scaleDown: - # stabilizationWindowSeconds: 300 - # policies: - # - type: Pods - # value: 1 - # periodSeconds: 180 - # scaleUp: - # stabilizationWindowSeconds: 300 - # policies: - # - type: Pods - # value: 2 - # periodSeconds: 60 - -# Additional volumes on the output Deployment definition. -volumes: [] -# - name: foo -# secret: -# secretName: mysecret -# optional: false - -# Additional volumeMounts on the output Deployment definition. -volumeMounts: [] -# - name: foo -# mountPath: "/etc/foo" -# readOnly: true - -nodeSelector: {} - -tolerations: [] - -affinity: {} - -db: - # Use an existing postgres server/cluster - useExisting: false - - # How to connect to the existing postgres server/cluster - endpoint: localhost - database: litellm - url: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME) - # Authenticate with a short-lived token instead of a password. useIAMAuth is - # AWS RDS / Aurora IAM database authentication (the pod needs AWS credentials, - # e.g. IRSA or EKS Pod Identity, and the database user needs GRANT rds_iam); - # useAzureEntraAuth is Azure Database for PostgreSQL with a Microsoft Entra ID - # identity (workload identity). Mutually exclusive, and both need - # useExisting: true with deployStandalone: false. The chart then sets - # IAM_TOKEN_DB_AUTH or AZURE_POSTGRESQL_AUTH, emits DATABASE_HOST, DATABASE_PORT, - # DATABASE_USERNAME and DATABASE_NAME and skips DATABASE_PASSWORD and db.url: - # the proxy, the migrations job and the collector sidecar each mint their own - # token and build DATABASE_URL from those parts. Set DATABASE_SSLMODE / - # DATABASE_SSLROOTCERT under envVars for TLS options you would otherwise put - # in db.url. - useIAMAuth: false - useAzureEntraAuth: false - # Postgres port, used to build the token-auth URL (password auth takes the - # port from db.url) - port: 5432 - # Postgres schema for the LiteLLM tables, rendered as DATABASE_SCHEMA under - # token auth only (password auth carries it as ?schema= in db.url) - schema: "" - # Database user as a plain value. Empty means read it from - # db.secret.usernameKey. With token auth the user name is the only credential - # the chart needs, so setting it here removes the need for db.secret entirely. - username: "" - secret: - name: postgres - usernameKey: username - passwordKey: password - # Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint - endpointKey: "" - # Optional: when set, DATABASE_URL_READ_REPLICA will be sourced from this - # secret key instead of db.readReplicaUrl. Prefer this over the plain - # value: read-replica URLs typically embed credentials, and a value - # written to db.readReplicaUrl ends up visible in the rendered pod spec - # and the Helm release secret. - readReplicaUrlKey: "" - # Optional: when set, a DATABASE_READER_HOST env var is sourced from this - # secret key, so db.readReplicaUrl can compose the reader URL from - # individual secret components, e.g. - # postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME) - # Use this when your secret store holds the bare reader hostname rather - # than a full connection URL. Only takes effect when readReplicaUrl is - # set; ignored when readReplicaUrlKey is set. - readReplicaEndpointKey: "" - - # Optional read-replica routing. When set, the proxy sends read-only - # queries (find_*, count, group_by, query_raw/_first) to this URL while - # writes continue to go to db.url. Useful for Aurora-style clusters with - # separate reader/writer endpoints. Leave empty to keep single-DB behavior. - # When IAM_TOKEN_DB_AUTH is enabled, the reader URL is auto-refreshed - # alongside the writer (host/port/user/db are parsed from this URL once - # at startup; only the IAM token rotates). - # - # If the URL embeds credentials, prefer db.secret.readReplicaUrlKey over - # this field — the plain value is rendered into the pod spec and the - # Helm release secret. This field is intended for credential-less URLs - # only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime). - readReplicaUrl: "" - - # In-container connection pool (PgBouncer, transaction mode) shared by every - # worker in the pod. Without it each --num_workers worker opens its own - # connection_limit connections to Postgres, so a pod's footprint against the - # database's connection ceiling is workers x connection_limit and grows with - # every replica. With it, the pod holds at most maxDbConnections upstream - # connections no matter how many workers run; the workers connect to the pool - # over loopback, with no extra network hop. Migrations still go straight to - # Postgres. Starting profile for numWorkers: 4 is maxDbConnections: 20, so - # a database with a 5000-connection ceiling fits roughly 200 replicas. - connectionPool: - enabled: false - maxDbConnections: 20 - maxClientConn: 1000 - - # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster. - # The Stackgres Operator must already be installed within the target - # Kubernetes cluster. - # TODO: Stackgres deployment currently unsupported - useStackgresOperator: false - - # Use the Postgres Helm chart to create a single node, stand alone postgres - # instance. See the "postgresql" top level key for additional configuration. - deployStandalone: true - -# Lifecycle hooks for the LiteLLM container -# -# Prefer the native /health/drain preStop hook over a fixed `sleep`: it marks -# the pod NotReady and blocks only until in-flight requests actually finish -# (bounded by GRACEFUL_SHUTDOWN_TIMEOUT, default 30s), instead of always -# waiting the worst-case duration. After SIGTERM the server waits up to -# GRACEFUL_SHUTDOWN_TIMEOUT again for connections still open and then cancels -# them, so set terminationGracePeriodSeconds a few seconds above twice -# GRACEFUL_SHUTDOWN_TIMEOUT to leave room for teardown before SIGKILL. -# -# /health/drain is off by default; enable it with -# general_settings.enable_drain_endpoint: true. The kubelet calls preStop -# hooks without proxy credentials, so when the health port is reachable from -# other pods (the common case) also set -# general_settings.drain_endpoint_token (or the DRAIN_ENDPOINT_TOKEN env -# var) and send the same value on the X-Drain-Token header from the hook. -# Calls missing/wrong the token get a 401 and have no side effect. -# Example: -# lifecycle: -# preStop: -# httpGet: -# path: /health/drain -# port: 4000 -# httpHeaders: -# - name: X-Drain-Token -# value: -lifecycle: {} - -# Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored -# otherwise) -# -# Bitnami retired the versioned tags under docker.io/bitnami and republished the -# archived builds under docker.io/bitnamilegacy, so the subchart's own image -# defaults no longer resolve. The repository below points at the same build the -# subchart was released with, which keeps the on-disk data directory layout -# identical for existing installs. -# -# Keep the tag pinned. docker.io/bitnami still publishes a floating `latest`, -# and starting a newer PostgreSQL major against an existing data directory -# leaves the server refusing to boot ("database files are incompatible with -# server") with no way back other than a dump taken beforehand. Crossing a major -# version is a dump-and-restore, not an image bump. The chart refuses to render -# an unpinned tag for this reason -postgresql: - architecture: standalone - image: - repository: bitnamilegacy/postgresql - tag: 16.2.0-debian-12-r6 - volumePermissions: - image: - repository: bitnamilegacy/os-shell - tag: 12-debian-12-r16 - metrics: - image: - repository: bitnamilegacy/postgres-exporter - tag: 0.15.0-debian-12-r14 - auth: - username: litellm - database: litellm - - # You should override these on the helm command line with - # `--set postgresql.auth.postgres-password=,postgresql.auth.password=` - password: NoTaGrEaTpAsSwOrD - postgres-password: NoTaGrEaTpAsSwOrD - - # A secret is created by this chart (litellm-helm) with the credentials that - # the new Postgres instance should use. - # existingSecret: "" - # secretKeys: - # userPasswordKey: password - -# Redis is the proxy's coordination store: cross-pod tpm/rpm rate limits, spend -# tracking, and the pod lock manager. Enabling this deploys the bundled Redis -# subchart, wires REDIS_HOST / REDIS_PORT / REDIS_PASSWORD into the proxy, and -# renders a `general_settings.coordination_redis` block into the proxy config. -# -# To point at an existing Redis instead, leave `enabled: false` and pass a -# secret for REDIS_HOST, REDIS_PORT, REDIS_PASSWORD or REDIS_URL; the proxy -# falls back to those env vars for coordination. Set `cache: true` in the proxy -# config only if you also want LLM response caching, which is independent of -# coordination -# -# When `redis.sentinel.enabled` is set, the coordination block is rendered with -# `sentinel_nodes` and `service_name` (from `redis.sentinel.masterSet`) instead -# of host/port, because a plain Redis client cannot talk to the sentinel port -# -# The image repositories carry the same bitnamilegacy repoint as postgresql -# above; the versioned tags the subchart ships with are gone from -# docker.io/bitnami -redis: - enabled: false - architecture: standalone - image: - repository: bitnamilegacy/redis - tag: 7.2.4-debian-12-r9 - sentinel: - image: - repository: bitnamilegacy/redis-sentinel - tag: 7.2.4-debian-12-r7 - metrics: - image: - repository: bitnamilegacy/redis-exporter - tag: 1.58.0-debian-12-r4 - volumePermissions: - image: - repository: bitnamilegacy/os-shell - tag: 12-debian-12-r16 - sysctl: - image: - repository: bitnamilegacy/os-shell - tag: 12-debian-12-r16 - kubectl: - image: - repository: bitnamilegacy/kubectl - tag: 1.29.2-debian-12-r3 - coordination: - # Set to false to keep the bundled Redis for response caching only and leave - # `general_settings.coordination_redis` out of the rendered config. A - # `coordination_redis` block you define yourself in `proxy_config` always wins - enabled: true - -# Prisma migration job settings -migrationJob: - enabled: true # Enable or disable the schema migration Job - backoffLimit: 4 # Backoff limit for Job restarts - # Wall-clock budget for the whole Job, shared across every `backoffLimit` - # retry rather than granted per attempt. Without it a migration that blocks - # on the database never fails, and when the Helm hook is enabled the release - # waits on it forever: `helm upgrade` and any GitOps controller driving it - # stop reconciling the whole chart until someone deletes the Job by hand. - # Set to null to opt out and restore the unbounded behaviour. - activeDeadlineSeconds: 1800 - # Optional service account for the migration job. - # Only used when migrationJob.hooks.helm.enabled=true and serviceAccount.create=true. - # In that case, pre-install/pre-upgrade hooks run before normal resources, so this defaults to "default". - serviceAccountName: "" - annotations: {} - jobLabels: {} # Custom labels for the Job metadata - jobAnnotations: {} # Custom annotations for the Job metadata - podLabels: {} # Custom labels for the Job pod template - command: [] # Override container command (defaults to ["python", "litellm/proxy/prisma_migration.py"]) - args: [] # Override container args - ttlSecondsAfterFinished: 120 - resources: {} - # Unset by default. This job runs the database migration and exits, so it does not - # need the steady-state headroom the proxy does; size it from your own migration - # runs rather than from the proxy figures above. - extraContainers: [] - extraInitContainers: [] - - # Hook configuration - hooks: - argocd: - enabled: true - helm: - enabled: false - -# Log level for the litellm proxy (sets LITELLM_LOG in the deployment env). -# Rendered as a direct `env:` entry, which in Kubernetes takes precedence over -# any `envFrom:` source. If you currently source LITELLM_LOG from an -# environmentSecret or environmentConfigMap, set `logLevel: ""` here to -# disable injection — otherwise this value silently overrides your secret / -# configmap entry. -# -# Setting LITELLM_LOG inside `envVars:` below also wins: the template skips -# this injection entirely when envVars already defines LITELLM_LOG. -logLevel: INFO - -# Additional environment variables to be added to the deployment as a map of key-value pairs -envVars: {} - -# USE_DDTRACE: "true" -# Additional environment variables to be added to the deployment as a list of k8s env vars -extraEnvVars: {} - -# if you want to override the container command, you can do so here -command: {} -# if you want to override the container args, you can do so here -args: {} - -# - name: EXTRA_ENV_VAR -# value: EXTRA_ENV_VAR_VALUE -# Additional Kubernetes resources to deploy with litellm -extraResources: [] - -# - apiVersion: v1 -# kind: ConfigMap -# metadata: -# name: my-extra-config -# data: -# foo: bar -# Pod Disruption Budget -pdb: - enabled: false - # Set exactly one of the following. If both are set, minAvailable takes precedence. - minAvailable: null # e.g. "50%" or 1 - maxUnavailable: null # e.g. 1 or "20%" - annotations: {} - labels: {} - -serviceMonitor: - enabled: false - labels: - {} - # test: test - annotations: - {} - # kubernetes.io/test: test - interval: 15s - scrapeTimeout: 10s - relabelings: [] - # - targetLabel: __meta_kubernetes_pod_node_name - # replacement: $1 - # action: replace - namespaceSelector: - matchNames: [] - # - test-namespace - -lens: - enabled: false - url: "" - publicUrl: "" - service: - name: "" - port: 4318 - gatewaySecret: - name: "" - key: gateway-secret - serviceTokenSecret: - name: "" - key: service-token diff --git a/helm/litellm/.helmignore b/helm/litellm/.helmignore new file mode 100644 index 00000000000..6b8d3002acd --- /dev/null +++ b/helm/litellm/.helmignore @@ -0,0 +1,20 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +*.swp +*.bak +*.tmp +*.orig +*~ +.project +.idea/ +*.tmproj +.vscode/ +/tests/ +/ci/ diff --git a/helm/litellm/Chart.yaml b/helm/litellm/Chart.yaml index e67f5790c7e..7f308a454f5 100644 --- a/helm/litellm/Chart.yaml +++ b/helm/litellm/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: litellm -description: LiteLLM componentized — gateway, UI backend, and UI as separate services +description: Deploys LiteLLM from one image, either componentized (gateway, UI backend, and UI as separate Deployments) or as a single monolith Deployment type: application -version: 0.1.0 -appVersion: "0.1.0" +version: 1.0.0 +appVersion: "1.104.0" annotations: org.opencontainers.image.source: "https://github.com/BerriAI/litellm" diff --git a/helm/litellm/README.md b/helm/litellm/README.md index 97b94754402..5ca63d90bcd 100644 --- a/helm/litellm/README.md +++ b/helm/litellm/README.md @@ -1,6 +1,51 @@ -# LiteLLM componentized chart +# litellm Helm chart -This chart deploys separate gateway, backend and UI services +Deploys [LiteLLM](https://github.com/BerriAI/litellm) from one image, `ghcr.io/berriai/litellm` (mirrored at `docker.litellm.ai/berriai/litellm`), in one of two layouts: + +- componentized (default): the `gateway` (LLM data plane, port 4000), the `backend` (management API, port 4001) and the `ui` (static dashboard, port 3000) each get their own Deployment, Service, HPA and PDB so they scale independently +- monolith: `monolith.enabled: true` renders one Deployment and Service running the full proxy (`args: [proxy, ...]`), which serves the gateway routes, the management routes and the Admin UI from a single process + +Both layouts run the same image. The image entrypoint dispatches on the first container argument (`proxy`, `gateway`, `backend`, `ui`, `migrations`, `metrics`, `collector`), so the chart only ever sets `args` and never `command` + +## Requirements + +Kubernetes 1.25+ and Helm 3.8+. Bring your own PostgreSQL (`database.writer.*`) and, optionally, Redis (`redis.*`). The proxy also needs a master key, either an existing Secret named by `masterKey.secretName` or one the chart generates with `masterKey.generate: true` and `masterKey.secretName: ""` + +## Install + +```bash +kubectl create secret generic litellm-master-key-secret --from-literal=master-key=sk-change-me +helm install litellm helm/litellm \ + --set database.writer.host=postgres.example.com \ + --set database.writer.dbname=litellm \ + --set database.writer.passwordSecret.name=litellm-db-secret +``` + +The image tag defaults to the chart's `appVersion`. Override it with `image.tag`, or pin the bytes with `image.digest`, which renders as `repository:tag@digest` + +## Monolith quickstart + +```bash +helm install litellm helm/litellm \ + --set monolith.enabled=true \ + --set masterKey.secretName="" \ + --set masterKey.generate=true \ + --set database.writer.host=postgres.example.com \ + --set database.writer.dbname=litellm \ + --set database.writer.passwordSecret.name=litellm-db-secret +kubectl port-forward svc/litellm-litellm 4000:4000 +``` + +With `monolith.enabled: true`: + +- one Deployment and one Service named `-litellm` render, running `args: [proxy, --port, 4000, --config, /app/config/config.yaml]` plus `monolith.extraArgs` +- the gateway, backend and ui Deployments, Services, HPAs, PDBs and ServiceMonitors are not rendered, whatever `gateway.enabled`, `backend.enabled` and `ui.enabled` say +- every `gateway.*` value configures the monolith pod: `config`, `numWorkers`, `resources`, probes, `securityContext`, `hpa`, `keda`, `pdb`, `metricsServer`, `collector`, `volumes`, `extraEnv`, scheduling. The monolith runs as `serviceAccounts.gateway` +- every `backend.*` and `ui.*` value is ignored +- the Ingress sends every path, built in or from `ingress.extraPaths`, to the monolith Service +- the migrations Job renders exactly as in componentized mode + +## Lens This chart does not deploy Lens. Deploy Lens from [its own chart](https://github.com/BerriAI/lens/tree/main/helm/lens), then set `lens.enabled: true`, `lens.url`, `lens.publicUrl`, `lens.gatewaySecret.name` and `lens.serviceTokenSecret.name`. The secret keys default to `gateway-secret` and `service-token`. Set `lens.service.name` when that Lens runs in this release's namespace and the chart ingress should route `/lens-ingest` to it. The old `lensWorker` values are no longer read, and enabling them fails the render @@ -22,22 +67,114 @@ backend: gatewayUrl: https://gateway.example.com ``` -## Routes to another Service +## Testing -For a shared hostname, `ingress.extraPaths` can route to a Service in the same namespace using `serviceName` and `servicePort`. The port can be a number or a named Service port. Omit `service` when using these fields - -```yaml -ingress: - enabled: true - className: alb - host: gateway.example.com - extraPaths: - - path: /moyai - pathType: Prefix - serviceName: moyai-ingress - servicePort: 8080 +```bash +helm lint helm/litellm +helm unittest -f 'tests/*.yaml' helm/litellm +helm test --logs ``` -The Service and application must already support the path prefix. The chart adds the path to the existing Ingress and leaves its built-in routes in place. It does not create the target Service or rewrite requests +## Migrating from the litellm-helm chart -For AWS ALB, configure the target Service's `alb.ingress.kubernetes.io/healthcheck-path` annotation when the application needs a different health check from LiteLLM. Existing `service` aliases (`gateway`, `backend`, `ui`) remain supported +The `litellm-helm` chart (`oci://ghcr.io/berriai/litellm-helm`) is retired; its published packages stay available for a grace period. Its flat values described one monolith Deployment, so the equivalent install here is `monolith.enabled: true` with the values moved under `gateway.*`. The per-component `gateway.image`, `backend.image`, `ui.image` and `migrations.image` blocks of earlier `helm/litellm` versions are gone too: the chart is a major bump to `1.0.0` and every container uses the top-level `image` + +| litellm-helm value | litellm value | +|---|---| +| (implicit single Deployment) | `monolith.enabled: true` | +| `image.repository` / `image.tag` / `image.pullPolicy` | `image.repository` / `image.tag` / `image.pullPolicy` (`image.digest` is new) | +| `replicaCount` | `gateway.replicaCount` | +| `args` | `monolith.extraArgs` (appended after the chart's proxy arguments). Drop any `--port` argument and its value so the proxy keeps listening on port 4000, matching the container port and health checks. Set `gateway.service.port` to change the external Service port | +| `command` | removed: the image entrypoint dispatcher must stay in place | +| `proxy_config` | `gateway.config.proxy_config` | +| `proxyConfigMap.create: false` + `proxyConfigMap.name` | Set `gateway.config.create: false`, mount your ConfigMap with `gateway.volumes` / `gateway.volumeMounts`, and pass `--config` followed by the mounted file path in `monolith.extraArgs` | +| `masterkeySecretName` / `masterkeySecretKey` | `masterKey.secretName` / `masterKey.secretKey`. The default Secret field changes from `masterkey` to `master-key`. When reusing a Secret with the old default field, explicitly set `masterKey.secretKey: masterkey` | +| `masterkeySecretName: ""` (auto generated Secret) | `masterKey.generate: true` with `masterKey.secretName: ""` (creates a new key; copy the old key first, see below, to keep existing credentials valid) | +| `db.useExisting`, `db.endpoint`, `db.database` | `database.writer.host`, `database.writer.port`, `database.writer.dbname` | +| `db.useIAMAuth` / `db.useAzureEntraAuth` | `database.writer.useIAMAuth` / `database.writer.useAzureEntraAuth` (mutually exclusive). Both token-auth modes still require a username in the Secret referenced by `database.writer.passwordSecret.name` and `usernameKey` | +| `db.schema` | `database.writer.schema` | +| `db.username` | Store the username in the Secret referenced by `database.writer.passwordSecret.name`, under its `usernameKey`. There is no literal username value in the new chart | +| `db.secret.name` / `db.secret.usernameKey` / `db.secret.passwordKey` | `database.writer.passwordSecret.name` / `usernameKey` / `passwordKey` | +| `db.secret.endpointKey` (writer host read from a Secret) | `database.writer.passwordSecret.hostKey` with `database.writer.host` left empty | +| `db.url` (custom URL template) | `database.writer.passwordSecret.urlKey` pointing at a Secret key that holds the complete writer URL, with the other `database.writer.*` connection values left empty | +| `db.readReplicaUrl` (credential-less reader URL) | `database.reader.host`, `database.reader.port`, `database.reader.dbname`, with `database.reader.passwordSecret.*` for its credentials (or `database.reader.useIAMAuth: true`) | +| `db.secret.readReplicaEndpointKey` (reader host read from a Secret) | `database.reader.passwordSecret.hostKey` with `database.reader.host` left empty | +| `db.secret.readReplicaUrlKey` (full reader URL in a Secret) | `database.reader.passwordSecret.urlKey` with the other `database.reader.*` connection values left empty | +| `db.connectionPool.*` | `database.connectionPool.*` | +| `db.deployStandalone: true` / `postgresql.*` | removed: the chart ships no PostgreSQL, point `database.writer.*` at your own | +| `redis.enabled: true` (bundled) | removed: the chart ships no Redis, point `redis.host` at your own | +| external Redis via `envVars` | `redis.host`, `redis.port`, `redis.passwordSecret.*`, `redis.cluster` | +| `envVars` / `extraEnvVars` | `gateway.extraEnv` (list of `name` / `value` or `valueFrom` entries). The retired chart also passed these to the migration Job, so copy any it needs, such as the AWS region and credentials for IAM database auth, into `migrationJob.extraEnv` too | +| `environmentSecrets` | `gateway.envSecrets` | +| `environmentConfigMaps` | `gateway.envConfigMaps` | +| `logLevel` | `gateway.logLevel` | +| `resources` | `gateway.resources` | +| `livenessProbe` / `readinessProbe` / `startupProbe` | `gateway.livenessProbe` / `gateway.readinessProbe` / `gateway.startupProbe`, written as full Kubernetes probes: the old `path: /health/readiness` becomes `httpGet: { path: /health/readiness, port: http }`, the timing fields (`initialDelaySeconds`, `periodSeconds`, `timeoutSeconds`, `successThreshold`, `failureThreshold`) carry over unchanged | +| `securityContext` / `podSecurityContext` | `gateway.securityContext` / `gateway.podSecurityContext`. With `readOnlyRootFilesystem: true` the retired chart added `emptyDir` mounts at `/tmp`, `/.cache` and `/.npm`; this chart does not, so add them through `gateway.volumes` / `gateway.volumeMounts`. The retired chart applied both to the migration Job as well; set `migrationJob.securityContext` / `migrationJob.podSecurityContext` to keep the Job under the same policy | +| `service.*` | `gateway.service.*` | +| `ingress.className`, `ingress.annotations`, `ingress.tls` | same keys (routes to the monolith Service in monolith mode) | +| `ingress.hosts[].host` | `ingress.host`, one host per release; the built-in paths are rendered for that host, so `ingress.hosts[].paths` has no equivalent. For more than one hostname, add a second Ingress through `extraResources` or point a DNS alias at the one host | +| `autoscaling.*` | `gateway.hpa.*`. Defaults change from 100 to 10 maximum replicas and from 60% to 70% CPU utilization, with an 80% memory target now enabled. To keep the old scaling behavior, set `gateway.hpa.maxReplicas: 100`, `gateway.hpa.targetCPUUtilizationPercentage: 60` and `gateway.hpa.targetMemoryUtilizationPercentage: 0` | +| `keda.*` | `gateway.keda.*` with these renames: `minReplicas` / `maxReplicas` are `minReplicaCount` / `maxReplicaCount` (the new default maximum is 10, the old one was 100), `behavior` moves to `advanced.horizontalPodAutoscalerConfig.behavior`, `restoreToOriginalReplicaCount` moves to `advanced.restoreToOriginalReplicaCount`, and `keda.prometheus.requestsPerSecond` / `tokensPerSecond` are `gateway.keda.prometheus.targetRequestsPerSecond` / `targetTokensPerSecond`. `pollingInterval`, `cooldownPeriod`, `fallback`, `triggers` and `scaledObject.annotations` (for example KEDA's pause annotations) keep their names | +| `numWorkers` | `gateway.numWorkers` | +| `pdb.*` | `gateway.pdb.*`, except `labels` and `annotations` are no longer supported. To retain custom PDB metadata, set `gateway.pdb.enabled: false` and define the PDB through `extraResources` | +| `metricsServer.*` | `gateway.metricsServer.*` | +| `serviceMonitor.*` | `gateway.serviceMonitor.*` (`labels`, `annotations`, `interval`, `scrapeTimeout`, `relabelings`, `namespaceSelector.matchNames`); it now requires `gateway.metricsServer.enabled` | +| `collector.*` | `gateway.collector.*`, except `command` is no longer supported and `scaleOnProxyContainerCpu` becomes `scaleOnGatewayContainerCpu`. Its default changes from `false` to `true`. Set `gateway.collector.scaleOnGatewayContainerCpu: false` to keep pod-wide CPU scaling. For a custom collector command, disable the built-in sidecar with `gateway.collector.enabled: false` and define it through `gateway.extraContainers`, with its environment, volumes and volume mounts configured explicitly | +| `billingMetrics.*` | `billingMetrics.*` | +| `migrationJob.*` | `migrationJob.*` | +| `volumes` / `volumeMounts` | `gateway.volumes` / `gateway.volumeMounts`. The retired chart also mounted these in the migration Job; copy any mounts the Job needs, such as a database CA certificate, to `migrationJob.volumes` / `migrationJob.volumeMounts` too | +| `extraContainers` / `extraInitContainers` | `gateway.extraContainers` / `gateway.extraInitContainers` | +| `lifecycle` | `gateway.lifecycle` | +| `strategy` | `gateway.strategy` | +| `deploymentAnnotations` / `deploymentLabels` / `deploymentMinReadySeconds` | `gateway.deploymentAnnotations` / `gateway.deploymentLabels` / `gateway.minReadySeconds` | +| `podAnnotations` / `podLabels` | `gateway.podAnnotations` / `gateway.podLabels`. The retired chart ran `podAnnotations` through `tpl`; this chart renders them verbatim, so a copied template such as `checksum/config: {{ ... }}` stays literal text. Resolve it before install instead, e.g. `--set-string gateway.podAnnotations.checksum/config=$(kubectl get configmap my-config -o yaml \| sha256sum \| cut -d' ' -f1)`. The chart's own proxy ConfigMap already rolls the pods when `gateway.config` changes. The retired chart also put `podLabels` on the migration Job pod; copy any the Job needs, such as a network-policy selector, to `migrationJob.podLabels` | +| `nodeSelector` / `tolerations` / `affinity` / `topologySpreadConstraints` | `gateway.nodeSelector` / `gateway.tolerations` / `gateway.affinity` / `gateway.topologySpreadConstraints`. The retired chart also applied `nodeSelector`, `tolerations` and `affinity` to the migration Job; copy them to `migrationJob.nodeSelector` / `migrationJob.tolerations` / `migrationJob.affinity` too, or a Job that needs tainted or zoned nodes to reach the database stays Pending and blocks the install | +| `terminationGracePeriodSeconds` | `gateway.terminationGracePeriodSeconds` | +| `serviceAccount.*` | `serviceAccounts.gateway.*`. The retired chart also shared an existing ServiceAccount with migrations; set `migrationJob.serviceAccountName` to that existing account too when the Job needs its permissions, such as for IAM database auth | +| `lens.*` | `lens.*`. Deploy Lens from its own chart. The removed `lensWorker.*` values fail the render when enabled | +| `extraResources` | `extraResources` | +| `nameOverride: "litellm"` | `nameOverride: ""` (the chart name is already `litellm`) | +| `fullnameOverride` | `fullnameOverride`. The new default always includes the chart-name suffix, even when the release name already contains it. Set this to the existing Service name to retain its DNS name | + +A minimal migration: + +```yaml +monolith: + enabled: true +masterKey: + secretName: litellm-master-key-secret +database: + writer: + host: postgres.example.com + dbname: litellm + passwordSecret: + name: litellm-db-secret + usernameKey: username + passwordKey: password +gateway: + replicaCount: 2 + config: + proxy_config: + model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + envSecrets: + - litellm-provider-keys +``` + +The monolith Service is named `-litellm`. To retain a different old Service name, set `fullnameOverride` to that existing name. The generated Secret in this chart has `helm.sh/resource-policy: keep` and is reused across upgrades of this chart. The retired chart stored its generated key under a different Secret and data key, and `helm uninstall` deletes that Secret, so copy it into a Secret you own before uninstalling the old release to keep existing credentials valid. Run the following in the old release's namespace, replacing `OLD_MASTER_KEY_SECRET` with the actual existing Secret name. For a release named `litellm` without a `fullnameOverride`, that name is `litellm-masterkey`: + +```bash +set -euo pipefail +OLD_MASTER_KEY_SECRET=litellm-masterkey +old_master_key="$(kubectl get secret "$OLD_MASTER_KEY_SECRET" -o jsonpath='{.data.masterkey}' | base64 -d)" +test -n "$old_master_key" +kubectl create secret generic litellm-master-key-secret \ + --from-literal=master-key="$old_master_key" +unset old_master_key +``` + +Then set `masterKey.secretName: litellm-master-key-secret`, as in the minimal example above. Releases that set `masterkeySecretName` already point `masterKey.secretName` and `masterKey.secretKey` at that Secret diff --git a/helm/litellm/ci/postgres.yaml b/helm/litellm/ci/postgres.yaml new file mode 100644 index 00000000000..01472950b3b --- /dev/null +++ b/helm/litellm/ci/postgres.yaml @@ -0,0 +1,58 @@ +# Throwaway PostgreSQL for the local chart smoke test. Applied with +# kubectl before `helm install`; the chart itself ships no database. +apiVersion: v1 +kind: Secret +metadata: + name: litellm-ci-postgres +type: Opaque +stringData: + username: litellm + password: litellm-ci +--- +apiVersion: v1 +kind: Service +metadata: + name: litellm-ci-postgres +spec: + selector: + app: litellm-ci-postgres + ports: + - port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: litellm-ci-postgres +spec: + replicas: 1 + selector: + matchLabels: + app: litellm-ci-postgres + template: + metadata: + labels: + app: litellm-ci-postgres + spec: + containers: + - name: postgres + image: postgres:16.10-alpine@sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297 + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + value: litellm + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: litellm-ci-postgres + key: username + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: litellm-ci-postgres + key: password + readinessProbe: + exec: + command: [pg_isready, -U, litellm, -d, litellm] + periodSeconds: 2 diff --git a/helm/litellm/ci/test-values.yaml b/helm/litellm/ci/test-values.yaml new file mode 100644 index 00000000000..850dd48ee63 --- /dev/null +++ b/helm/litellm/ci/test-values.yaml @@ -0,0 +1,34 @@ +# Values for the local chart smoke test: a monolith install on a +# kind cluster from the image built by the pipeline (image.repository / tag / +# pullPolicy are passed with --set), backed by the throwaway PostgreSQL in +# ci/postgres.yaml so the migrations Job and the proxy have a database. +monolith: + enabled: true + +masterKey: + secretName: "" + generate: true + +gateway: + replicaCount: 1 + hpa: + enabled: false + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + memory: 2Gi + config: + create: true + proxy_config: + general_settings: {} + +database: + writer: + host: litellm-ci-postgres + dbname: litellm + passwordSecret: + name: litellm-ci-postgres + usernameKey: username + passwordKey: password diff --git a/helm/litellm/templates/NOTES.txt b/helm/litellm/templates/NOTES.txt index 135dcb876e0..a046e4e5280 100644 --- a/helm/litellm/templates/NOTES.txt +++ b/helm/litellm/templates/NOTES.txt @@ -1,4 +1,16 @@ -LiteLLM componentized — release {{ .Release.Name }} in namespace {{ .Release.Namespace }}. +{{- if .Values.monolith.enabled }} +LiteLLM monolith: release {{ .Release.Name }} in namespace {{ .Release.Namespace }}, image {{ include "litellm.image" . }} + +One Deployment runs the full proxy (gateway routes, management routes and the Admin UI): + - proxy : Service {{ include "litellm.workload.fullname" . }} on port {{ .Values.gateway.service.port }} + +Port-forward: + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.workload.fullname" . }} {{ .Values.gateway.service.port }} + +The monolith takes its configuration from the gateway.* values (config, resources, probes, hpa, pdb, keda, +metricsServer, collector, volumes, env, scheduling). backend.* and ui.* are ignored while monolith.enabled is true +{{- else }} +LiteLLM componentized: release {{ .Release.Name }} in namespace {{ .Release.Namespace }}, image {{ include "litellm.image" . }} Components: {{- if .Values.gateway.enabled }} @@ -16,43 +28,48 @@ Port-forward examples: kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.backend.fullname" . }} {{ .Values.backend.service.port }} kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "litellm.ui.fullname" . }} {{ .Values.ui.service.port }} +Set monolith.enabled=true to run everything as one Deployment instead +{{- end }} + Reminders: - Sensitive values come from Secret references only. Before installing, set: - - masterKey.secretName (Secret with the proxy master key) + - masterKey.secretName (Secret with the proxy master key, or + masterKey.generate: true with secretName "" to let the chart create one) - database.writer.{host,port,dbname} (writer connection pieces) - database.writer.passwordSecret.{name,usernameKey,passwordKey} (Secret holding the writer DB username + password) - database.writer.passwordSecret.{hostKey,urlKey} (optional, read writer host or full URL from that Secret) - - database.writer.useIAMAuth: true (optional — chart sets IAM_TOKEN_DB_AUTH=true and + - database.writer.useIAMAuth: true (optional: chart sets IAM_TOKEN_DB_AUTH=true and omits DATABASE_PASSWORD / DATABASE_URL so the proxy mints the URL from an IAM token at startup) - database.reader.host / database.reader.passwordSecret.{hostKey,urlKey} (optional, enables read-replica routing. The Secret name is required for hostKey, urlKey, and for host with password auth) - - database.reader.useIAMAuth: true (optional, requires database.writer.useIAMAuth: true — + - database.reader.useIAMAuth: true (optional, requires database.writer.useIAMAuth: true; chart emits DATABASE_*_READ_REPLICA env vars and omits DATABASE_PASSWORD_READ_REPLICA / DATABASE_URL_READ_REPLICA so the proxy mints the reader URL from an IAM token at startup) - - redis.passwordSecret.name (optional — set when redis.host is provided and the + - redis.host (optional: the proxy's coordination Redis) + - redis.passwordSecret.name (optional: set when redis.host is provided and the cache requires auth) - redis.passwordSecret.{hostKey,urlKey} (optional, read Redis host or full URL from that Secret) (the Secret name is required when either key is set) - - redis.cluster: true (optional — chart sets REDIS_CLUSTER_NODES from + - redis.cluster: true (optional: chart sets REDIS_CLUSTER_NODES from redis.host / redis.port so the proxy's Cache() constructs a RedisClusterCache; the cluster client discovers remaining nodes from CLUSTER SLOTS) - - Per-component extras (gateway / backend / ui): - - {component}.extraEnv / envConfigMaps / envSecrets (the latter two are lists of resource names → - envFrom configMapRef / secretRef) + - Per-component extras (gateway / backend / ui; gateway.* also drives the monolith): + - {component}.extraEnv / envConfigMaps / envSecrets (the latter two are lists of resource names rendered + as envFrom configMapRef / secretRef) - {component}.logLevel (renders as LITELLM_LOG) - gateway.config.proxy_config (rendered into a ConfigMap and mounted at - /app/config/config.yaml; gateway reads it via - CONFIG_FILE_PATH) + /app/config/config.yaml; read via CONFIG_FILE_PATH) - {component}.pdb.{enabled,minAvailable,maxUnavailable} (per-component PodDisruptionBudget; disabled by - default — with hpa.minReplicas of 1, minAvailable: 1 + default: with hpa.minReplicas of 1, minAvailable: 1 would block node drains) - {component}.topologySpreadConstraints (standard k8s list, e.g. spread replicas across topology.kubernetes.io/zone) - - Enable ingress.enabled=true to dispatch / → ui, gateway data-plane prefixes → gateway, and the catch-all → backend. + - Enable ingress.enabled=true to dispatch / to ui, gateway data-plane prefixes to gateway and the catch-all to + backend; in monolith mode every path goes to the monolith Service diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 18e938564e9..13222d8284e 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -1,11 +1,116 @@ {{/* -Common naming + label helpers shared by gateway, backend, and ui templates. +Common naming + label helpers shared by gateway, backend, ui, and monolith templates. */}} {{- define "litellm.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +The one image reference every container in the chart uses. `repository:tag`, +or `repository:tag@digest` when image.digest is set; the tag falls back to +the chart appVersion. +*/}} +{{- define "litellm.image" -}} +{{- $tag := .Values.image.tag | default .Chart.AppVersion -}} +{{- if .Values.image.digest -}} +{{- printf "%s:%s@%s" .Values.image.repository $tag .Values.image.digest -}} +{{- else -}} +{{- printf "%s:%s" .Values.image.repository $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Componentized mode renders the gateway / backend / ui Deployments only when +monolith mode is off. +*/}} +{{- define "litellm.gateway.render" -}} +{{- if and .Values.gateway.enabled (not .Values.monolith.enabled) -}}true{{- end -}} +{{- end -}} + +{{- define "litellm.backend.render" -}} +{{- if and .Values.backend.enabled (not .Values.monolith.enabled) -}}true{{- end -}} +{{- end -}} + +{{- define "litellm.ui.render" -}} +{{- if and .Values.ui.enabled (not .Values.monolith.enabled) -}}true{{- end -}} +{{- end -}} + +{{/* +The proxy Deployment (monolith mode) and the gateway Deployment +(componentized mode) are the same pod spec fed by .Values.gateway, so the +templates that serve both (HPA, KEDA, PDB, metrics Service, ServiceMonitor) +resolve their name, component label and selector through these. +*/}} +{{- define "litellm.proxy.fullname" -}} +{{- include "litellm.fullname" . -}} +{{- end -}} + +{{- define "litellm.proxy.selectorLabels" -}} +app.kubernetes.io/name: {{ include "litellm.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: proxy +{{- end -}} + +{{- define "litellm.workload.componentName" -}} +{{- if .Values.monolith.enabled -}}proxy{{- else -}}gateway{{- end -}} +{{- end -}} + +{{- define "litellm.workload.fullname" -}} +{{- if .Values.monolith.enabled -}} +{{- include "litellm.proxy.fullname" . -}} +{{- else -}} +{{- include "litellm.gateway.fullname" . -}} +{{- end -}} +{{- end -}} + +{{- define "litellm.workload.selectorLabels" -}} +{{- if .Values.monolith.enabled -}} +{{- include "litellm.proxy.selectorLabels" . -}} +{{- else -}} +{{- include "litellm.gateway.selectorLabels" . -}} +{{- end -}} +{{- end -}} + +{{- define "litellm.workload.render" -}} +{{- if or .Values.monolith.enabled .Values.gateway.enabled -}}true{{- end -}} +{{- end -}} + +{{/* +Container arguments for the proxy (monolith) container. The image entrypoint +dispatches on the first argument. +*/}} +{{- define "litellm.proxy.args" -}} +- proxy +- --port +- "4000" +{{- if .Values.gateway.config.create }} +- --config +- /app/config/config.yaml +{{- end }} +{{- with .Values.monolith.extraArgs }} +{{ toYaml . }} +{{- end }} +{{- end -}} + +{{/* +Master key Secret reference. `masterKey.secretName` when set, otherwise the +Secret the chart generates when `masterKey.generate` is true. +*/}} +{{- define "litellm.masterKey.generatedSecretName" -}} +{{- printf "%s-masterkey" (include "litellm.fullname" .) -}} +{{- end -}} + +{{- define "litellm.masterKey.secretName" -}} +{{- if .Values.masterKey.secretName -}} +{{- .Values.masterKey.secretName -}} +{{- else if .Values.masterKey.generate -}} +{{- include "litellm.masterKey.generatedSecretName" . -}} +{{- else -}} +{{- fail "masterKey.secretName is required (the chart never accepts an inline master key); set it to an existing Secret or set masterKey.generate: true" -}} +{{- end -}} +{{- end -}} + {{- define "litellm.fullname" -}} {{- if .Values.fullnameOverride -}} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} @@ -58,56 +163,27 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} {{- end -}} -{{/* -Enterprise billable-request metering. Wired into gateway and backend, not the -migrations job. The gateway serves nearly all billable traffic, but the backend -keeps the named-server MCP transport (/{mcp_server_name}/mcp), which writes a -SpendLogs row, so metering only the gateway would silently drop that traffic. -The client certificate identifies the deployment to LiteLLM's collector, so it is -mounted read-only from an existing Secret rather than passed through the -environment. -*/}} -{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}} -{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}} - {{- define "litellm.billingMetricsEnv" -}} - name: LITELLM_BILLING_METRICS_ENDPOINT value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }} -- name: LITELLM_BILLING_METRICS_CLIENT_CERT - value: {{ printf "%s/tls.crt" (include "litellm.billingMetrics.certDir" .) | quote }} -- name: LITELLM_BILLING_METRICS_CLIENT_KEY - value: {{ printf "%s/tls.key" (include "litellm.billingMetrics.certDir" .) | quote }} -{{- if .Values.billingMetrics.caSecretName }} -- name: LITELLM_BILLING_METRICS_CA_CERT - value: {{ printf "%s/ca.crt" (include "litellm.billingMetrics.caDir" .) | quote }} -{{- end }} +- name: LITELLM_BILLING_METRICS_TOKEN_URL + value: {{ required "billingMetrics.tokenUrl is required when billingMetrics.enabled is true" .Values.billingMetrics.tokenUrl | quote }} +- name: LITELLM_BILLING_METRICS_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }} + key: client-id +- name: LITELLM_BILLING_METRICS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }} + key: client-secret {{- with .Values.billingMetrics.exportIntervalMs }} - name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS value: {{ . | quote }} {{- end }} {{- end -}} -{{- define "litellm.billingMetricsVolumes" -}} -- name: billing-metrics-mtls - secret: - secretName: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)" .Values.billingMetrics.secretName }} -{{- if .Values.billingMetrics.caSecretName }} -- name: billing-metrics-mtls-ca - secret: - secretName: {{ .Values.billingMetrics.caSecretName }} -{{- end }} -{{- end -}} - -{{- define "litellm.billingMetricsVolumeMounts" -}} -- name: billing-metrics-mtls - mountPath: {{ include "litellm.billingMetrics.certDir" . }} - readOnly: true -{{- if .Values.billingMetrics.caSecretName }} -- name: billing-metrics-mtls-ca - mountPath: {{ include "litellm.billingMetrics.caDir" . }} - readOnly: true -{{- end }} -{{- end -}} {{/* Per-component selector labels — used in both Service selectors and Deployment matchLabels. @@ -137,6 +213,7 @@ Each component (gateway, backend, ui) has its own SA config under .Values.serviceAccounts.. When `create` is true and `name` is empty the chart defaults to "-litellm-". When `create` is false the chart uses the provided name, or the namespace `default` SA. +The monolith pod runs as the gateway ServiceAccount. */}} {{- define "litellm.gateway.serviceAccountName" -}} {{- if .Values.serviceAccounts.gateway.create -}} @@ -215,6 +292,22 @@ by the controller rather than declared, so nothing there can collide. {{- toYaml .podLabels }} {{- end -}} +{{/* +LITELLM_MASTER_KEY, and PROXY_MASTER_KEY for configs copied from litellm-helm, +for the app containers only. The migrations Job runs as a +pre-install hook, before the chart's generated master key Secret exists, and +migrations/run.py never reads the key, so the Job must not reference it. +*/}} +{{- define "litellm.masterKeyEnv" -}} +{{- range list "LITELLM_MASTER_KEY" "PROXY_MASTER_KEY" }} +- name: {{ . }} + valueFrom: + secretKeyRef: + name: {{ include "litellm.masterKey.secretName" $ }} + key: {{ $.Values.masterKey.secretKey | default "master-key" }} +{{- end }} +{{- end -}} + {{/* Master-key + database + redis env block — shared by gateway, backend, and the migrations Job. @@ -258,11 +351,6 @@ IAM_TOKEN_DB_AUTH / AZURE_POSTGRESQL_AUTH toggle that only the writer sets. {{- define "litellm.serverEnv" -}} {{- $root := .root -}} {{- $component := .component -}} -- name: LITELLM_MASTER_KEY - valueFrom: - secretKeyRef: - name: {{ required "masterKey.secretName is required (the chart no longer accepts an inline master key)" $root.Values.masterKey.secretName }} - key: {{ $root.Values.masterKey.secretKey | default "master-key" }} {{- if $component.logLevel }} - name: LITELLM_LOG value: {{ $component.logLevel | quote }} @@ -305,6 +393,11 @@ IAM_TOKEN_DB_AUTH / AZURE_POSTGRESQL_AUTH toggle that only the writer sets. secretKeyRef: name: {{ required "database.writer.passwordSecret.name is required" .passwordSecret.name }} key: {{ .passwordSecret.usernameKey | default "username" }} +- name: DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: {{ .passwordSecret.name }} + key: {{ .passwordSecret.usernameKey | default "username" }} - name: DATABASE_NAME value: {{ required "database.writer.dbname is required" .dbname | quote }} {{- if .schema }} @@ -502,7 +595,7 @@ harmless no-op for the Job and authoritative for the app pods. {{- end -}} {{/* -In-container PgBouncer env for the gateway container. Under IAM or Entra auth the pooler mints and renews the database token itself. +In-container PgBouncer env for the gateway and proxy containers. Under IAM or Entra auth the pooler mints and renews the database token itself. */}} {{- define "litellm.connectionPoolEnv" -}} {{- with .Values.database.connectionPool -}} @@ -539,7 +632,7 @@ than silently replaced by the fallback. {{- $max := $component.pdb.maxUnavailable -}} {{- $minSet := not (or (kindIs "invalid" $min) (eq (printf "%v" $min) "")) -}} {{- $maxSet := not (or (kindIs "invalid" $max) (eq (printf "%v" $max) "")) -}} -{{- if and $component.enabled $component.pdb $component.pdb.enabled }} +{{- if and .enabled $component.pdb $component.pdb.enabled }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: @@ -597,6 +690,16 @@ ImplementationSpecific {{- end -}} {{- end -}} +{{/* +Service spec fields shared by the component and monolith Services. +Invoke with the component's `service` dict. +*/}} +{{- define "litellm.service.extras" -}} +{{- if and (eq .type "LoadBalancer") .loadBalancerClass -}} +loadBalancerClass: {{ .loadBalancerClass | quote }} +{{- end }} +{{- end -}} + {{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}} {{/* @@ -655,3 +758,27 @@ shutdown drain window. {{- end }} {{- end }} {{- end -}} +{{- define "litellm.liteadminEnv" -}} +{{- $root := .root -}} +{{- with $root.Values.liteadmin }} +{{- if .enabled }} +- name: LITELLM_ADMIN_AGENT_URL + value: {{ printf "http://%s-liteadmin:10000" (include "litellm.fullname" $root | trunc 53 | trimSuffix "-") | quote }} +- name: ADMIN_AGENT_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: {{ required "liteadmin.existingSecret is required" .existingSecret }} + key: ADMIN_AGENT_SERVICE_TOKEN +{{- $hasProxyBaseUrl := false }} +{{- range $.component.extraEnv }} +{{- if eq .name "PROXY_BASE_URL" }} +{{- $hasProxyBaseUrl = true }} +{{- end }} +{{- end }} +{{- if not $hasProxyBaseUrl }} +- name: PROXY_BASE_URL + value: {{ required "liteadmin.gatewayUrl is required" .gatewayUrl | quote }} +{{- end }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/helm/litellm/templates/_workload.tpl b/helm/litellm/templates/_workload.tpl new file mode 100644 index 00000000000..2b6b1434be9 --- /dev/null +++ b/helm/litellm/templates/_workload.tpl @@ -0,0 +1,519 @@ +{{/* +The pod that runs the LLM data plane. In componentized mode it is the gateway +Deployment (`args: [gateway, ...]`), in monolith mode the proxy Deployment +(`args: [proxy, ...]`) that also serves the management API and the Admin UI. +Both are configured by .Values.gateway; the name, component label, selector +and args come from the litellm.workload.* helpers. +*/}} +{{- define "litellm.workload.deployment" -}} +{{- $component := include "litellm.workload.componentName" . -}} +{{- $fullname := include "litellm.workload.fullname" . -}} +{{- if and .Values.gateway.hpa.enabled .Values.gateway.keda.enabled }} +{{- fail "gateway.hpa.enabled and gateway.keda.enabled are mutually exclusive: two autoscalers on one Deployment fight over the replica count, so set gateway.hpa.enabled: false when using KEDA" }} +{{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $fullname }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: {{ $component }} + {{- with .Values.gateway.deploymentLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.gateway.deploymentAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and (not .Values.gateway.hpa.enabled) (not .Values.gateway.keda.enabled) (not (kindIs "invalid" .Values.gateway.replicaCount)) }} + replicas: {{ .Values.gateway.replicaCount }} + {{- end }} + {{- $minReady := .Values.gateway.minReadySeconds }} + {{- if not (or (kindIs "invalid" $minReady) (eq (printf "%v" $minReady) "")) }} + minReadySeconds: {{ $minReady }} + {{- end }} + {{- with .Values.gateway.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "litellm.workload.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- if .Values.gateway.config.create }} + checksum/config: {{ include (print $.Template.BasePath "/gateway/configmap.yaml") . | sha256sum }} + {{- end }} + {{- with .Values.gateway.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "litellm.workload.selectorLabels" . | nindent 8 }} + {{- with .Values.gateway.podLabels }} + {{- include "litellm.podLabels" (dict "podLabels" . "componentName" "gateway") | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "litellm.gateway.serviceAccountName" . }} + automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }} + {{- with .Values.gateway.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.extraInitContainers }} + initContainers: + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + containers: + - name: {{ $component }} + image: {{ include "litellm.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.gateway.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + args: + {{- if .Values.monolith.enabled }} + {{- include "litellm.proxy.args" . | nindent 12 }} + {{- else }} + - gateway + - --host + - 0.0.0.0 + - --port + - "4000" + {{- end }} + ports: + - name: http + containerPort: 4000 + protocol: TCP + env: + {{- include "litellm.lensConnectionEnv" . | nindent 12 }} + {{- if and (not .Values.monolith.enabled) .Values.ui.publicUrl }} + - name: LITELLM_CORS_ORIGINS + value: {{ include "litellm.ui.publicOrigin" . | quote }} + {{- end }} + {{- include "litellm.masterKeyEnv" $ | nindent 12 }} + {{- if .Values.monolith.enabled }} + {{- include "litellm.liteadminEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }} + {{- end }} + {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }} + {{- if .Values.gateway.config.create }} + - name: CONFIG_FILE_PATH + value: /app/config/config.yaml + {{- end }} + {{- if .Values.gateway.numWorkers }} + - name: NUM_WORKERS + value: {{ .Values.gateway.numWorkers | quote }} + {{- end }} + {{- if .Values.database.connectionPool.enabled }} + {{- include "litellm.connectionPoolEnv" $ | nindent 12 }} + {{- end }} + {{- if .Values.billingMetrics.enabled }} + {{- include "litellm.billingMetricsEnv" . | nindent 12 }} + {{- end }} + {{- if .Values.gateway.metricsServer.enabled }} + {{- if eq (int .Values.gateway.metricsServer.port) 4000 }} + {{- fail (printf "gateway.metricsServer.port must differ from the %s port 4000" $component) }} + {{- end }} + - name: PROMETHEUS_MULTIPROC_DIR + value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} + {{- end }} + {{- if .Values.gateway.collector.enabled }} + {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} + {{- end }} + {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} + {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} + volumeMounts: + {{- if .Values.gateway.config.create }} + - name: gateway-config + mountPath: /app/config/config.yaml + subPath: config.yaml + {{- end }} + {{- if .Values.gateway.metricsServer.enabled }} + - name: prometheus-multiproc + mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }} + {{- end }} + {{- if include "litellm.gateway.collectorSocketDir" . }} + - name: collector-socket + mountPath: {{ include "litellm.gateway.collectorSocketDir" . }} + {{- end }} + {{- with .Values.gateway.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- with .Values.gateway.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.gateway.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.gateway.startupProbe }} + startupProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.gateway.lifecycle }} + lifecycle: + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.gateway.resources | nindent 12 }} + {{- if .Values.gateway.metricsServer.enabled }} + - name: metrics + image: {{ include "litellm.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.gateway.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + args: + - metrics + - --port + - {{ .Values.gateway.metricsServer.port | quote }} + env: + - name: PROMETHEUS_MULTIPROC_DIR + value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} + ports: + - name: metrics + containerPort: {{ .Values.gateway.metricsServer.port }} + protocol: TCP + volumeMounts: + - name: prometheus-multiproc + mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }} + readinessProbe: + tcpSocket: { port: metrics } + periodSeconds: 10 + livenessProbe: + tcpSocket: { port: metrics } + periodSeconds: 15 + failureThreshold: 6 + resources: + {{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }} + {{- end }} + {{- if .Values.gateway.collector.enabled }} + - name: collector + image: {{ include "litellm.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.gateway.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + args: + - collector + env: + {{- include "litellm.masterKeyEnv" $ | nindent 12 }} + {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }} + {{- if .Values.gateway.config.create }} + - name: CONFIG_FILE_PATH + value: /app/config/config.yaml + {{- end }} + {{- if .Values.database.connectionPool.enabled }} + {{- include "litellm.connectionPoolEnv" $ | nindent 12 }} + {{- end }} + {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} + - name: LITELLM_JOB_ROLE + value: collector + {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} + {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts (include "litellm.gateway.collectorSocketDir" .) }} + volumeMounts: + {{- if .Values.gateway.config.create }} + - name: gateway-config + mountPath: /app/config/config.yaml + subPath: config.yaml + {{- end }} + {{- if include "litellm.gateway.collectorSocketDir" . }} + - name: collector-socket + mountPath: {{ include "litellm.gateway.collectorSocketDir" . }} + {{- end }} + {{- with .Values.gateway.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + resources: + {{- toYaml .Values.gateway.collector.resources | nindent 12 }} + {{- end }} + {{- with .Values.gateway.extraContainers }} + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} + volumes: + {{- if .Values.gateway.config.create }} + - name: gateway-config + configMap: + name: {{ include "litellm.workload.fullname" . }}-config + {{- end }} + {{- if .Values.gateway.metricsServer.enabled }} + - name: prometheus-multiproc + emptyDir: {} + {{- end }} + {{- if include "litellm.gateway.collectorSocketDir" . }} + - name: collector-socket + emptyDir: + sizeLimit: 1Mi + {{- end }} + {{- with .Values.gateway.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- with .Values.gateway.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- $gracePeriod := .Values.gateway.terminationGracePeriodSeconds }} + {{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }} + terminationGracePeriodSeconds: {{ $gracePeriod }} + {{- end }} +{{- end -}} + +{{- define "litellm.workload.service" -}} +{{- $component := include "litellm.workload.componentName" . -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "litellm.workload.fullname" . }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: {{ $component }} + {{- with .Values.gateway.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.gateway.service.type }} + {{- with include "litellm.service.extras" .Values.gateway.service }} + {{- . | nindent 2 }} + {{- end }} + ports: + - port: {{ .Values.gateway.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "litellm.workload.selectorLabels" . | nindent 4 }} +{{- end -}} + +{{- define "litellm.workload.hpa" -}} +{{- $component := include "litellm.workload.componentName" . -}} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "litellm.workload.fullname" . }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: {{ $component }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "litellm.workload.fullname" . }} + minReplicas: {{ .Values.gateway.hpa.minReplicas }} + maxReplicas: {{ .Values.gateway.hpa.maxReplicas }} + metrics: + {{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }} + {{- if and .Values.gateway.collector.enabled .Values.gateway.collector.scaleOnGatewayContainerCpu }} + - type: ContainerResource + containerResource: + name: cpu + container: {{ $component }} + target: + type: Utilization + averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }} + {{- else }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }} + {{- end }} + {{- end }} + {{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }} + {{- end }} + {{- with .Values.gateway.hpa.targetRequestsPerSecond }} + - type: Pods + pods: + metric: + name: litellm_requests_per_second + target: + type: AverageValue + averageValue: {{ toJson . | trimAll "\"" | quote }} + {{- end }} + {{- with .Values.gateway.hpa.targetTokensPerSecond }} + - type: Pods + pods: + metric: + name: litellm_tokens_per_second + target: + type: AverageValue + averageValue: {{ toJson . | trimAll "\"" | quote }} + {{- end }} + {{- with .Values.gateway.hpa.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} + +{{/* +KEDA ScaledObject targeting the workload Deployment. The optional Prometheus +triggers query the same per pod counters the HPA workload metrics use, +scoped to this release's metrics Service through the `job` label the +ServiceMonitor gives every scrape. +*/}} +{{- define "litellm.workload.keda" -}} +{{- $component := include "litellm.workload.componentName" . -}} +{{- $keda := .Values.gateway.keda -}} +{{- $prom := $keda.prometheus -}} +{{- $wantsPrometheus := or $prom.targetRequestsPerSecond $prom.targetTokensPerSecond -}} +{{- if and $wantsPrometheus (not $prom.serverAddress) }} +{{- fail "gateway.keda.prometheus.serverAddress is required when gateway.keda.prometheus.targetRequestsPerSecond or targetTokensPerSecond is set" }} +{{- end }} +{{- if and $wantsPrometheus (not .Values.gateway.metricsServer.enabled) }} +{{- fail "gateway.metricsServer.enabled is required when gateway.keda.prometheus.targetRequestsPerSecond or targetTokensPerSecond is set" }} +{{- end }} +{{- $selector := printf "namespace=%q,job=%q" .Release.Namespace (printf "%s-metrics" (include "litellm.workload.fullname" .)) -}} +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ include "litellm.workload.fullname" . }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: {{ $component }} + {{- with (dig "scaledObject" "annotations" (dict) $keda) }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "litellm.workload.fullname" . }} + minReplicaCount: {{ $keda.minReplicaCount }} + maxReplicaCount: {{ $keda.maxReplicaCount }} + pollingInterval: {{ $keda.pollingInterval }} + cooldownPeriod: {{ $keda.cooldownPeriod }} + {{- with $keda.advanced }} + advanced: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $keda.fallback }} + {{- $fallback := deepCopy . }} + {{- if not (hasKey $fallback "failureThreshold") }} + {{- $_ := set $fallback "failureThreshold" 3 }} + {{- end }} + {{- if not (hasKey $fallback "replicas") }} + {{- $_ := set $fallback "replicas" $keda.maxReplicaCount }} + {{- end }} + fallback: + {{- toYaml $fallback | nindent 4 }} + {{- end }} + triggers: + {{- with $keda.triggers }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $prom.targetRequestsPerSecond }} + - type: prometheus + metricType: AverageValue + metadata: + serverAddress: {{ $prom.serverAddress | quote }} + query: {{ printf "sum(rate(litellm_proxy_total_requests_metric_total{%s}[1m]))" $selector | quote }} + threshold: {{ toJson . | trimAll "\"" | quote }} + {{- end }} + {{- with $prom.targetTokensPerSecond }} + - type: prometheus + metricType: AverageValue + metadata: + serverAddress: {{ $prom.serverAddress | quote }} + query: {{ printf "sum(rate(litellm_total_tokens_metric_total{%s}[1m]))" $selector | quote }} + threshold: {{ toJson . | trimAll "\"" | quote }} + {{- end }} +{{- end -}} + +{{- define "litellm.workload.serviceMetrics" -}} +{{- $component := include "litellm.workload.componentName" . -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "litellm.workload.fullname" . }}-metrics + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: {{ $component }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.gateway.metricsServer.port }} + targetPort: metrics + protocol: TCP + name: metrics + selector: + {{- include "litellm.workload.selectorLabels" . | nindent 4 }} +{{- end -}} + +{{- define "litellm.workload.serviceMonitor" -}} +{{- $component := include "litellm.workload.componentName" . -}} +{{- if not .Values.gateway.metricsServer.enabled }} +{{- fail "gateway.serviceMonitor.enabled requires gateway.metricsServer.enabled: the http port serves /metrics/ behind virtual-key auth, so an unauthenticated scrape gets 401" }} +{{- end }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "litellm.workload.fullname" . }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: {{ $component }} + {{- with .Values.gateway.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.gateway.serviceMonitor.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "litellm.workload.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + {{- with .Values.gateway.serviceMonitor.namespaceSelector.matchNames }} + {{- toYaml . | nindent 6 }} + {{- else }} + - {{ .Release.Namespace | quote }} + {{- end }} + endpoints: + - port: metrics + path: /metrics/ + interval: {{ .Values.gateway.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.gateway.serviceMonitor.scrapeTimeout }} + scheme: {{ .Values.gateway.serviceMonitor.scheme }} + {{- with .Values.gateway.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gateway.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} diff --git a/helm/litellm/templates/backend/deployment.yaml b/helm/litellm/templates/backend/deployment.yaml index 7360261dab1..9118c6f3d88 100644 --- a/helm/litellm/templates/backend/deployment.yaml +++ b/helm/litellm/templates/backend/deployment.yaml @@ -1,4 +1,4 @@ -{{- if .Values.backend.enabled }} +{{- if include "litellm.backend.render" . }} {{- $componentHosts := .Values.ingress.componentHosts | default dict -}} {{- $hasComponentHosts := or $componentHosts.ui $componentHosts.backend $componentHosts.gateway -}} apiVersion: apps/v1 @@ -8,10 +8,21 @@ metadata: labels: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: backend + {{- with .Values.backend.deploymentLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.backend.deploymentAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: {{- if and (not .Values.backend.hpa.enabled) (not (kindIs "invalid" .Values.backend.replicaCount)) }} replicas: {{ .Values.backend.replicaCount }} {{- end }} + {{- $minReady := .Values.backend.minReadySeconds }} + {{- if not (or (kindIs "invalid" $minReady) (eq (printf "%v" $minReady) "")) }} + minReadySeconds: {{ $minReady }} + {{- end }} {{- with .Values.backend.strategy }} strategy: {{- toYaml . | nindent 4 }} @@ -46,14 +57,24 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.backend.extraInitContainers }} + initContainers: + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} containers: - name: backend - image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.backend.image.pullPolicy }} + image: {{ include "litellm.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.backend.securityContext }} securityContext: {{- toYaml . | nindent 12 }} {{- end }} + args: + - backend + - --host + - 0.0.0.0 + - --port + - "4001" ports: - name: http containerPort: 4001 @@ -66,6 +87,8 @@ spec: - name: LITELLM_CORS_ALLOW_CREDENTIALS value: "true" {{- end }} + {{- include "litellm.masterKeyEnv" $ | nindent 12 }} + {{- include "litellm.liteadminEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }} {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }} {{- if .Values.backend.gatewayUrl }} - name: LITELLM_GATEWAY_URL @@ -79,16 +102,13 @@ spec: {{- include "litellm.billingMetricsEnv" . | nindent 12 }} {{- end }} {{- include "litellm.envFrom" .Values.backend | nindent 10 }} - {{- if or .Values.gateway.config.create .Values.backend.volumeMounts .Values.billingMetrics.enabled }} + {{- if or .Values.gateway.config.create .Values.backend.volumeMounts }} volumeMounts: {{- if .Values.gateway.config.create }} - name: gateway-config mountPath: /app/config/config.yaml subPath: config.yaml {{- end }} - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }} - {{- end }} {{- with .Values.backend.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} @@ -114,15 +134,12 @@ spec: {{- with .Values.backend.extraContainers }} {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} - {{- if or .Values.gateway.config.create .Values.backend.volumes .Values.billingMetrics.enabled }} + {{- if or .Values.gateway.config.create .Values.backend.volumes }} volumes: {{- if .Values.gateway.config.create }} - name: gateway-config configMap: - name: {{ include "litellm.gateway.fullname" . }}-config - {{- end }} - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsVolumes" . | nindent 8 }} + name: {{ include "litellm.workload.fullname" . }}-config {{- end }} {{- with .Values.backend.volumes }} {{- toYaml . | nindent 8 }} diff --git a/helm/litellm/templates/backend/hpa.yaml b/helm/litellm/templates/backend/hpa.yaml index a414092fb39..62efaacb830 100644 --- a/helm/litellm/templates/backend/hpa.yaml +++ b/helm/litellm/templates/backend/hpa.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.backend.enabled .Values.backend.hpa.enabled }} +{{- if and (include "litellm.backend.render" .) .Values.backend.hpa.enabled }} apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: diff --git a/helm/litellm/templates/backend/poddisruptionbudget.yaml b/helm/litellm/templates/backend/poddisruptionbudget.yaml index 02853ac879c..ed40bda0d7c 100644 --- a/helm/litellm/templates/backend/poddisruptionbudget.yaml +++ b/helm/litellm/templates/backend/poddisruptionbudget.yaml @@ -1,5 +1,6 @@ {{- include "litellm.pdb" (dict "root" $ + "enabled" (include "litellm.backend.render" .) "component" .Values.backend "componentName" "backend" "fullname" (include "litellm.backend.fullname" .) diff --git a/helm/litellm/templates/backend/service.yaml b/helm/litellm/templates/backend/service.yaml index d480c654784..cd11ad95ccf 100644 --- a/helm/litellm/templates/backend/service.yaml +++ b/helm/litellm/templates/backend/service.yaml @@ -1,4 +1,4 @@ -{{- if .Values.backend.enabled }} +{{- if include "litellm.backend.render" . }} apiVersion: v1 kind: Service metadata: @@ -6,8 +6,15 @@ metadata: labels: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: backend + {{- with .Values.backend.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: type: {{ .Values.backend.service.type }} + {{- with include "litellm.service.extras" .Values.backend.service }} + {{- . | nindent 2 }} + {{- end }} ports: - port: {{ .Values.backend.service.port }} targetPort: http diff --git a/helm/litellm/templates/extra-resources.yaml b/helm/litellm/templates/extra-resources.yaml new file mode 100644 index 00000000000..bd809ffb696 --- /dev/null +++ b/helm/litellm/templates/extra-resources.yaml @@ -0,0 +1,4 @@ +{{- range .Values.extraResources }} +--- +{{ tpl (toYaml .) $ }} +{{- end }} diff --git a/helm/litellm/templates/gateway/configmap.yaml b/helm/litellm/templates/gateway/configmap.yaml index d262bf25b87..8944aaa0a2f 100644 --- a/helm/litellm/templates/gateway/configmap.yaml +++ b/helm/litellm/templates/gateway/configmap.yaml @@ -2,7 +2,10 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "litellm.gateway.fullname" . }}-config + name: {{ include "litellm.workload.fullname" . }}-config + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: {{ include "litellm.workload.componentName" . }} data: config.yaml: | {{ .Values.gateway.config.proxy_config | toYaml | indent 6 }} diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml index b0e7f0dcebb..9db08850e26 100644 --- a/helm/litellm/templates/gateway/deployment.yaml +++ b/helm/litellm/templates/gateway/deployment.yaml @@ -1,252 +1,3 @@ -{{- if .Values.gateway.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "litellm.gateway.fullname" . }} - labels: - {{- include "litellm.commonLabels" . | nindent 4 }} - app.kubernetes.io/component: gateway -spec: - {{- if and (not .Values.gateway.hpa.enabled) (not (kindIs "invalid" .Values.gateway.replicaCount)) }} - replicas: {{ .Values.gateway.replicaCount }} - {{- end }} - {{- with .Values.gateway.strategy }} - strategy: - {{- toYaml . | nindent 4 }} - {{- end }} - selector: - matchLabels: - {{- include "litellm.gateway.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - {{- if .Values.gateway.config.create }} - checksum/config: {{ include (print $.Template.BasePath "/gateway/configmap.yaml") . | sha256sum }} - {{- end }} - {{- with .Values.gateway.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "litellm.gateway.selectorLabels" . | nindent 8 }} - {{- with .Values.gateway.podLabels }} - {{- include "litellm.podLabels" (dict "podLabels" . "componentName" "gateway") | nindent 8 }} - {{- end }} - spec: - serviceAccountName: {{ include "litellm.gateway.serviceAccountName" . }} - automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }} - {{- with .Values.gateway.podSecurityContext }} - securityContext: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: gateway - image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.gateway.image.pullPolicy }} - {{- with .Values.gateway.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 4000 - protocol: TCP - env: - {{- include "litellm.lensConnectionEnv" . | nindent 12 }} - {{- if .Values.ui.publicUrl }} - - name: LITELLM_CORS_ORIGINS - value: {{ include "litellm.ui.publicOrigin" . | quote }} - {{- end }} - {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }} - {{- if .Values.gateway.config.create }} - - name: CONFIG_FILE_PATH - value: /app/config/config.yaml - {{- end }} - {{- if .Values.gateway.numWorkers }} - - name: NUM_WORKERS - value: {{ .Values.gateway.numWorkers | quote }} - {{- end }} - {{- if .Values.database.connectionPool.enabled }} - {{- include "litellm.connectionPoolEnv" $ | nindent 12 }} - {{- end }} - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsEnv" . | nindent 12 }} - {{- end }} - {{- if .Values.gateway.metricsServer.enabled }} - {{- if eq (int .Values.gateway.metricsServer.port) 4000 }} - {{- fail "gateway.metricsServer.port must differ from the gateway port 4000" }} - {{- end }} - - name: PROMETHEUS_MULTIPROC_DIR - value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} - {{- end }} - {{- if .Values.gateway.collector.enabled }} - {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} - {{- end }} - {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} - {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} - volumeMounts: - {{- if .Values.gateway.config.create }} - - name: gateway-config - mountPath: /app/config/config.yaml - subPath: config.yaml - {{- end }} - {{- if .Values.gateway.metricsServer.enabled }} - - name: prometheus-multiproc - mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }} - {{- end }} - {{- if include "litellm.gateway.collectorSocketDir" . }} - - name: collector-socket - mountPath: {{ include "litellm.gateway.collectorSocketDir" . }} - {{- end }} - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }} - {{- end }} - {{- with .Values.gateway.volumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- end }} - {{- with .Values.gateway.livenessProbe }} - livenessProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.gateway.readinessProbe }} - readinessProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.gateway.startupProbe }} - startupProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.gateway.lifecycle }} - lifecycle: - {{- toYaml . | nindent 12 }} - {{- end }} - resources: - {{- toYaml .Values.gateway.resources | nindent 12 }} - {{- if .Values.gateway.metricsServer.enabled }} - - name: metrics - image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.gateway.image.pullPolicy }} - {{- with .Values.gateway.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - command: - - python - - -m - - litellm.proxy.prometheus_metrics_server - - --port - - {{ .Values.gateway.metricsServer.port | quote }} - env: - - name: PROMETHEUS_MULTIPROC_DIR - value: {{ include "litellm.gateway.prometheusMultiprocDir" . }} - ports: - - name: metrics - containerPort: {{ .Values.gateway.metricsServer.port }} - protocol: TCP - volumeMounts: - - name: prometheus-multiproc - mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }} - readinessProbe: - tcpSocket: { port: metrics } - periodSeconds: 10 - livenessProbe: - tcpSocket: { port: metrics } - periodSeconds: 15 - failureThreshold: 6 - resources: - {{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }} - {{- end }} - {{- if .Values.gateway.collector.enabled }} - - name: collector - image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.gateway.image.pullPolicy }} - {{- with .Values.gateway.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - command: - - python - - -m - - litellm.proxy.collector - env: - {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }} - {{- if .Values.gateway.config.create }} - - name: CONFIG_FILE_PATH - value: /app/config/config.yaml - {{- end }} - {{- if .Values.database.connectionPool.enabled }} - {{- include "litellm.connectionPoolEnv" $ | nindent 12 }} - {{- end }} - {{- include "litellm.gateway.collectorEnv" . | nindent 12 }} - - name: LITELLM_JOB_ROLE - value: collector - {{- include "litellm.envFrom" .Values.gateway | nindent 10 }} - {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts (include "litellm.gateway.collectorSocketDir" .) }} - volumeMounts: - {{- if .Values.gateway.config.create }} - - name: gateway-config - mountPath: /app/config/config.yaml - subPath: config.yaml - {{- end }} - {{- if include "litellm.gateway.collectorSocketDir" . }} - - name: collector-socket - mountPath: {{ include "litellm.gateway.collectorSocketDir" . }} - {{- end }} - {{- with .Values.gateway.volumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- end }} - resources: - {{- toYaml .Values.gateway.collector.resources | nindent 12 }} - {{- end }} - {{- with .Values.gateway.extraContainers }} - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }} - volumes: - {{- if .Values.gateway.config.create }} - - name: gateway-config - configMap: - name: {{ include "litellm.gateway.fullname" . }}-config - {{- end }} - {{- if .Values.gateway.metricsServer.enabled }} - - name: prometheus-multiproc - emptyDir: {} - {{- end }} - {{- if include "litellm.gateway.collectorSocketDir" . }} - - name: collector-socket - emptyDir: - sizeLimit: 1Mi - {{- end }} - {{- if .Values.billingMetrics.enabled }} - {{- include "litellm.billingMetricsVolumes" . | nindent 8 }} - {{- end }} - {{- with .Values.gateway.volumes }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- end }} - {{- with .Values.gateway.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.gateway.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.gateway.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.gateway.topologySpreadConstraints }} - topologySpreadConstraints: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- $gracePeriod := .Values.gateway.terminationGracePeriodSeconds }} - {{- if not (or (kindIs "invalid" $gracePeriod) (eq (printf "%v" $gracePeriod) "")) }} - terminationGracePeriodSeconds: {{ $gracePeriod }} - {{- end }} +{{- if include "litellm.gateway.render" . }} +{{- include "litellm.workload.deployment" . }} {{- end }} diff --git a/helm/litellm/templates/gateway/hpa.yaml b/helm/litellm/templates/gateway/hpa.yaml index e7094e96106..d416a6056bd 100644 --- a/helm/litellm/templates/gateway/hpa.yaml +++ b/helm/litellm/templates/gateway/hpa.yaml @@ -1,65 +1,3 @@ -{{- if and .Values.gateway.enabled .Values.gateway.hpa.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "litellm.gateway.fullname" . }} - labels: - {{- include "litellm.commonLabels" . | nindent 4 }} - app.kubernetes.io/component: gateway -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "litellm.gateway.fullname" . }} - minReplicas: {{ .Values.gateway.hpa.minReplicas }} - maxReplicas: {{ .Values.gateway.hpa.maxReplicas }} - metrics: - {{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }} - {{- if and .Values.gateway.collector.enabled .Values.gateway.collector.scaleOnGatewayContainerCpu }} - - type: ContainerResource - containerResource: - name: cpu - container: gateway - target: - type: Utilization - averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }} - {{- else }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }} - {{- end }} - {{- end }} - {{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }} - {{- end }} - {{- with .Values.gateway.hpa.targetRequestsPerSecond }} - - type: Pods - pods: - metric: - name: litellm_requests_per_second - target: - type: AverageValue - averageValue: {{ toJson . | trimAll "\"" | quote }} - {{- end }} - {{- with .Values.gateway.hpa.targetTokensPerSecond }} - - type: Pods - pods: - metric: - name: litellm_tokens_per_second - target: - type: AverageValue - averageValue: {{ toJson . | trimAll "\"" | quote }} - {{- end }} - {{- with .Values.gateway.hpa.behavior }} - behavior: - {{- toYaml . | nindent 4 }} - {{- end }} +{{- if and (include "litellm.gateway.render" .) .Values.gateway.hpa.enabled }} +{{- include "litellm.workload.hpa" . }} {{- end }} diff --git a/helm/litellm/templates/gateway/keda.yaml b/helm/litellm/templates/gateway/keda.yaml new file mode 100644 index 00000000000..d46085ffef1 --- /dev/null +++ b/helm/litellm/templates/gateway/keda.yaml @@ -0,0 +1,3 @@ +{{- if and (include "litellm.gateway.render" .) .Values.gateway.keda.enabled }} +{{- include "litellm.workload.keda" . }} +{{- end }} diff --git a/helm/litellm/templates/gateway/poddisruptionbudget.yaml b/helm/litellm/templates/gateway/poddisruptionbudget.yaml index 15e89af17d7..313e3cc7535 100644 --- a/helm/litellm/templates/gateway/poddisruptionbudget.yaml +++ b/helm/litellm/templates/gateway/poddisruptionbudget.yaml @@ -1,5 +1,6 @@ {{- include "litellm.pdb" (dict "root" $ + "enabled" (include "litellm.gateway.render" .) "component" .Values.gateway "componentName" "gateway" "fullname" (include "litellm.gateway.fullname" .) diff --git a/helm/litellm/templates/gateway/service-metrics.yaml b/helm/litellm/templates/gateway/service-metrics.yaml index ad9bc05a9fd..e33ae8c63e3 100644 --- a/helm/litellm/templates/gateway/service-metrics.yaml +++ b/helm/litellm/templates/gateway/service-metrics.yaml @@ -1,18 +1,3 @@ -{{- if and .Values.gateway.enabled .Values.gateway.metricsServer.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "litellm.gateway.fullname" . }}-metrics - labels: - {{- include "litellm.commonLabels" . | nindent 4 }} - app.kubernetes.io/component: gateway -spec: - type: ClusterIP - ports: - - port: {{ .Values.gateway.metricsServer.port }} - targetPort: metrics - protocol: TCP - name: metrics - selector: - {{- include "litellm.gateway.selectorLabels" . | nindent 4 }} +{{- if and (include "litellm.gateway.render" .) .Values.gateway.metricsServer.enabled }} +{{- include "litellm.workload.serviceMetrics" . }} {{- end }} diff --git a/helm/litellm/templates/gateway/service.yaml b/helm/litellm/templates/gateway/service.yaml index 03a4167a0ab..8f3e7ad0673 100644 --- a/helm/litellm/templates/gateway/service.yaml +++ b/helm/litellm/templates/gateway/service.yaml @@ -1,18 +1,3 @@ -{{- if .Values.gateway.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "litellm.gateway.fullname" . }} - labels: - {{- include "litellm.commonLabels" . | nindent 4 }} - app.kubernetes.io/component: gateway -spec: - type: {{ .Values.gateway.service.type }} - ports: - - port: {{ .Values.gateway.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "litellm.gateway.selectorLabels" . | nindent 4 }} +{{- if include "litellm.gateway.render" . }} +{{- include "litellm.workload.service" . }} {{- end }} diff --git a/helm/litellm/templates/gateway/servicemonitor.yaml b/helm/litellm/templates/gateway/servicemonitor.yaml index e1bafa6e388..3e5d538b4e0 100644 --- a/helm/litellm/templates/gateway/servicemonitor.yaml +++ b/helm/litellm/templates/gateway/servicemonitor.yaml @@ -1,28 +1,3 @@ -{{- if and .Values.gateway.enabled .Values.gateway.serviceMonitor.enabled }} -{{- if not .Values.gateway.metricsServer.enabled }} -{{- fail "gateway.serviceMonitor.enabled requires gateway.metricsServer.enabled: the http port serves /metrics/ behind virtual-key auth, so an unauthenticated scrape gets 401" }} -{{- end }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ include "litellm.gateway.fullname" . }} - labels: - {{- include "litellm.commonLabels" . | nindent 4 }} - app.kubernetes.io/component: gateway - {{- with .Values.gateway.serviceMonitor.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - selector: - matchLabels: - {{- include "litellm.gateway.selectorLabels" . | nindent 6 }} - namespaceSelector: - matchNames: - - {{ .Release.Namespace | quote }} - endpoints: - - port: metrics - path: /metrics/ - interval: {{ .Values.gateway.serviceMonitor.interval }} - scrapeTimeout: {{ .Values.gateway.serviceMonitor.scrapeTimeout }} - scheme: http +{{- if and (include "litellm.gateway.render" .) .Values.gateway.serviceMonitor.enabled }} +{{- include "litellm.workload.serviceMonitor" . }} {{- end }} diff --git a/helm/litellm/templates/ingress.yaml b/helm/litellm/templates/ingress.yaml index 9f778e78087..3fb138aacdb 100644 --- a/helm/litellm/templates/ingress.yaml +++ b/helm/litellm/templates/ingress.yaml @@ -4,6 +4,7 @@ {{- if not (and $componentHosts.ui $componentHosts.backend $componentHosts.gateway) }} {{- fail "ingress.componentHosts.ui, backend, and gateway must all be set together" }} {{- end }} +{{- if not .Values.monolith.enabled }} {{- if not .Values.gateway.enabled }} {{- fail "ingress.componentHosts requires gateway.enabled=true" }} {{- end }} @@ -13,12 +14,14 @@ {{- if not .Values.ui.enabled }} {{- fail "ingress.componentHosts requires ui.enabled=true" }} {{- end }} +{{- end }} {{- if .Values.ingress.host }} {{- fail "ingress.host must be empty when ingress.componentHosts is configured" }} {{- end }} {{- if .Values.ingress.extraPaths }} {{- fail "ingress.extraPaths must be empty when ingress.componentHosts is configured" }} {{- end }} +{{- if not .Values.monolith.enabled }} {{- if not (trim .Values.ui.backendUrl) }} {{- fail "ui.backendUrl is required when ingress.componentHosts is configured" }} {{- end }} @@ -31,24 +34,30 @@ {{- fail "ui.publicUrl is required when ingress.componentHosts is configured" }} {{- end }} {{- end }} +{{- end }} {{- if .Values.ingress.enabled -}} +{{- $monolith := .Values.monolith.enabled -}} {{- $gatewayName := include "litellm.gateway.fullname" . -}} {{- $backendName := include "litellm.backend.fullname" . -}} {{- $uiName := include "litellm.ui.fullname" . -}} {{- $gatewayPort := .Values.gateway.service.port -}} {{- $backendPort := .Values.backend.service.port -}} {{- $uiPort := .Values.ui.service.port -}} +{{- $proxyName := include "litellm.proxy.fullname" . -}} {{- $controller := .Values.ingress.controller | default "alb" -}} {{- if not (has $controller (list "alb" "nginx")) }} {{- fail (printf "ingress.controller: unknown controller %q, expected one of alb, nginx" $controller) }} {{- end }} {{/* Backends addressable from ingress.extraPaths, keyed by the `service` field. + In monolith mode the one proxy Service serves every component, so every + key resolves to it and an entry's `service` only has to be a known name. */}} +{{- $proxyBackend := dict "name" $proxyName "port" $gatewayPort -}} {{- $extraPathBackends := dict - "gateway" (dict "name" $gatewayName "port" $gatewayPort) - "backend" (dict "name" $backendName "port" $backendPort) - "ui" (dict "name" $uiName "port" $uiPort) + "gateway" (ternary $proxyBackend (dict "name" $gatewayName "port" $gatewayPort) $monolith) + "backend" (ternary $proxyBackend (dict "name" $backendName "port" $backendPort) $monolith) + "ui" (ternary $proxyBackend (dict "name" $uiName "port" $uiPort) $monolith) -}} {{/* UI paths (Next.js static export). @@ -122,7 +131,7 @@ at "/" Prefix would swallow the whole backend management API) instead of adding to it. */}} -{{- $builtinPathKeys := list "/test|Exact" "/debug/memory/summary|Exact" "/|Prefix" -}} +{{- $builtinPathKeys := ternary (list "/|Prefix") (list "/test|Exact" "/debug/memory/summary|Exact" "/|Prefix") $monolith -}} {{- if $hasComponentHosts }} apiVersion: networking.k8s.io/v1 kind: Ingress @@ -150,9 +159,9 @@ spec: pathType: Prefix backend: service: - name: {{ $uiName }} + name: {{ $extraPathBackends.ui.name }} port: - number: {{ $uiPort }} + number: {{ $extraPathBackends.ui.port }} - host: {{ $componentHosts.backend | quote }} http: paths: @@ -160,9 +169,9 @@ spec: pathType: Prefix backend: service: - name: {{ $backendName }} + name: {{ $extraPathBackends.backend.name }} port: - number: {{ $backendPort }} + number: {{ $extraPathBackends.backend.port }} - host: {{ $componentHosts.gateway | quote }} http: paths: @@ -179,9 +188,9 @@ spec: pathType: Prefix backend: service: - name: {{ $gatewayName }} + name: {{ $extraPathBackends.gateway.name }} port: - number: {{ $gatewayPort }} + number: {{ $extraPathBackends.gateway.port }} {{- else }} apiVersion: networking.k8s.io/v1 kind: Ingress @@ -207,6 +216,7 @@ spec: {{- end }} http: paths: + {{- if not $monolith }} # --- UI (Next.js static export) --- {{- range $uiPaths }} {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" .path "pathType" .pathType) }} @@ -249,6 +259,7 @@ spec: port: number: {{ $gatewayPort }} {{- end }} + {{- end }} {{- if and .Values.lens.enabled .Values.lens.service.name }} {{- $builtinPathKeys = append $builtinPathKeys "/lens-ingest|Prefix" }} - path: /lens-ingest @@ -324,6 +335,16 @@ spec: port: {{- toYaml $port | nindent 18 }} {{- end }} + {{- if $monolith }} + # --- Monolith: the proxy serves the UI, the data plane and the management API --- + - path: / + pathType: Prefix + backend: + service: + name: {{ $proxyName }} + port: + number: {{ $gatewayPort }} + {{- else }} # --- Catch-all → backend (management API: /key/*, /user/*, /team/*, ...) --- - path: / pathType: Prefix @@ -332,5 +353,6 @@ spec: name: {{ $backendName }} port: number: {{ $backendPort }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/litellm-helm/templates/liteadmin.yaml b/helm/litellm/templates/liteadmin.yaml similarity index 96% rename from helm/litellm-helm/templates/liteadmin.yaml rename to helm/litellm/templates/liteadmin.yaml index 711edaf6913..bc5f2c9510b 100644 --- a/helm/litellm-helm/templates/liteadmin.yaml +++ b/helm/litellm/templates/liteadmin.yaml @@ -32,7 +32,7 @@ spec: runAsNonRoot: true containers: - name: liteadmin - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + image: {{ include "litellm.image" . | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: ["--admin-agent"] securityContext: diff --git a/helm/litellm/templates/migrations-job.yaml b/helm/litellm/templates/migrations-job.yaml index de1cc2b103b..8179c26cbd7 100644 --- a/helm/litellm/templates/migrations-job.yaml +++ b/helm/litellm/templates/migrations-job.yaml @@ -1,7 +1,7 @@ {{- if .Values.migrationJob.enabled -}} # Pre-install / pre-upgrade hook that runs `prisma migrate deploy` against -# the writer database before the gateway and backend Deployments are rolled -# out. Required because the gateway and backend both spin up Prisma at +# the writer database before the application Deployments are rolled out. +# Required because the proxy, gateway and backend all spin up Prisma at # startup and assume the LiteLLM schema (LiteLLM_Config, # LiteLLM_VerificationToken, LiteLLM_SpendLogs, ...) already exists. # @@ -16,9 +16,13 @@ metadata: labels: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: migrations - {{- if or .Values.migrationJob.hooks.helm.enabled .Values.migrationJob.hooks.argocd.enabled }} + {{- with .Values.migrationJob.jobLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- $helmHook := .Values.migrationJob.hooks.helm.enabled }} + {{- if or $helmHook .Values.migrationJob.hooks.argocd.enabled .Values.migrationJob.jobAnnotations }} annotations: - {{- if .Values.migrationJob.hooks.helm.enabled }} + {{- if $helmHook }} helm.sh/hook: pre-install,pre-upgrade helm.sh/hook-delete-policy: before-hook-creation helm.sh/hook-weight: {{ .Values.migrationJob.hooks.helm.weight | default "0" | quote }} @@ -27,6 +31,9 @@ metadata: argocd.argoproj.io/hook: PreSync argocd.argoproj.io/hook-delete-policy: BeforeHookCreation {{- end }} + {{- with .Values.migrationJob.jobAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} spec: backoffLimit: {{ .Values.migrationJob.backoffLimit }} @@ -43,6 +50,10 @@ spec: {{- $chartLabels := merge (dict "app.kubernetes.io/component" "migrations") (fromYaml (include "litellm.commonLabels" .)) }} labels: {{- toYaml (merge (deepCopy .Values.migrationJob.podLabels) $chartLabels) | nindent 8 }} + {{- with .Values.migrationJob.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} spec: restartPolicy: Never serviceAccountName: {{ include "litellm.migrations.serviceAccountName" . }} @@ -55,14 +66,29 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.migrationJob.extraInitContainers }} + initContainers: + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} containers: - name: prisma-migrations - image: "{{ .Values.migrationJob.image.repository }}:{{ .Values.migrationJob.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.migrationJob.image.pullPolicy }} + image: {{ include "litellm.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.migrationJob.securityContext }} securityContext: {{- toYaml . | nindent 12 }} {{- end }} + {{- with .Values.migrationJob.command }} + command: {{ toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.migrationJob.args }} + args: {{ toYaml .Values.migrationJob.args | nindent 12 }} + {{- else if .Values.migrationJob.command }} + args: [] + {{- else }} + args: + - migrations + {{- end }} env: {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.migrationJob) | nindent 12 }} {{- with .Values.migrationJob.volumeMounts }} @@ -73,6 +99,9 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} + {{- with .Values.migrationJob.extraContainers }} + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} {{- with .Values.migrationJob.volumes }} volumes: {{- toYaml . | nindent 8 }} diff --git a/helm/litellm/templates/monolith/deployment.yaml b/helm/litellm/templates/monolith/deployment.yaml new file mode 100644 index 00000000000..7bad16386e1 --- /dev/null +++ b/helm/litellm/templates/monolith/deployment.yaml @@ -0,0 +1,3 @@ +{{- if .Values.monolith.enabled }} +{{- include "litellm.workload.deployment" . }} +{{- end }} diff --git a/helm/litellm/templates/monolith/hpa.yaml b/helm/litellm/templates/monolith/hpa.yaml new file mode 100644 index 00000000000..ee9180e71bb --- /dev/null +++ b/helm/litellm/templates/monolith/hpa.yaml @@ -0,0 +1,3 @@ +{{- if and .Values.monolith.enabled .Values.gateway.hpa.enabled }} +{{- include "litellm.workload.hpa" . }} +{{- end }} diff --git a/helm/litellm/templates/monolith/keda.yaml b/helm/litellm/templates/monolith/keda.yaml new file mode 100644 index 00000000000..d4054432e38 --- /dev/null +++ b/helm/litellm/templates/monolith/keda.yaml @@ -0,0 +1,3 @@ +{{- if and .Values.monolith.enabled .Values.gateway.keda.enabled }} +{{- include "litellm.workload.keda" . }} +{{- end }} diff --git a/helm/litellm/templates/monolith/poddisruptionbudget.yaml b/helm/litellm/templates/monolith/poddisruptionbudget.yaml new file mode 100644 index 00000000000..2be83e57dfc --- /dev/null +++ b/helm/litellm/templates/monolith/poddisruptionbudget.yaml @@ -0,0 +1,7 @@ +{{- include "litellm.pdb" (dict + "root" $ + "enabled" .Values.monolith.enabled + "component" .Values.gateway + "componentName" "proxy" + "fullname" (include "litellm.proxy.fullname" .) + "selectorLabels" (include "litellm.proxy.selectorLabels" .)) }} diff --git a/helm/litellm/templates/monolith/service-metrics.yaml b/helm/litellm/templates/monolith/service-metrics.yaml new file mode 100644 index 00000000000..07a68c453df --- /dev/null +++ b/helm/litellm/templates/monolith/service-metrics.yaml @@ -0,0 +1,3 @@ +{{- if and .Values.monolith.enabled .Values.gateway.metricsServer.enabled }} +{{- include "litellm.workload.serviceMetrics" . }} +{{- end }} diff --git a/helm/litellm/templates/monolith/service.yaml b/helm/litellm/templates/monolith/service.yaml new file mode 100644 index 00000000000..4f87259983a --- /dev/null +++ b/helm/litellm/templates/monolith/service.yaml @@ -0,0 +1,3 @@ +{{- if .Values.monolith.enabled }} +{{- include "litellm.workload.service" . }} +{{- end }} diff --git a/helm/litellm/templates/monolith/servicemonitor.yaml b/helm/litellm/templates/monolith/servicemonitor.yaml new file mode 100644 index 00000000000..29a51a379f2 --- /dev/null +++ b/helm/litellm/templates/monolith/servicemonitor.yaml @@ -0,0 +1,3 @@ +{{- if and .Values.monolith.enabled .Values.gateway.serviceMonitor.enabled }} +{{- include "litellm.workload.serviceMonitor" . }} +{{- end }} diff --git a/helm/litellm/templates/secret-masterkey.yaml b/helm/litellm/templates/secret-masterkey.yaml new file mode 100644 index 00000000000..3af2ee6f07d --- /dev/null +++ b/helm/litellm/templates/secret-masterkey.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.masterKey.generate (not .Values.masterKey.secretName) }} +{{- $name := include "litellm.masterKey.generatedSecretName" . }} +{{- $key := .Values.masterKey.secretKey | default "master-key" }} +{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }} +{{- $value := "" }} +{{- if and $existing (index $existing.data $key) }} +{{- $value = index $existing.data $key | b64dec }} +{{- else }} +{{- $value = printf "sk-%s" (randAlphaNum 32) }} +{{- end }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +data: + {{ $key }}: {{ $value | b64enc }} +{{- end }} diff --git a/helm/litellm/templates/serviceaccount.yaml b/helm/litellm/templates/serviceaccount.yaml index a2fc52f47c0..9b1171a3e90 100644 --- a/helm/litellm/templates/serviceaccount.yaml +++ b/helm/litellm/templates/serviceaccount.yaml @@ -1,5 +1,5 @@ {{- $prev := false -}} -{{- if .Values.serviceAccounts.gateway.create -}} +{{- if and .Values.serviceAccounts.gateway.create (include "litellm.workload.render" .) -}} {{- $prev = true }} apiVersion: v1 kind: ServiceAccount @@ -14,7 +14,7 @@ metadata: {{- end }} automountServiceAccountToken: {{ .Values.serviceAccounts.gateway.automount }} {{- end }} -{{- if .Values.serviceAccounts.backend.create }} +{{- if and .Values.serviceAccounts.backend.create (include "litellm.backend.render" .) }} {{- if $prev }} --- {{- end }} @@ -32,7 +32,7 @@ metadata: {{- end }} automountServiceAccountToken: {{ .Values.serviceAccounts.backend.automount }} {{- end }} -{{- if .Values.serviceAccounts.ui.create }} +{{- if and .Values.serviceAccounts.ui.create (include "litellm.ui.render" .) }} {{- if $prev }} --- {{- end }} diff --git a/helm/litellm/templates/tests/test-connection.yaml b/helm/litellm/templates/tests/test-connection.yaml new file mode 100644 index 00000000000..bc258c92790 --- /dev/null +++ b/helm/litellm/templates/tests/test-connection.yaml @@ -0,0 +1,53 @@ +{{- if include "litellm.workload.render" . }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "litellm.fullname" . }}-test-connection + labels: + {{- include "litellm.commonLabels" . | nindent 4 }} + app.kubernetes.io/component: test + annotations: + helm.sh/hook: test + helm.sh/hook-delete-policy: before-hook-creation +spec: + restartPolicy: Never + {{- with .Values.gateway.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + containers: + - name: health + image: {{ include "litellm.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.gateway.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + env: + - name: LITELLM_READINESS_URL + value: http://{{ include "litellm.workload.fullname" . }}:{{ .Values.gateway.service.port }}/health/readiness + args: + - python + - -c + - | + import json, os, sys, time, urllib.error, urllib.request + url = os.environ["LITELLM_READINESS_URL"] + for attempt in range(1, 31): + try: + with urllib.request.urlopen(url, timeout=5) as response: + body = response.read().decode() + print(url, response.status, body) + db = json.loads(body).get("db") + if db != "connected": + print(f"readiness reports db={db!r}: the proxy is not using the configured database") + sys.exit(1) + sys.exit(0) + except (urllib.error.URLError, OSError) as error: + print(f"attempt {attempt}: {url}: {error}") + time.sleep(2) + sys.exit(1) +{{- end }} diff --git a/helm/litellm/templates/ui/deployment.yaml b/helm/litellm/templates/ui/deployment.yaml index 394eea17451..486e1dae5c8 100644 --- a/helm/litellm/templates/ui/deployment.yaml +++ b/helm/litellm/templates/ui/deployment.yaml @@ -1,6 +1,6 @@ +{{- if include "litellm.ui.render" . }} {{- $componentHosts := .Values.ingress.componentHosts | default dict -}} {{- $hasComponentHosts := or $componentHosts.ui $componentHosts.backend $componentHosts.gateway -}} -{{- if .Values.ui.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -8,10 +8,21 @@ metadata: labels: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: ui + {{- with .Values.ui.deploymentLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ui.deploymentAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: {{- if and (not .Values.ui.hpa.enabled) (not (kindIs "invalid" .Values.ui.replicaCount)) }} replicas: {{ .Values.ui.replicaCount }} {{- end }} + {{- $minReady := .Values.ui.minReadySeconds }} + {{- if not (or (kindIs "invalid" $minReady) (eq (printf "%v" $minReady) "")) }} + minReadySeconds: {{ $minReady }} + {{- end }} {{- with .Values.ui.strategy }} strategy: {{- toYaml . | nindent 4 }} @@ -41,14 +52,20 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.ui.extraInitContainers }} + initContainers: + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} containers: - name: ui - image: "{{ .Values.ui.image.repository }}:{{ .Values.ui.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.ui.image.pullPolicy }} + image: {{ include "litellm.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.ui.securityContext }} securityContext: {{- toYaml . | nindent 12 }} {{- end }} + args: + - ui ports: - name: http containerPort: 3000 diff --git a/helm/litellm/templates/ui/hpa.yaml b/helm/litellm/templates/ui/hpa.yaml index a9b0b51129e..67e17b269c7 100644 --- a/helm/litellm/templates/ui/hpa.yaml +++ b/helm/litellm/templates/ui/hpa.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.ui.enabled .Values.ui.hpa.enabled }} +{{- if and (include "litellm.ui.render" .) .Values.ui.hpa.enabled }} apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: diff --git a/helm/litellm/templates/ui/poddisruptionbudget.yaml b/helm/litellm/templates/ui/poddisruptionbudget.yaml index f7a3a694e9c..fa4bb44612c 100644 --- a/helm/litellm/templates/ui/poddisruptionbudget.yaml +++ b/helm/litellm/templates/ui/poddisruptionbudget.yaml @@ -1,5 +1,6 @@ {{- include "litellm.pdb" (dict "root" $ + "enabled" (include "litellm.ui.render" .) "component" .Values.ui "componentName" "ui" "fullname" (include "litellm.ui.fullname" .) diff --git a/helm/litellm/templates/ui/service.yaml b/helm/litellm/templates/ui/service.yaml index 52b539fa00c..d544d9d6148 100644 --- a/helm/litellm/templates/ui/service.yaml +++ b/helm/litellm/templates/ui/service.yaml @@ -1,4 +1,4 @@ -{{- if .Values.ui.enabled }} +{{- if include "litellm.ui.render" . }} apiVersion: v1 kind: Service metadata: @@ -6,8 +6,15 @@ metadata: labels: {{- include "litellm.commonLabels" . | nindent 4 }} app.kubernetes.io/component: ui + {{- with .Values.ui.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: type: {{ .Values.ui.service.type }} + {{- with include "litellm.service.extras" .Values.ui.service }} + {{- . | nindent 2 }} + {{- end }} ports: - port: {{ .Values.ui.service.port }} targetPort: http diff --git a/helm/litellm/tests/billing_metrics_tests.yaml b/helm/litellm/tests/billing_metrics_tests.yaml index ceba0bd1430..d17b8192fce 100644 --- a/helm/litellm/tests/billing_metrics_tests.yaml +++ b/helm/litellm/tests/billing_metrics_tests.yaml @@ -1,39 +1,30 @@ -suite: test billingMetrics wiring on gateway and backend +suite: test bearer-only billingMetrics wiring templates: - gateway/deployment.yaml - gateway/configmap.yaml - backend/deployment.yaml + - backend/configmap.yaml - migrations-job.yaml values: - ./values/required.yaml tests: - - it: is off by default, adding no env, volume, or mount + - it: is disabled by default template: gateway/deployment.yaml asserts: - - notContains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: billing-mtls - - equal: - path: spec.template.spec.containers[0].volumeMounts - value: - - name: gateway-config - mountPath: /app/config/config.yaml - subPath: config.yaml - notContains: path: spec.template.spec.containers[0].env content: name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai + - notContains: + path: spec.template.spec.volumes + content: + name: billing-metrics - - it: renders the endpoint and the mounted cert paths when enabled + - it: wires gateway URLs and bearer credentials from Secret keys template: gateway/deployment.yaml set: billingMetrics: enabled: true - secretName: billing-mtls asserts: - contains: path: spec.template.spec.containers[0].env @@ -43,87 +34,68 @@ tests: - contains: path: spec.template.spec.containers[0].env content: - name: LITELLM_BILLING_METRICS_CLIENT_CERT - value: /etc/litellm/billing-mtls/tls.crt + name: LITELLM_BILLING_METRICS_TOKEN_URL + value: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token - contains: path: spec.template.spec.containers[0].env content: - name: LITELLM_BILLING_METRICS_CLIENT_KEY - value: /etc/litellm/billing-mtls/tls.key - - - it: mounts the cert secret read-only alongside the config volume - template: gateway/deployment.yaml - set: - billingMetrics: - enabled: true - secretName: billing-mtls - asserts: + name: LITELLM_BILLING_METRICS_CLIENT_ID + valueFrom: + secretKeyRef: + name: litellm-billing-metrics + key: client-id - contains: - path: spec.template.spec.volumes + path: spec.template.spec.containers[0].env content: - name: billing-metrics-mtls - secret: - secretName: billing-mtls - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls - mountPath: /etc/litellm/billing-mtls - readOnly: true - - # The production collector presents a public web-PKI certificate, so the CA - # override must stay absent unless a private collector is configured. - - it: omits the CA env, volume, and mount when no caSecretName is set - template: gateway/deployment.yaml - set: - billingMetrics: - enabled: true - secretName: billing-mtls - asserts: + name: LITELLM_BILLING_METRICS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: litellm-billing-metrics + key: client-secret - notContains: path: spec.template.spec.volumes content: - name: billing-metrics-mtls-ca - secret: - secretName: billing-ca + name: billing-metrics - notContains: - path: spec.template.spec.containers[0].env + path: spec.template.spec.containers[0].volumeMounts content: - name: LITELLM_BILLING_METRICS_CA_CERT - value: /etc/litellm/billing-mtls-ca/ca.crt + name: billing-metrics - - it: mounts the CA secret when caSecretName is set - template: gateway/deployment.yaml + - it: wires billing metrics on the backend without volumes + template: backend/deployment.yaml set: billingMetrics: enabled: true - secretName: billing-mtls - caSecretName: billing-ca asserts: - contains: path: spec.template.spec.containers[0].env content: - name: LITELLM_BILLING_METRICS_CA_CERT - value: /etc/litellm/billing-mtls-ca/ca.crt - - contains: + name: LITELLM_BILLING_METRICS_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: litellm-billing-metrics + key: client-secret + - notContains: path: spec.template.spec.volumes content: - name: billing-metrics-mtls-ca - secret: - secretName: billing-ca - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls-ca - mountPath: /etc/litellm/billing-mtls-ca - readOnly: true + name: billing-metrics - - it: passes the export interval through only when set + - it: does not add billing metrics to migrations + template: migrations-job.yaml + set: + billingMetrics: + enabled: true + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_BILLING_METRICS_ENDPOINT + + - it: renders an optional export interval template: gateway/deployment.yaml set: billingMetrics: enabled: true - secretName: billing-mtls exportIntervalMs: 5000 asserts: - contains: @@ -132,102 +104,27 @@ tests: name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS value: "5000" - - it: keeps user-supplied gateway volumes alongside the billing secret + - it: requires the endpoint when enabled template: gateway/deployment.yaml set: billingMetrics: enabled: true - secretName: billing-mtls - gateway.volumes: - - name: custom-callbacks - configMap: - name: my-callbacks - gateway.volumeMounts: - - name: custom-callbacks - mountPath: /app/callbacks + endpoint: "" asserts: - - contains: - path: spec.template.spec.volumes - content: - name: custom-callbacks - configMap: - name: my-callbacks - - contains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: billing-mtls + - failedTemplate: + errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true - # The backend keeps the named-server MCP transport (/{mcp_server_name}/mcp), - # which writes a SpendLogs row, so it must meter too or that traffic is lost. - - it: meters the backend as well, since it serves the MCP transport - template: backend/deployment.yaml - set: - billingMetrics: - enabled: true - secretName: billing-mtls - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: billing-metrics-mtls - mountPath: /etc/litellm/billing-mtls - readOnly: true - - contains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: billing-mtls - - - it: leaves the backend alone when metering is off - template: backend/deployment.yaml - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai - - # The migrations job runs prisma and serves no traffic; it must never receive - # the client key. - - it: never mounts the billing cert on the migrations job - template: migrations-job.yaml - set: - billingMetrics: - enabled: true - secretName: billing-mtls - asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: LITELLM_BILLING_METRICS_ENDPOINT - value: https://telemetry.litellm.ai - - isNull: - path: spec.template.spec.volumes - - # The conventional Secret name is the default, so enabling metering needs no - # secretName at all; the guard below only fires on an explicitly blanked one. - - it: uses the conventional secret name by default + - it: requires the token URL when enabled template: gateway/deployment.yaml set: billingMetrics: enabled: true + tokenUrl: "" asserts: - - contains: - path: spec.template.spec.volumes - content: - name: billing-metrics-mtls - secret: - secretName: litellm-billing-metrics-mtls + - failedTemplate: + errorMessage: billingMetrics.tokenUrl is required when billingMetrics.enabled is true - - it: fails loudly when the secretName is explicitly blanked + - it: requires the secret name when enabled template: gateway/deployment.yaml set: billingMetrics: @@ -235,15 +132,4 @@ tests: secretName: "" asserts: - failedTemplate: - errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key) - - - it: fails loudly when enabled without an endpoint - template: gateway/deployment.yaml - set: - billingMetrics: - enabled: true - endpoint: "" - secretName: billing-mtls - asserts: - - failedTemplate: - errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true + errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true diff --git a/helm/litellm/tests/collector_tests.yaml b/helm/litellm/tests/collector_tests.yaml index 4ef7e3c8ca4..8e9804dece8 100644 --- a/helm/litellm/tests/collector_tests.yaml +++ b/helm/litellm/tests/collector_tests.yaml @@ -34,7 +34,7 @@ tests: gateway.collector.enabled: true gateway.collector.bufferSize: 250 gateway.collector.onUnavailable: drop - gateway.image.tag: v1.102.0 + image.tag: v1.102.0 gateway.numWorkers: 4 database.connectionPool.enabled: true database.connectionPool.maxDbConnections: 8 @@ -80,14 +80,12 @@ tests: template: gateway/deployment.yaml - equal: path: spec.template.spec.containers[1].image - value: ghcr.io/berriai/litellm-gateway:v1.102.0 + value: ghcr.io/berriai/litellm:v1.102.0 template: gateway/deployment.yaml - equal: - path: spec.template.spec.containers[1].command + path: spec.template.spec.containers[1].args value: - - python - - -m - - litellm.proxy.collector + - collector template: gateway/deployment.yaml - contains: path: spec.template.spec.containers[1].env diff --git a/helm/litellm/tests/component_hosts_tests.yaml b/helm/litellm/tests/component_hosts_tests.yaml index b55b3b694a0..047f1e7006d 100644 --- a/helm/litellm/tests/component_hosts_tests.yaml +++ b/helm/litellm/tests/component_hosts_tests.yaml @@ -316,16 +316,16 @@ tests: path: spec.template.spec.containers[0].env[1].value value: "true" - equal: - path: spec.template.spec.containers[0].env[10].name + path: spec.template.spec.containers[0].env[12].name value: LITELLM_CORS_ORIGINS - equal: - path: spec.template.spec.containers[0].env[10].value + path: spec.template.spec.containers[0].env[12].value value: https://backend-override.example.com - equal: - path: spec.template.spec.containers[0].env[11].name + path: spec.template.spec.containers[0].env[13].name value: LITELLM_CORS_ALLOW_CREDENTIALS - equal: - path: spec.template.spec.containers[0].env[11].value + path: spec.template.spec.containers[0].env[13].value value: "false" - it: exposes the public UI origin to the gateway without enabling credentials @@ -389,10 +389,10 @@ tests: path: spec.template.spec.containers[0].env[0].value value: https://ui.example.com - equal: - path: spec.template.spec.containers[0].env[9].name + path: spec.template.spec.containers[0].env[11].name value: LITELLM_CORS_ORIGINS - equal: - path: spec.template.spec.containers[0].env[9].value + path: spec.template.spec.containers[0].env[11].value value: https://gateway-override.example.com --- diff --git a/helm/litellm/tests/extra_resources_tests.yaml b/helm/litellm/tests/extra_resources_tests.yaml new file mode 100644 index 00000000000..4935b3a5401 --- /dev/null +++ b/helm/litellm/tests/extra_resources_tests.yaml @@ -0,0 +1,45 @@ +suite: test extraResources +templates: + - extra-resources.yaml +values: + - ./values/required.yaml +tests: + - it: renders nothing by default + asserts: + - hasDocuments: + count: 0 + + - it: renders every entry as its own document and templates it against the release + set: + extraResources: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: "{{ .Release.Name }}-extra" + data: + chart: "{{ .Chart.Name }}" + - apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: deny-all + spec: + podSelector: {} + policyTypes: + - Ingress + asserts: + - hasDocuments: + count: 2 + - isKind: + of: ConfigMap + documentIndex: 0 + - equal: + path: metadata.name + value: RELEASE-NAME-extra + documentIndex: 0 + - equal: + path: data.chart + value: litellm + documentIndex: 0 + - isKind: + of: NetworkPolicy + documentIndex: 1 diff --git a/helm/litellm/tests/hpa_workload_metrics_tests.yaml b/helm/litellm/tests/hpa_workload_metrics_tests.yaml index a29c53c5ed9..9b790070819 100644 --- a/helm/litellm/tests/hpa_workload_metrics_tests.yaml +++ b/helm/litellm/tests/hpa_workload_metrics_tests.yaml @@ -201,6 +201,76 @@ tests: scrapeTimeout: 10s scheme: http + - it: scrapes over https with the given tlsConfig, for a mesh with STRICT mTLS + template: gateway/servicemonitor.yaml + set: + gateway.serviceMonitor.enabled: true + gateway.metricsServer.enabled: true + gateway.serviceMonitor.scheme: https + gateway.serviceMonitor.tlsConfig: + caFile: /etc/prom-certs/root-cert.pem + certFile: /etc/prom-certs/cert-chain.pem + keyFile: /etc/prom-certs/key.pem + insecureSkipVerify: true + asserts: + - equal: + path: spec.endpoints + value: + - port: metrics + path: /metrics/ + interval: 15s + scrapeTimeout: 10s + scheme: https + tlsConfig: + caFile: /etc/prom-certs/root-cert.pem + certFile: /etc/prom-certs/cert-chain.pem + keyFile: /etc/prom-certs/key.pem + insecureSkipVerify: true + + - it: carries litellm-helm annotations, relabelings and namespaceSelector onto the ServiceMonitor + template: gateway/servicemonitor.yaml + release: + namespace: llm + set: + gateway.serviceMonitor.enabled: true + gateway.metricsServer.enabled: true + gateway.serviceMonitor.annotations: + team: platform + gateway.serviceMonitor.relabelings: + - targetLabel: node + sourceLabels: [__meta_kubernetes_pod_node_name] + action: replace + gateway.serviceMonitor.namespaceSelector.matchNames: + - llm + - shared + asserts: + - equal: + path: metadata.annotations + value: + team: platform + - equal: + path: spec.endpoints[0].relabelings + value: + - targetLabel: node + sourceLabels: [__meta_kubernetes_pod_node_name] + action: replace + - equal: + path: spec.namespaceSelector.matchNames + value: + - llm + - shared + + - it: renders no ServiceMonitor annotations or relabelings by default + template: gateway/servicemonitor.yaml + set: + gateway.serviceMonitor.enabled: true + gateway.metricsServer.enabled: true + asserts: + - notExists: + path: metadata.annotations + - notExists: + path: spec.endpoints[0].relabelings + - it: honours a custom scrape interval template: gateway/servicemonitor.yaml set: diff --git a/helm/litellm/tests/ingress_monolith_tests.yaml b/helm/litellm/tests/ingress_monolith_tests.yaml new file mode 100644 index 00000000000..08f7c598260 --- /dev/null +++ b/helm/litellm/tests/ingress_monolith_tests.yaml @@ -0,0 +1,164 @@ +suite: test ingress routing in monolith mode +templates: + - ingress.yaml +values: + - ./values/required.yaml +tests: + - it: monolith mode routes all component hosts to the proxy Service + set: + monolith.enabled: true + ingress.enabled: true + gateway.enabled: false + backend.enabled: false + ui.enabled: false + ingress.componentHosts.ui: ui.example.com + ingress.componentHosts.backend: backend.example.com + ingress.componentHosts.gateway: gateway.example.com + gateway.service.port: 8080 + asserts: + - lengthEqual: + path: spec.rules + count: 3 + - equal: + path: spec.rules[0].http.paths[0].backend.service + value: + name: RELEASE-NAME-litellm + port: + number: 8080 + - equal: + path: spec.rules[1].http.paths[0].backend.service + value: + name: RELEASE-NAME-litellm + port: + number: 8080 + - equal: + path: spec.rules[2].http.paths[0].backend.service + value: + name: RELEASE-NAME-litellm + port: + number: 8080 + + - it: componentized mode still splits traffic across ui, gateway and backend Services + set: + ingress.enabled: true + asserts: + - equal: + path: spec.rules[0].http.paths[0].backend.service.name + value: RELEASE-NAME-litellm-ui + - contains: + path: spec.rules[0].http.paths + content: + path: /v1/chat + pathType: Prefix + backend: + service: + name: RELEASE-NAME-litellm-gateway + port: + number: 4000 + - equal: + path: spec.rules[0].http.paths[-1].backend.service.name + value: RELEASE-NAME-litellm-backend + + - it: monolith mode routes a single catch-all to the monolith Service and nothing else + set: + monolith.enabled: true + ingress.enabled: true + asserts: + - lengthEqual: + path: spec.rules[0].http.paths + count: 1 + - equal: + path: spec.rules[0].http.paths[0] + value: + path: / + pathType: Prefix + backend: + service: + name: RELEASE-NAME-litellm + port: + number: 4000 + - notMatchRegexRaw: + pattern: RELEASE-NAME-litellm-(gateway|backend|ui) + + - it: monolith mode uses the gateway service port for the catch-all + set: + monolith.enabled: true + ingress.enabled: true + gateway.service.port: 8080 + asserts: + - equal: + path: spec.rules[0].http.paths[0].backend.service.port.number + value: 8080 + + - it: monolith mode sends every extraPaths entry to the monolith Service regardless of its service field + set: + monolith.enabled: true + ingress.enabled: true + ingress.extraPaths: + - path: /watsonx + - path: /admin + service: backend + pathType: Exact + - path: /dashboard + service: ui + asserts: + - lengthEqual: + path: spec.rules[0].http.paths + count: 4 + - equal: + path: spec.rules[0].http.paths[0] + value: + path: /watsonx + pathType: Prefix + backend: + service: + name: RELEASE-NAME-litellm + port: + number: 4000 + - equal: + path: spec.rules[0].http.paths[1].backend.service.name + value: RELEASE-NAME-litellm + - equal: + path: spec.rules[0].http.paths[1].pathType + value: Exact + - equal: + path: spec.rules[0].http.paths[2].backend.service.name + value: RELEASE-NAME-litellm + - equal: + path: spec.rules[0].http.paths[3].path + value: / + + - it: monolith mode keeps rejecting an extra path at the root + set: + monolith.enabled: true + ingress.enabled: true + ingress.extraPaths: + - path: / + asserts: + - failedTemplate: + errorPattern: "ingress.extraPaths\\[0\\]: path / is already routed" + + - it: monolith mode keeps the host, TLS and class settings + set: + monolith.enabled: true + ingress.enabled: true + ingress.className: nginx + ingress.controller: nginx + ingress.host: llm.example.com + ingress.tls: + - hosts: + - llm.example.com + secretName: llm-tls + asserts: + - equal: + path: spec.ingressClassName + value: nginx + - equal: + path: spec.rules[0].host + value: llm.example.com + - equal: + path: spec.tls[0].secretName + value: llm-tls + - equal: + path: spec.rules[0].http.paths[0].backend.service.name + value: RELEASE-NAME-litellm diff --git a/helm/litellm/tests/keda_tests.yaml b/helm/litellm/tests/keda_tests.yaml new file mode 100644 index 00000000000..1f48654fa73 --- /dev/null +++ b/helm/litellm/tests/keda_tests.yaml @@ -0,0 +1,266 @@ +suite: test KEDA ScaledObject +templates: + - gateway/keda.yaml + - gateway/hpa.yaml + - gateway/deployment.yaml + - gateway/configmap.yaml +values: + - ./values/required.yaml +release: + name: rel + namespace: llm +tests: + - it: renders no ScaledObject by default + template: gateway/keda.yaml + asserts: + - hasDocuments: + count: 0 + + - it: passes user triggers through and adds no prometheus triggers by default + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.triggers: + - type: cpu + metricType: Utilization + metadata: + value: "60" + templates: + - gateway/keda.yaml + - gateway/hpa.yaml + asserts: + - hasDocuments: + count: 0 + template: gateway/hpa.yaml + - isKind: + of: ScaledObject + template: gateway/keda.yaml + - equal: + path: metadata.name + value: rel-litellm-gateway + template: gateway/keda.yaml + - equal: + path: spec.scaleTargetRef + value: + apiVersion: apps/v1 + kind: Deployment + name: rel-litellm-gateway + template: gateway/keda.yaml + - equal: + path: spec.minReplicaCount + value: 1 + template: gateway/keda.yaml + - equal: + path: spec.maxReplicaCount + value: 10 + template: gateway/keda.yaml + - equal: + path: spec.triggers + value: + - type: cpu + metricType: Utilization + metadata: + value: "60" + template: gateway/keda.yaml + + - it: leaves spec.replicas off the Deployment so KEDA owns the replica count + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.replicaCount: 3 + template: gateway/deployment.yaml + asserts: + - notExists: + path: spec.replicas + + - it: scales on requests per second against the metrics Service job + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.prometheus.serverAddress: http://prometheus-operated.monitoring.svc:9090 + gateway.metricsServer.enabled: true + gateway.keda.prometheus.targetRequestsPerSecond: 90 + template: gateway/keda.yaml + asserts: + - lengthEqual: + path: spec.triggers + count: 1 + - equal: + path: spec.triggers[0] + value: + type: prometheus + metricType: AverageValue + metadata: + serverAddress: http://prometheus-operated.monitoring.svc:9090 + threshold: "90" + query: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm-gateway-metrics"}[1m])) + + - it: scales on tokens per second on its own + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.prometheus.serverAddress: http://prom:9090 + gateway.metricsServer.enabled: true + gateway.keda.prometheus.targetTokensPerSecond: 6000000 + template: gateway/keda.yaml + asserts: + - lengthEqual: + path: spec.triggers + count: 1 + - equal: + path: spec.triggers[0].type + value: prometheus + - equal: + path: spec.triggers[0].metadata.threshold + value: "6000000" + - equal: + path: spec.triggers[0].metadata.query + value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm-gateway-metrics"}[1m])) + + - it: appends the requests and tokens triggers after user triggers + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.metricsServer.enabled: true + gateway.keda.triggers: + - type: cpu + metricType: Utilization + metadata: + value: "60" + gateway.keda.prometheus.serverAddress: http://prom:9090 + gateway.keda.prometheus.targetRequestsPerSecond: 90 + gateway.keda.prometheus.targetTokensPerSecond: 6000000 + template: gateway/keda.yaml + asserts: + - lengthEqual: + path: spec.triggers + count: 3 + - equal: + path: spec.triggers[0].type + value: cpu + - equal: + path: spec.triggers[1].metadata.threshold + value: "90" + - equal: + path: spec.triggers[2].metadata.threshold + value: "6000000" + - notMatchRegexRaw: + pattern: "\\* *60|per_minute|PerMinute" + + - it: fills a partial fallback with the litellm-helm defaults + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.maxReplicaCount: 20 + gateway.keda.fallback: + replicas: 4 + template: gateway/keda.yaml + asserts: + - equal: + path: spec.fallback + value: + failureThreshold: 3 + replicas: 4 + + - it: defaults fallback replicas to maxReplicaCount and keeps an explicit zero threshold + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.maxReplicaCount: 20 + gateway.keda.fallback: + failureThreshold: 0 + template: gateway/keda.yaml + asserts: + - equal: + path: spec.fallback + value: + failureThreshold: 0 + replicas: 20 + + - it: renders advanced, fallback, polling and cooldown settings + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.minReplicaCount: 2 + gateway.keda.maxReplicaCount: 20 + gateway.keda.pollingInterval: 15 + gateway.keda.cooldownPeriod: 120 + gateway.keda.fallback: + failureThreshold: 3 + replicas: 4 + gateway.keda.advanced: + restoreToOriginalReplicaCount: true + template: gateway/keda.yaml + asserts: + - equal: + path: spec.minReplicaCount + value: 2 + - equal: + path: spec.maxReplicaCount + value: 20 + - equal: + path: spec.pollingInterval + value: 15 + - equal: + path: spec.cooldownPeriod + value: 120 + - equal: + path: spec.fallback + value: + failureThreshold: 3 + replicas: 4 + - equal: + path: spec.advanced.restoreToOriginalReplicaCount + value: true + + - it: refuses a workload target without a prometheus server address + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.prometheus.targetRequestsPerSecond: 90 + template: gateway/keda.yaml + asserts: + - failedTemplate: + errorMessage: gateway.keda.prometheus.serverAddress is required when gateway.keda.prometheus.targetRequestsPerSecond or targetTokensPerSecond is set + + - it: refuses a workload target without the metrics sidecar the trigger queries + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.prometheus.serverAddress: http://prom:9090 + gateway.keda.prometheus.targetRequestsPerSecond: 90 + template: gateway/keda.yaml + asserts: + - failedTemplate: + errorMessage: gateway.metricsServer.enabled is required when gateway.keda.prometheus.targetRequestsPerSecond or targetTokensPerSecond is set + + - it: refuses to run the HPA and KEDA against the same Deployment + set: + gateway.hpa.enabled: true + gateway.keda.enabled: true + template: gateway/deployment.yaml + asserts: + - failedTemplate: + errorPattern: gateway.hpa.enabled and gateway.keda.enabled are mutually exclusive + + - it: renders scaledObject annotations, such as the KEDA pause annotation, on the ScaledObject + template: gateway/keda.yaml + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.scaledObject.annotations: + autoscaling.keda.sh/paused-replicas: "2" + asserts: + - equal: + path: metadata.annotations + value: + autoscaling.keda.sh/paused-replicas: "2" + + - it: renders no ScaledObject annotations by default + template: gateway/keda.yaml + set: + gateway.hpa.enabled: false + gateway.keda.enabled: true + asserts: + - notExists: + path: metadata.annotations diff --git a/helm/litellm/tests/liteadmin_tests.yaml b/helm/litellm/tests/liteadmin_tests.yaml new file mode 100644 index 00000000000..429896ba9b4 --- /dev/null +++ b/helm/litellm/tests/liteadmin_tests.yaml @@ -0,0 +1,149 @@ +suite: test the LiteAdmin worker and its proxy wiring +templates: + - liteadmin.yaml + - gateway/configmap.yaml + - gateway/deployment.yaml + - backend/deployment.yaml + - monolith/deployment.yaml +values: + - ./values/required.yaml +tests: + - it: renders nothing when liteadmin is disabled + template: liteadmin.yaml + asserts: + - hasDocuments: + count: 0 + + - it: enabled renders a Deployment, Service and PVC on the shared image + set: + liteadmin.enabled: true + liteadmin.existingSecret: liteadmin-secret + liteadmin.gatewayUrl: http://gateway.example.com + liteadmin.model: gpt-4o + template: liteadmin.yaml + asserts: + - hasDocuments: + count: 3 + - isKind: + of: Deployment + documentIndex: 0 + - isKind: + of: Service + documentIndex: 1 + - isKind: + of: PersistentVolumeClaim + documentIndex: 2 + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0 + documentIndex: 0 + - equal: + path: spec.template.spec.containers[0].args + value: ["--admin-agent"] + documentIndex: 0 + + - it: refuses to render without liteadmin.existingSecret + set: + liteadmin.enabled: true + liteadmin.gatewayUrl: http://gateway.example.com + liteadmin.model: gpt-4o + templates: + - backend/deployment.yaml + template: backend/deployment.yaml + asserts: + - failedTemplate: + errorPattern: liteadmin.existingSecret is required + + - it: wires the liteadmin env onto the backend pod in componentized mode + set: + liteadmin.enabled: true + liteadmin.existingSecret: liteadmin-secret + liteadmin.gatewayUrl: http://gateway.example.com + liteadmin.model: gpt-4o + template: backend/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_ADMIN_AGENT_URL + value: http://RELEASE-NAME-litellm-liteadmin:10000 + - contains: + path: spec.template.spec.containers[0].env + content: + name: ADMIN_AGENT_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: liteadmin-secret + key: ADMIN_AGENT_SERVICE_TOKEN + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROXY_BASE_URL + value: http://gateway.example.com + + - it: leaves the liteadmin env off the gateway pod in componentized mode + set: + liteadmin.enabled: true + liteadmin.existingSecret: liteadmin-secret + liteadmin.gatewayUrl: http://gateway.example.com + liteadmin.model: gpt-4o + template: gateway/deployment.yaml + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_ADMIN_AGENT_URL + - notContains: + path: spec.template.spec.containers[0].env + content: + name: ADMIN_AGENT_SERVICE_TOKEN + + - it: wires the liteadmin env onto the monolith pod in monolith mode + set: + monolith.enabled: true + liteadmin.enabled: true + liteadmin.existingSecret: liteadmin-secret + liteadmin.gatewayUrl: http://gateway.example.com + liteadmin.model: gpt-4o + template: monolith/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_ADMIN_AGENT_URL + value: http://RELEASE-NAME-litellm-liteadmin:10000 + - contains: + path: spec.template.spec.containers[0].env + content: + name: ADMIN_AGENT_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: liteadmin-secret + key: ADMIN_AGENT_SERVICE_TOKEN + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROXY_BASE_URL + value: http://gateway.example.com + + - it: a user-supplied PROXY_BASE_URL in extraEnv wins over liteadmin.gatewayUrl + set: + liteadmin.enabled: true + liteadmin.existingSecret: liteadmin-secret + liteadmin.gatewayUrl: http://gateway.example.com + liteadmin.model: gpt-4o + backend.extraEnv: + - name: PROXY_BASE_URL + value: https://custom.example.com + template: backend/deployment.yaml + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: PROXY_BASE_URL + value: http://gateway.example.com + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROXY_BASE_URL + value: https://custom.example.com diff --git a/helm/litellm/tests/masterkey_secret_tests.yaml b/helm/litellm/tests/masterkey_secret_tests.yaml new file mode 100644 index 00000000000..8dd261053e4 --- /dev/null +++ b/helm/litellm/tests/masterkey_secret_tests.yaml @@ -0,0 +1,156 @@ +suite: test the generated master key Secret +templates: + - secret-masterkey.yaml + - gateway/deployment.yaml + - backend/deployment.yaml + - gateway/configmap.yaml + - monolith/deployment.yaml + - migrations-job.yaml +values: + - ./values/required.yaml +tests: + - it: renders no Secret when the operator supplies masterKey.secretName + template: secret-masterkey.yaml + asserts: + - hasDocuments: + count: 0 + + - it: refuses to render the workload without a Secret name when masterKey.generate is off + set: + masterKey.secretName: "" + template: gateway/deployment.yaml + asserts: + - failedTemplate: + errorPattern: masterKey.secretName is required + + - it: generates a Secret whose key starts with sk- (base64 c2st) and keeps it across uninstall + set: + masterKey.secretName: "" + masterKey.generate: true + template: secret-masterkey.yaml + asserts: + - isKind: + of: Secret + - equal: + path: metadata.name + value: RELEASE-NAME-litellm-masterkey + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + - matchRegex: + path: data["master-key"] + pattern: ^c2st + + - it: stores the value under masterKey.secretKey + set: + masterKey.secretName: "" + masterKey.generate: true + masterKey.secretKey: LITELLM_MASTER_KEY + template: secret-masterkey.yaml + asserts: + - matchRegex: + path: data.LITELLM_MASTER_KEY + pattern: ^c2st + + - it: reuses the master key already stored in the cluster instead of generating a new one on upgrade + set: + masterKey.secretName: "" + masterKey.generate: true + template: secret-masterkey.yaml + kubernetesProvider: + scheme: + "v1/Secret": + gvr: + version: "v1" + resource: "secrets" + namespaced: true + objects: + - kind: Secret + apiVersion: v1 + metadata: + name: RELEASE-NAME-litellm-masterkey + namespace: NAMESPACE + data: + master-key: c2stZXhpc3Rpbmcta2V5 + asserts: + - equal: + path: data["master-key"] + value: c2stZXhpc3Rpbmcta2V5 + + - it: points the app containers at the generated Secret and keeps it out of the pre-install migrations Job + set: + masterKey.secretName: "" + masterKey.generate: true + monolith.enabled: true + templates: + - monolith/deployment.yaml + - migrations-job.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_MASTER_KEY + valueFrom: + secretKeyRef: + name: RELEASE-NAME-litellm-masterkey + key: master-key + template: monolith/deployment.yaml + - equal: + path: metadata.annotations["helm.sh/hook"] + value: pre-install,pre-upgrade + template: migrations-job.yaml + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_MASTER_KEY + any: true + template: migrations-job.yaml + + - it: an explicit masterKey.secretName wins over masterKey.generate + set: + masterKey.secretName: operator-secret + masterKey.generate: true + templates: + - secret-masterkey.yaml + - gateway/deployment.yaml + asserts: + - hasDocuments: + count: 0 + template: secret-masterkey.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_MASTER_KEY + valueFrom: + secretKeyRef: + name: operator-secret + key: master-key + template: gateway/deployment.yaml + + - it: exposes the key as PROXY_MASTER_KEY too, so a proxy_config copied from litellm-helm keeps resolving + set: + monolith.enabled: true + templates: + - monolith/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROXY_MASTER_KEY + valueFrom: + secretKeyRef: + name: litellm-master-key-secret + key: master-key + - it: exposes PROXY_MASTER_KEY on the componentized gateway and backend + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: PROXY_MASTER_KEY + valueFrom: + secretKeyRef: + name: litellm-master-key-secret + key: master-key diff --git a/helm/litellm/tests/metrics_server_tests.yaml b/helm/litellm/tests/metrics_server_tests.yaml index 0e7d9d9e9ee..1a48b7be671 100644 --- a/helm/litellm/tests/metrics_server_tests.yaml +++ b/helm/litellm/tests/metrics_server_tests.yaml @@ -38,7 +38,7 @@ tests: gateway.metricsServer.enabled: true gateway.metricsServer.port: 4101 gateway.service.type: LoadBalancer - gateway.image.tag: v1.101.0 + image.tag: v1.101.0 asserts: - contains: path: spec.template.spec.containers[0].env @@ -58,14 +58,12 @@ tests: template: gateway/deployment.yaml - equal: path: spec.template.spec.containers[1].image - value: ghcr.io/berriai/litellm-gateway:v1.101.0 + value: ghcr.io/berriai/litellm:v1.101.0 template: gateway/deployment.yaml - equal: - path: spec.template.spec.containers[1].command + path: spec.template.spec.containers[1].args value: - - python - - -m - - litellm.proxy.prometheus_metrics_server + - metrics - --port - "4101" template: gateway/deployment.yaml diff --git a/helm/litellm/tests/migration_job_tests.yaml b/helm/litellm/tests/migration_job_tests.yaml index 2ebb1b44926..86251e8a1d2 100644 --- a/helm/litellm/tests/migration_job_tests.yaml +++ b/helm/litellm/tests/migration_job_tests.yaml @@ -91,7 +91,7 @@ tests: app.kubernetes.io/name: litellm app.kubernetes.io/instance: RELEASE-NAME app.kubernetes.io/managed-by: Helm - helm.sh/chart: litellm-0.1.0 + helm.sh/chart: litellm-1.0.0 app.kubernetes.io/component: migrations - it: renders pod-level and container-level securityContext in their own scopes @@ -254,3 +254,76 @@ tests: path: spec.template.spec.nodeSelector - isNull: path: spec.template.spec.tolerations + + - it: sets jobLabels and jobAnnotations on the Job metadata + set: + migrationJob.jobLabels: + environment: production + migrationJob.jobAnnotations: + example.com/cost-center: "1234" + asserts: + - equal: + path: metadata.labels.environment + value: production + - equal: + path: metadata.annotations['example.com/cost-center'] + value: "1234" + - equal: + path: metadata.annotations['helm.sh/hook'] + value: pre-install,pre-upgrade + + - it: renders jobAnnotations when both hooks are off + set: + migrationJob.hooks.helm.enabled: false + migrationJob.jobAnnotations: + example.com/cost-center: "1234" + asserts: + - equal: + path: metadata.annotations + value: + example.com/cost-center: "1234" + + - it: runs the image entrypoint with the migrations component by default + asserts: + - notExists: + path: spec.template.spec.containers[0].command + - equal: + path: spec.template.spec.containers[0].args + value: [migrations] + + - it: overrides the container command and args + set: + migrationJob.command: [sh] + migrationJob.args: [-c, echo migrating] + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: [sh] + - equal: + path: spec.template.spec.containers[0].args + value: [-c, echo migrating] + + - it: clears the container args when only command is set + set: + migrationJob.command: [sh] + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: [sh] + - equal: + path: spec.template.spec.containers[0].args + value: [] + + - it: puts migrationJob.annotations on the Job pod, as litellm-helm did + set: + migrationJob.annotations: + sidecar.istio.io/inject: "false" + asserts: + - equal: + path: spec.template.metadata.annotations + value: + sidecar.istio.io/inject: "false" + - it: renders no pod annotations by default + asserts: + - notExists: + path: spec.template.metadata.annotations diff --git a/helm/litellm/tests/monolith_tests.yaml b/helm/litellm/tests/monolith_tests.yaml new file mode 100644 index 00000000000..cb8aca608ec --- /dev/null +++ b/helm/litellm/tests/monolith_tests.yaml @@ -0,0 +1,539 @@ +suite: test monolith mode +templates: + - monolith/deployment.yaml + - monolith/service.yaml + - monolith/hpa.yaml + - monolith/keda.yaml + - monolith/poddisruptionbudget.yaml + - monolith/service-metrics.yaml + - monolith/servicemonitor.yaml + - gateway/deployment.yaml + - gateway/service.yaml + - gateway/hpa.yaml + - gateway/keda.yaml + - gateway/poddisruptionbudget.yaml + - gateway/service-metrics.yaml + - gateway/servicemonitor.yaml + - gateway/configmap.yaml + - backend/deployment.yaml + - backend/service.yaml + - backend/hpa.yaml + - backend/poddisruptionbudget.yaml + - ui/deployment.yaml + - ui/service.yaml + - ui/hpa.yaml + - ui/poddisruptionbudget.yaml + - serviceaccount.yaml + - migrations-job.yaml +values: + - ./values/required.yaml +tests: + - it: monolith ignores a leftover invalid UI public URL + set: + monolith.enabled: true + ui.publicUrl: not-an-origin + template: monolith/deployment.yaml + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_CORS_ORIGINS + any: true + + - it: renders no monolith resources by default + templates: + - monolith/deployment.yaml + - monolith/service.yaml + - monolith/hpa.yaml + - monolith/poddisruptionbudget.yaml + - monolith/service-metrics.yaml + - monolith/servicemonitor.yaml + asserts: + - hasDocuments: + count: 0 + + - it: componentized mode renders one Deployment per component + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Deployment + + - it: monolith mode renders exactly one Deployment, running the proxy dispatcher + set: + monolith.enabled: true + templates: + - monolith/deployment.yaml + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + asserts: + - hasDocuments: + count: 1 + template: monolith/deployment.yaml + - hasDocuments: + count: 0 + template: gateway/deployment.yaml + - hasDocuments: + count: 0 + template: backend/deployment.yaml + - hasDocuments: + count: 0 + template: ui/deployment.yaml + - isKind: + of: Deployment + template: monolith/deployment.yaml + - equal: + path: metadata.name + value: RELEASE-NAME-litellm + template: monolith/deployment.yaml + - equal: + path: metadata.labels["app.kubernetes.io/component"] + value: proxy + template: monolith/deployment.yaml + - equal: + path: spec.template.spec.containers[0].name + value: proxy + template: monolith/deployment.yaml + - equal: + path: spec.template.spec.containers[0].args[0] + value: proxy + template: monolith/deployment.yaml + - equal: + path: spec.template.spec.containers[0].args + value: + - proxy + - --port + - "4000" + - --config + - /app/config/config.yaml + template: monolith/deployment.yaml + - notExists: + path: spec.template.spec.containers[0].command + template: monolith/deployment.yaml + + - it: monolith appends monolith.extraArgs after the chart's own proxy arguments + set: + monolith.enabled: true + monolith.extraArgs: + - --detailed_debug + - --run_granian + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].args + value: + - proxy + - --port + - "4000" + - --config + - /app/config/config.yaml + - --detailed_debug + - --run_granian + + - it: monolith drops the --config pair when the chart does not create the ConfigMap + set: + monolith.enabled: true + gateway.config.create: false + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].args + value: + - proxy + - --port + - "4000" + - notExists: + path: spec.template.spec.volumes + + - it: monolith mounts the proxy config ConfigMap the chart renders under the monolith name + set: + monolith.enabled: true + gateway.config.proxy_config: + model_list: + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + templates: + - monolith/deployment.yaml + - gateway/configmap.yaml + asserts: + - equal: + path: metadata.name + value: RELEASE-NAME-litellm-config + template: gateway/configmap.yaml + - contains: + path: spec.template.spec.volumes + content: + name: gateway-config + configMap: + name: RELEASE-NAME-litellm-config + template: monolith/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: CONFIG_FILE_PATH + value: /app/config/config.yaml + template: monolith/deployment.yaml + + - it: monolith mode renders exactly one Service and no component Services + set: + monolith.enabled: true + templates: + - monolith/service.yaml + - gateway/service.yaml + - backend/service.yaml + - ui/service.yaml + asserts: + - hasDocuments: + count: 1 + template: monolith/service.yaml + - hasDocuments: + count: 0 + template: gateway/service.yaml + - hasDocuments: + count: 0 + template: backend/service.yaml + - hasDocuments: + count: 0 + template: ui/service.yaml + - equal: + path: metadata.name + value: RELEASE-NAME-litellm + template: monolith/service.yaml + - equal: + path: spec.selector + value: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: proxy + template: monolith/service.yaml + - equal: + path: spec.ports[0].port + value: 4000 + template: monolith/service.yaml + + - it: monolith Service selector matches the monolith pod labels + set: + monolith.enabled: true + templates: + - monolith/deployment.yaml + asserts: + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: RELEASE-NAME + app.kubernetes.io/component: proxy + - equal: + path: spec.template.metadata.labels["app.kubernetes.io/component"] + value: proxy + + - it: monolith reuses gateway resources, replicas, numWorkers and probes + set: + monolith.enabled: true + gateway.replicaCount: 3 + gateway.hpa.enabled: false + gateway.numWorkers: 6 + gateway.resources: + requests: + cpu: "2" + memory: 4Gi + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.replicas + value: 3 + - equal: + path: spec.template.spec.containers[0].resources.requests.cpu + value: "2" + - contains: + path: spec.template.spec.containers[0].env + content: + name: NUM_WORKERS + value: "6" + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /health/readiness + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.path + value: /health/liveliness + + - it: monolith takes the rollout settings a litellm-helm install carried from gateway.minReadySeconds and gateway.strategy + set: + monolith.enabled: true + gateway.minReadySeconds: 60 + gateway.strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + backend.minReadySeconds: 5 + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.minReadySeconds + value: 60 + - equal: + path: spec.strategy + value: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + + - it: monolith ignores backend and ui replica overrides + set: + monolith.enabled: true + gateway.hpa.enabled: false + gateway.replicaCount: 2 + backend.replicaCount: 9 + ui.replicaCount: 9 + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.replicas + value: 2 + + - it: monolith HPA targets the monolith Deployment and the gateway HPA is not rendered + set: + monolith.enabled: true + gateway.hpa.enabled: true + templates: + - monolith/hpa.yaml + - gateway/hpa.yaml + - backend/hpa.yaml + - ui/hpa.yaml + asserts: + - hasDocuments: + count: 1 + template: monolith/hpa.yaml + - hasDocuments: + count: 0 + template: gateway/hpa.yaml + - hasDocuments: + count: 0 + template: backend/hpa.yaml + - hasDocuments: + count: 0 + template: ui/hpa.yaml + - equal: + path: spec.scaleTargetRef + value: + apiVersion: apps/v1 + kind: Deployment + name: RELEASE-NAME-litellm + template: monolith/hpa.yaml + + - it: monolith KEDA ScaledObject targets the monolith Deployment + set: + monolith.enabled: true + gateway.hpa.enabled: false + gateway.keda.enabled: true + gateway.keda.prometheus.serverAddress: http://prometheus:9090 + gateway.metricsServer.enabled: true + gateway.keda.prometheus.targetRequestsPerSecond: 50 + templates: + - monolith/keda.yaml + - gateway/keda.yaml + asserts: + - hasDocuments: + count: 1 + template: monolith/keda.yaml + - hasDocuments: + count: 0 + template: gateway/keda.yaml + - equal: + path: spec.scaleTargetRef.name + value: RELEASE-NAME-litellm + template: monolith/keda.yaml + - equal: + path: spec.triggers[0].metadata.query + value: sum(rate(litellm_proxy_total_requests_metric_total{namespace="NAMESPACE",job="RELEASE-NAME-litellm-metrics"}[1m])) + template: monolith/keda.yaml + + - it: monolith PDB selects the monolith pods + set: + monolith.enabled: true + gateway.pdb.enabled: true + gateway.pdb.maxUnavailable: 1 + templates: + - monolith/poddisruptionbudget.yaml + - gateway/poddisruptionbudget.yaml + - backend/poddisruptionbudget.yaml + - ui/poddisruptionbudget.yaml + asserts: + - hasDocuments: + count: 1 + template: monolith/poddisruptionbudget.yaml + - hasDocuments: + count: 0 + template: gateway/poddisruptionbudget.yaml + - hasDocuments: + count: 0 + template: backend/poddisruptionbudget.yaml + - hasDocuments: + count: 0 + template: ui/poddisruptionbudget.yaml + - equal: + path: spec.maxUnavailable + value: 1 + template: monolith/poddisruptionbudget.yaml + - equal: + path: spec.selector.matchLabels["app.kubernetes.io/component"] + value: proxy + template: monolith/poddisruptionbudget.yaml + + - it: monolith metrics sidecar, metrics Service and ServiceMonitor follow the monolith name + set: + monolith.enabled: true + gateway.metricsServer.enabled: true + gateway.serviceMonitor.enabled: true + templates: + - monolith/deployment.yaml + - monolith/service-metrics.yaml + - monolith/servicemonitor.yaml + - gateway/service-metrics.yaml + - gateway/servicemonitor.yaml + asserts: + - hasDocuments: + count: 0 + template: gateway/service-metrics.yaml + - hasDocuments: + count: 0 + template: gateway/servicemonitor.yaml + - equal: + path: metadata.name + value: RELEASE-NAME-litellm-metrics + template: monolith/service-metrics.yaml + - equal: + path: spec.selector["app.kubernetes.io/component"] + value: proxy + template: monolith/service-metrics.yaml + - equal: + path: spec.selector.matchLabels["app.kubernetes.io/component"] + value: proxy + template: monolith/servicemonitor.yaml + - equal: + path: spec.template.spec.containers[1].name + value: metrics + template: monolith/deployment.yaml + - equal: + path: spec.template.spec.containers[1].args[0] + value: metrics + template: monolith/deployment.yaml + + - it: monolith collector sidecar runs the collector dispatcher from the same image + set: + monolith.enabled: true + gateway.collector.enabled: true + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[1].name + value: collector + - equal: + path: spec.template.spec.containers[1].args + value: + - collector + - equal: + path: spec.template.spec.containers[1].image + value: ghcr.io/berriai/litellm:1.104.0 + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_COLLECTOR_ENABLED + value: "true" + + - it: monolith renders only the gateway ServiceAccount and runs the pod with it + set: + monolith.enabled: true + serviceAccounts.gateway.create: true + serviceAccounts.backend.create: true + serviceAccounts.ui.create: true + templates: + - serviceaccount.yaml + - monolith/deployment.yaml + asserts: + - hasDocuments: + count: 1 + template: serviceaccount.yaml + - equal: + path: metadata.name + value: RELEASE-NAME-litellm-gateway + template: serviceaccount.yaml + - equal: + path: spec.template.spec.serviceAccountName + value: RELEASE-NAME-litellm-gateway + template: monolith/deployment.yaml + + - it: the migrations Job renders the same way in monolith mode + set: + monolith.enabled: true + template: migrations-job.yaml + asserts: + - isKind: + of: Job + - equal: + path: spec.template.spec.containers[0].args + value: + - migrations + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0 + + - it: monolith gets the writer user under both names the proxy CLI and the componentized entrypoints read + set: + monolith.enabled: true + template: monolith/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_USER + valueFrom: + secretKeyRef: + name: litellm-writer-secret + key: username + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: litellm-writer-secret + key: username + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_NAME + value: litellm + + - it: disabling every component without monolith renders no application workload + set: + gateway.enabled: false + backend.enabled: false + ui.enabled: false + templates: + - monolith/deployment.yaml + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + asserts: + - hasDocuments: + count: 0 + + - it: monolith renders even when gateway.enabled is false + set: + monolith.enabled: true + gateway.enabled: false + templates: + - monolith/deployment.yaml + - monolith/service.yaml + asserts: + - hasDocuments: + count: 1 diff --git a/helm/litellm/tests/no_bundled_datastores_tests.yaml b/helm/litellm/tests/no_bundled_datastores_tests.yaml new file mode 100644 index 00000000000..dbafa5a9a11 --- /dev/null +++ b/helm/litellm/tests/no_bundled_datastores_tests.yaml @@ -0,0 +1,54 @@ +suite: the chart ships no PostgreSQL or Redis of its own +templates: + - gateway/deployment.yaml + - gateway/configmap.yaml + - migrations-job.yaml +tests: + - it: rejects the retired postgresql block instead of silently ignoring it + set: + postgresql.enabled: true + postgresql.auth.password: super-secret + asserts: + - failedTemplate: {} + + - it: rejects the retired redis.enabled flag instead of silently ignoring it + set: + redis.enabled: true + asserts: + - failedTemplate: {} + + - it: wires the writer straight from database.writer with no subchart in between + template: gateway/deployment.yaml + set: + database.writer.host: postgres.example.com + database.writer.dbname: proxydb + database.writer.passwordSecret.name: pg-secret + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_HOST + value: postgres.example.com + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_NAME + value: proxydb + - contains: + path: spec.template.spec.containers[0].env + content: + name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: pg-secret + key: password + + - it: keeps the migrations Job a pre-install hook whenever the Helm hook is enabled + template: migrations-job.yaml + set: + database.writer.host: postgres.example.com + database.writer.dbname: litellm + asserts: + - equal: + path: metadata.annotations["helm.sh/hook"] + value: pre-install,pre-upgrade diff --git a/helm/litellm/tests/redis_env_tests.yaml b/helm/litellm/tests/redis_env_tests.yaml index 79a14dfc6e5..849f13e5438 100644 --- a/helm/litellm/tests/redis_env_tests.yaml +++ b/helm/litellm/tests/redis_env_tests.yaml @@ -225,10 +225,10 @@ tests: name: REDIS_CLUSTER_NODES value: '[{"host":"$(REDIS_HOST)","port":6379}]' - equal: - path: spec.template.spec.containers[0].env[8].name + path: spec.template.spec.containers[0].env[10].name value: REDIS_HOST - equal: - path: spec.template.spec.containers[0].env[11].name + path: spec.template.spec.containers[0].env[13].name value: REDIS_CLUSTER_NODES - it: gateway gets REDIS_URL from urlKey without discrete Redis env diff --git a/helm/litellm/tests/rollout_strategy_tests.yaml b/helm/litellm/tests/rollout_strategy_tests.yaml index b12e2073c7c..af6e41d038e 100644 --- a/helm/litellm/tests/rollout_strategy_tests.yaml +++ b/helm/litellm/tests/rollout_strategy_tests.yaml @@ -64,3 +64,39 @@ tests: - notExists: path: spec.strategy template: ui/deployment.yaml + + - it: leaves minReadySeconds to the Kubernetes default when unset + asserts: + - notExists: + path: spec.minReadySeconds + + - it: renders the configured minReadySeconds on each deployment + set: + gateway.minReadySeconds: 60 + backend.minReadySeconds: 30 + ui.minReadySeconds: 10 + asserts: + - equal: + path: spec.minReadySeconds + value: 60 + template: gateway/deployment.yaml + - equal: + path: spec.minReadySeconds + value: 30 + template: backend/deployment.yaml + - equal: + path: spec.minReadySeconds + value: 10 + template: ui/deployment.yaml + + - it: renders an explicit minReadySeconds of 0 instead of dropping it + set: + gateway.minReadySeconds: 0 + asserts: + - equal: + path: spec.minReadySeconds + value: 0 + template: gateway/deployment.yaml + - notExists: + path: spec.minReadySeconds + template: backend/deployment.yaml diff --git a/helm/litellm/tests/shared_image_tests.yaml b/helm/litellm/tests/shared_image_tests.yaml new file mode 100644 index 00000000000..1e7837b1973 --- /dev/null +++ b/helm/litellm/tests/shared_image_tests.yaml @@ -0,0 +1,264 @@ +suite: test the shared image and the entrypoint dispatcher args +templates: + - gateway/deployment.yaml + - gateway/configmap.yaml + - backend/deployment.yaml + - ui/deployment.yaml + - monolith/deployment.yaml + - migrations-job.yaml + - templates/tests/test-connection.yaml +values: + - ./values/required.yaml +tests: + - it: every componentized container runs the shared image at the chart appVersion + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + - migrations-job.yaml + - templates/tests/test-connection.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0 + template: backend/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0 + template: ui/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0 + template: migrations-job.yaml + - equal: + path: spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0 + template: templates/tests/test-connection.yaml + + - it: image.repository and image.tag are applied to every container, sidecars included + set: + image.repository: registry.example.com/litellm + image.tag: v9.9.9 + image.pullPolicy: Always + gateway.metricsServer.enabled: true + gateway.collector.enabled: true + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + - migrations-job.yaml + - templates/tests/test-connection.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example.com/litellm:v9.9.9 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[1].image + value: registry.example.com/litellm:v9.9.9 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[2].image + value: registry.example.com/litellm:v9.9.9 + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: registry.example.com/litellm:v9.9.9 + template: backend/deployment.yaml + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + template: backend/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: registry.example.com/litellm:v9.9.9 + template: ui/deployment.yaml + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + template: ui/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: registry.example.com/litellm:v9.9.9 + template: migrations-job.yaml + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + template: migrations-job.yaml + - equal: + path: spec.containers[0].image + value: registry.example.com/litellm:v9.9.9 + template: templates/tests/test-connection.yaml + + - it: the monolith and its sidecars run the shared image + set: + monolith.enabled: true + image.tag: v9.9.9 + gateway.metricsServer.enabled: true + gateway.collector.enabled: true + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:v9.9.9 + - equal: + path: spec.template.spec.containers[1].image + value: ghcr.io/berriai/litellm:v9.9.9 + - equal: + path: spec.template.spec.containers[2].image + value: ghcr.io/berriai/litellm:v9.9.9 + + - it: image.digest renders as repository:tag@digest everywhere + set: + image.tag: v9.9.9 + image.digest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + - migrations-job.yaml + - templates/tests/test-connection.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:v9.9.9@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:v9.9.9@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + template: backend/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:v9.9.9@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + template: ui/deployment.yaml + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:v9.9.9@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + template: migrations-job.yaml + - equal: + path: spec.containers[0].image + value: ghcr.io/berriai/litellm:v9.9.9@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + template: templates/tests/test-connection.yaml + + - it: image.digest with an empty tag falls back to the appVersion tag + set: + image.digest: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + monolith.enabled: true + template: monolith/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/berriai/litellm:1.104.0@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + + - it: componentized containers select their component through args and never set command + set: + gateway.metricsServer.enabled: true + gateway.collector.enabled: true + templates: + - gateway/deployment.yaml + - backend/deployment.yaml + - ui/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].args + value: + - gateway + - --host + - 0.0.0.0 + - --port + - "4000" + template: gateway/deployment.yaml + - notExists: + path: spec.template.spec.containers[0].command + template: gateway/deployment.yaml + - notExists: + path: spec.template.spec.containers[1].command + template: gateway/deployment.yaml + - notExists: + path: spec.template.spec.containers[2].command + template: gateway/deployment.yaml + - equal: + path: spec.template.spec.containers[0].args + value: + - backend + - --host + - 0.0.0.0 + - --port + - "4001" + template: backend/deployment.yaml + - notExists: + path: spec.template.spec.containers[0].command + template: backend/deployment.yaml + - equal: + path: spec.template.spec.containers[0].args + value: + - ui + template: ui/deployment.yaml + - notExists: + path: spec.template.spec.containers[0].command + template: ui/deployment.yaml + + - it: the migrations Job runs the migrations dispatcher and nothing else + template: migrations-job.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].args + value: + - migrations + - notExists: + path: spec.template.spec.containers[0].command + + - it: the Helm test pod polls the workload Service readiness endpoint through the image entrypoint + template: templates/tests/test-connection.yaml + asserts: + - isKind: + of: Pod + - equal: + path: metadata.annotations["helm.sh/hook"] + value: test + - equal: + path: spec.containers[0].env[0] + value: + name: LITELLM_READINESS_URL + value: http://RELEASE-NAME-litellm-gateway:4000/health/readiness + - equal: + path: spec.containers[0].args[0] + value: python + - equal: + path: spec.containers[0].args[1] + value: -c + - matchRegex: + path: spec.containers[0].args[2] + pattern: LITELLM_READINESS_URL + - matchRegex: + path: spec.containers[0].args[2] + pattern: 'db != "connected"' + - matchRegex: + path: spec.containers[0].args[2] + pattern: sys\.exit\(1\) + - notExists: + path: spec.containers[0].command + + - it: the Helm test pod targets the monolith Service in monolith mode + set: + monolith.enabled: true + template: templates/tests/test-connection.yaml + asserts: + - equal: + path: spec.containers[0].env[0].value + value: http://RELEASE-NAME-litellm:4000/health/readiness + + - it: the removed per-component image blocks are rejected by the values schema + set: + gateway.image.tag: v1 + template: gateway/deployment.yaml + asserts: + - failedTemplate: {} diff --git a/helm/litellm/values.schema.json b/helm/litellm/values.schema.json new file mode 100644 index 00000000000..3aafbe68fe6 --- /dev/null +++ b/helm/litellm/values.schema.json @@ -0,0 +1,183 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "LiteLLM chart values", + "type": "object", + "additionalProperties": true, + "properties": { + "image": { + "type": "object", + "additionalProperties": false, + "required": [ + "repository" + ], + "properties": { + "repository": { + "type": "string", + "minLength": 1 + }, + "tag": { + "type": "string" + }, + "pullPolicy": { + "type": "string", + "enum": [ + "Always", + "IfNotPresent", + "Never" + ] + }, + "digest": { + "type": "string", + "pattern": "^(sha256:[a-f0-9]{64})?$" + } + } + }, + "imagePullSecrets": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + } + } + } + }, + "monolith": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "extraArgs": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "gateway": { + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "type": "boolean" + }, + "replicaCount": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "numWorkers": { + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "image": false + } + }, + "backend": { + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "type": "boolean" + }, + "replicaCount": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "image": false + } + }, + "ui": { + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "type": "boolean" + }, + "replicaCount": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "image": false + } + }, + "migrationJob": { + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "type": "boolean" + }, + "image": false + } + }, + "masterKey": { + "type": "object", + "additionalProperties": true, + "properties": { + "secretName": { + "type": "string" + }, + "secretKey": { + "type": "string" + }, + "generate": { + "type": "boolean" + } + } + }, + "postgresql": false, + "redis": { + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": false, + "cluster": { + "type": "boolean" + }, + "host": { + "type": "string" + }, + "port": { + "type": [ + "integer", + "string" + ] + } + } + }, + "ingress": { + "type": "object", + "additionalProperties": true, + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "extraResources": { + "type": "array", + "items": { + "type": "object" + } + } + } +} diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 92a48153c5a..b4795cd9ca6 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -1,10 +1,64 @@ # LiteLLM helm chart values +# +# Two deployment modes, both from the single image below: +# +# componentized (default): `gateway.enabled`, `backend.enabled` and +# `ui.enabled` each render their own Deployment, Service, HPA and PDB, so +# the LLM data plane, the management API and the static dashboard scale +# independently. Their containers run `args: [gateway|backend|ui, ...]`. +# +# monolith: `monolith.enabled: true` renders ONE Deployment and Service +# running the full proxy (`args: [proxy, ...]`): gateway routes, +# management routes and the Admin UI served by the proxy itself. The +# gateway, backend and ui Deployments and Services are not rendered and +# the Ingress sends every path to the monolith Service. The monolith is +# configured through the `gateway.*` values (config, resources, probes, +# securityContext, hpa, pdb, keda, collector, metricsServer, volumes, +# env), so a monolith install only flips this flag. See `monolith` below +# for the precedence rules. nameOverride: "" fullnameOverride: "" +# The one image every container in this chart runs: gateway, backend, ui, +# monolith, the migrations Job, the metrics and collector sidecars and the +# Helm test pod. Its entrypoint dispatches on the first container argument +# (proxy, gateway, backend, ui, migrations, metrics, collector), which is why +# the templates only ever set `args` and never `command`: overriding the +# entrypoint would drop the wrappers it applies (ddtrace, Prometheus +# multiprocess dir cleanup). `tag` defaults to the chart's appVersion. When +# `digest` is set the image renders as `repository:tag@digest`, so the tag is +# informational and the digest pins the bytes. Also mirrored at +# docker.litellm.ai/berriai/litellm. +image: + repository: ghcr.io/berriai/litellm + tag: "" + pullPolicy: IfNotPresent + digest: "" + imagePullSecrets: [] +# Monolith mode. When enabled the chart renders a single Deployment named +# `-litellm` running `args: [proxy, --port, 4000, --config, +# /app/config/config.yaml]` (the --config pair only when gateway.config.create +# is true) plus `extraArgs`, and a Service of the same name. Precedence when +# this is on: +# - gateway.enabled / backend.enabled / ui.enabled are ignored: no +# component Deployment, Service, HPA, PDB or ServiceMonitor is rendered +# - every `backend.*` and `ui.*` value is ignored, including their +# ServiceAccounts; the monolith runs as serviceAccounts.gateway +# - every `gateway.*` value applies to the monolith pod instead of a gateway +# pod: config, numWorkers, resources, probes, securityContext, +# hpa / keda / pdb, metricsServer, collector, volumes, env, scheduling +# - ingress.extraPaths entries keep their `service` field but every path, +# built in or extra, targets the monolith Service +# - the migrations Job renders exactly as in componentized mode +monolith: + enabled: false + # Extra CLI arguments appended after the chart's own proxy arguments, e.g. + # ["--detailed_debug"] or ["--run_granian"]. + extraArgs: [] + # Optional Ingress wiring the three component Services behind a single L7 # entrypoint. Required when serving the static UI bundle over the network. ingress: @@ -86,10 +140,8 @@ serviceAccounts: # LiteLLM_VerificationToken, LiteLLM_SpendLogs, ...). Disable if your # pipeline runs migrations out-of-band. # -# Uses a dedicated `litellm-migrations` image (prisma CLI + the migration -# files from `litellm-proxy-extras`) instead of the backend image, so the -# Job doesn't drag in the rest of the proxy and doesn't run `prisma -# generate` — the migration engine doesn't need the generated client. +# Runs the shared image with `args: [migrations]`, which the entrypoint +# dispatches to the prisma migration runner without importing the proxy. migrationJob: enabled: true # Which controller is responsible for running the Job. @@ -150,8 +202,18 @@ migrationJob: # pod. Same shape as gateway.podSecurityContext / gateway.securityContext. podSecurityContext: {} securityContext: {} + # Extra labels and annotations on the Job object itself. + jobLabels: {} + jobAnnotations: {} # Extra pod labels on the Job pod, merged into the chart's common labels. podLabels: {} + # Annotations on the Job pod, e.g. sidecar.istio.io/inject: "false" so an + # injected sidecar cannot keep the pod running after the migration exits. + annotations: {} + # Override the container command and args. The default runs the image + # entrypoint with `migrations`; a command with no args clears the args. + command: [] + args: [] # Additional volumes on the Job pod and volumeMounts on its container, e.g. # the writable scratch space a read-only root filesystem needs. volumes: [] @@ -163,42 +225,39 @@ migrationJob: nodeSelector: {} tolerations: [] affinity: {} - image: - repository: ghcr.io/berriai/litellm-migrations - tag: "" # defaults to .Chart.AppVersion - pullPolicy: IfNotPresent + # Extra init containers and sidecars on the Job pod, rendered through `tpl`. + extraInitContainers: [] + extraContainers: [] # Extra env appended to the migration container. The migration entrypoint # uses the v2 resolver by default (no diff-and-force recovery — avoids the # schema thrashing seen during rolling deploys). To opt back into the v1 # resolver, append `- name: USE_V2_MIGRATION_RESOLVER` / `value: "false"`. extraEnv: [] -# Required: a master key used by gateway + backend to mint/verify proxy tokens. -# Must reference an existing Secret. +# Master key used by the proxy to mint and verify virtual keys. Reference an +# existing Secret through `secretName` / `secretKey`, or set `generate: true` +# and leave `secretName` empty to have the chart create +# `-litellm-masterkey` with a random `sk-...` key under `secretKey`. +# The generated Secret is looked up on upgrade so the key never rotates by +# accident (`helm template` cannot look anything up and renders a fresh key +# each time; that is expected). The chart never accepts an inline key. masterKey: secretName: litellm-master-key-secret # name of a Secret containing the master key secretKey: master-key + generate: false # Optional: enterprise billable-request metering. When enabled, the gateway and -# backend count successful requests to inference, MCP, and A2A endpoints and push -# them to LiteLLM's collector over mutual TLS. Both components serve billable -# routes: the backend keeps the named-server MCP transport. Requires an -# enterprise license. The client certificate identifies the deployment, so it is -# mounted read-only from an existing Secret and never passed through the env. +# backend count successful requests to inference, MCP, and A2A endpoints. Requires +# an enterprise license. billingMetrics: enabled: false - endpoint: https://telemetry.litellm.ai # collector to push the counter to - # An existing Secret holding the client certificate under tls.crt and its key - # under tls.key, usually created from the onboarding artifact. The default is - # the conventional name, so the common path is to create that Secret and set - # enabled: true. Override only if yours is named differently. - secretName: litellm-billing-metrics-mtls - # Only for private or test collectors whose server certificate is not on the - # public web PKI. The production collector needs no CA override. - caSecretName: "" # existing Secret holding ca.crt - exportIntervalMs: "" # push cadence; the proxy defaults to 60000 + endpoint: https://telemetry.litellm.ai + tokenUrl: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token + secretName: litellm-billing-metrics + exportIntervalMs: "" -# External Postgres connection. +# Postgres connection. The chart ships no database; point `writer` at your +# own PostgreSQL (managed or self hosted) and its credentials Secret. database: writer: host: "" @@ -258,24 +317,23 @@ database: maxDbConnections: 20 maxClientConn: 1000 -# Optional Redis. Leave host empty to disable. +# Redis is the proxy's coordination store: cross-pod tpm/rpm rate limits, +# spend tracking, and the pod lock manager. The chart emits REDIS_HOST / +# REDIS_PORT / REDIS_PASSWORD, which the proxy picks up through its +# coordination Redis env fallback. Response caching is separate and off unless +# you enable it in `proxy_config.litellm_settings.cache`. # -# This is the proxy's coordination store: cross-pod tpm/rpm rate limits, spend -# tracking, and the pod lock manager. The chart emits REDIS_HOST / REDIS_PORT / -# REDIS_PASSWORD, which the proxy picks up through its coordination Redis env -# fallback. Response caching is separate and off unless you enable it in -# `proxy_config.litellm_settings.cache`. +# Set `host` (and `passwordSecret.name` when auth is required). Set +# `cluster: true` for Redis Cluster mode (AWS ElastiCache Cluster, self-hosted +# Redis Cluster): the chart emits REDIS_CLUSTER_NODES from `host` / `port` as +# the single seed and the client discovers the rest from CLUSTER SLOTS. The +# chart ships no Redis of its own. # # For full control, define `general_settings.coordination_redis` in # `proxy_config` (host/port/password/username/url/ssl/startup_nodes/ # sentinel_nodes/sentinel_password/service_name, each accepting os.environ/VAR # refs). An explicit block overrides these env vars. # -# Set `cluster: true` for Redis Cluster mode (e.g. AWS ElastiCache Cluster, -# self-hosted Redis Cluster). The chart emits REDIS_CLUSTER_NODES from -# `host` / `port` as the single seed; the cluster client discovers the -# remaining nodes from CLUSTER SLOTS at startup. -# # For a TLS Redis endpoint set `ssl: true`, which the chart emits as REDIS_SSL. # `sslCertReqs` (none / optional / required) becomes REDIS_SSL_CERT_REQS and # `sslCaCerts` (path to a CA bundle mounted in the pod, e.g. via @@ -296,12 +354,17 @@ redis: hostKey: "" # Secret key containing the Redis host urlKey: "" # Secret key containing the complete Redis URL +# Arbitrary extra Kubernetes manifests rendered verbatim with the release, +# e.g. a NetworkPolicy or an ExternalSecret the proxy needs. +extraResources: [] + # ---------- gateway (LLM data plane) ---------- +# In monolith mode this whole block configures the monolith pod instead. gateway: enabled: true logLevel: INFO - # Number of uvicorn worker processes per gateway pod. Sets NUM_WORKERS, - # consumed by the gateway image entrypoint. Default is 1. + # Number of worker processes per pod. Sets NUM_WORKERS, read by the gateway + # launcher and by the proxy's --num_workers default. Default is 1. numWorkers: 1 extraEnv: [] # Add extra environment variables to the gateway envConfigMaps: [] # Add extra environment variables to the gateway from config maps @@ -320,7 +383,6 @@ gateway: # scrape never runs on an inference worker. Adds a `metrics` port to the pod # and a dedicated ClusterIP `-metrics` Service; point your scrape # config at it. The port has no virtual-key auth: keep it off public ingress. - # Needs the gateway image v1.101.0 or newer. metricsServer: enabled: false port: 4001 @@ -338,8 +400,15 @@ gateway: serviceMonitor: enabled: false labels: {} + annotations: {} interval: 15s scrapeTimeout: 10s + relabelings: [] + # Empty scrapes the release namespace. + namespaceSelector: + matchNames: [] + scheme: http + tlsConfig: {} # Opt-in `collector` sidecar (same image, `python -m litellm.proxy.collector`) # that runs the post-response spend pipeline (cost calculation, spend logs, # spend counters, budget reservation reconciliation) so the uvicorn workers @@ -376,13 +445,13 @@ gateway: # ContainerResource metric of the `gateway` container only, so the # sidecar's CPU never drives inference replicas. Needs Kubernetes 1.30+. scaleOnGatewayContainerCpu: true - image: - repository: ghcr.io/berriai/litellm-gateway - tag: "" # defaults to .Chart.AppVersion - pullPolicy: IfNotPresent service: type: ClusterIP port: 4000 + annotations: {} + # For LoadBalancer Services on clusters with several load balancer + # implementations. + loadBalancerClass: "" resources: requests: cpu: "1" @@ -456,6 +525,41 @@ gateway: # counted when a response completes, so TPS trails long streams. targetRequestsPerSecond: "" targetTokensPerSecond: "" + # KEDA ScaledObject as an alternative to the HPA above. Mutually exclusive + # with hpa.enabled (two autoscalers on one Deployment fight each other), so + # set `hpa.enabled: false` when turning this on. Needs KEDA installed in the + # cluster. `triggers` are rendered verbatim; the two optional Prometheus + # workload targets below add prometheus triggers on the same counters the + # HPA workload metrics use, scraped through the metrics Service (enable + # metricsServer and serviceMonitor). + keda: + enabled: false + minReplicaCount: 1 + maxReplicaCount: 10 + pollingInterval: 30 + cooldownPeriod: 300 + # Rendered on the ScaledObject, e.g. autoscaling.keda.sh/paused-replicas: "2" + # to pause scaling at a fixed replica count + scaledObject: + annotations: {} + # Optional; rendered verbatim under spec.advanced + advanced: {} + # Optional; rendered under spec.fallback. A missing `failureThreshold` + # defaults to 3 and a missing `replicas` to maxReplicaCount, as in litellm-helm. + fallback: {} + # Custom triggers, rendered verbatim, e.g. + # - type: cpu + # metricType: Utilization + # metadata: + # value: "70" + triggers: [] + prometheus: + # Prometheus server URL, required when either workload target is set. + serverAddress: "" + # Per pod averages, same units as hpa.targetRequestsPerSecond / + # hpa.targetTokensPerSecond. Empty leaves the trigger out. + targetRequestsPerSecond: "" + targetTokensPerSecond: "" # PodDisruptionBudget for the gateway pods. Set exactly one of # `minAvailable` / `maxUnavailable` (minAvailable wins if both are set; # enabling without either falls back to `maxUnavailable: 1`). Disabled by @@ -465,6 +569,13 @@ gateway: enabled: false minAvailable: "" maxUnavailable: "" + # Extra annotations and labels on the Deployment object itself (not the + # pods), e.g. for a GitOps controller or a reloader. + deploymentAnnotations: {} + deploymentLabels: {} + # Seconds a new pod must be ready before the rollout treats it as + # available. Empty inherits the Kubernetes default of 0. + minReadySeconds: "" podAnnotations: {} # Extra pod labels, merged into the chart's selector labels. Do not # re-declare `app.kubernetes.io/name` / `instance` / `component` here: they @@ -491,7 +602,16 @@ gateway: # egress proxy. Rendered through `tpl`, so entries may reference chart # values and release metadata. extraContainers: [] + # Init containers, rendered through `tpl` the same way. + extraInitContainers: [] # Container lifecycle hooks (postStart / preStop) for the gateway container. + # + # Prefer the proxy's /health/drain preStop hook over a fixed `sleep`: it + # marks the pod NotReady and blocks only until in-flight requests finish + # (bounded by GRACEFUL_SHUTDOWN_TIMEOUT). It is off by default; enable it + # with general_settings.enable_drain_endpoint: true and, when the health + # port is reachable from other pods, set general_settings.drain_endpoint_token + # and send the same value on the X-Drain-Token header from the hook. lifecycle: {} # Grace period the kubelet allows between SIGTERM and SIGKILL. Leave empty # to inherit the Kubernetes default of 30s. Set it a few seconds above the @@ -522,13 +642,11 @@ backend: volumes: [] # Additional volumeMounts on the backend container. volumeMounts: [] - image: - repository: ghcr.io/berriai/litellm-backend - tag: "" - pullPolicy: IfNotPresent service: type: ClusterIP port: 4001 + annotations: {} + loadBalancerClass: "" resources: requests: cpu: "1" @@ -565,12 +683,16 @@ backend: enabled: false minAvailable: "" maxUnavailable: "" + deploymentAnnotations: {} + deploymentLabels: {} + minReadySeconds: "" podAnnotations: {} # Same shape as the gateway blocks of the same name. podLabels: {} podSecurityContext: {} securityContext: {} extraContainers: [] + extraInitContainers: [] lifecycle: {} terminationGracePeriodSeconds: "" nodeSelector: {} @@ -590,13 +712,13 @@ ui: volumes: [] # Additional volumeMounts on the ui container. volumeMounts: [] - image: - repository: ghcr.io/berriai/litellm-ui - tag: "" - pullPolicy: IfNotPresent service: type: ClusterIP port: 3000 + annotations: {} + loadBalancerClass: "" + # The dashboard expects to know where to reach the backend API. Set this to + # the externally-routable URL (typically the ingress host + /api or similar). # The backend's public origin, passed to the UI only when ingress.componentHosts is configured. backendUrl: "" # Public origin of the UI, used for backend and gateway CORS. @@ -634,17 +756,19 @@ ui: enabled: false minAvailable: "" maxUnavailable: "" + deploymentAnnotations: {} + deploymentLabels: {} + minReadySeconds: "" podAnnotations: {} # Same shape as the gateway blocks of the same name. The nginx runtime # writes its pid, cache, and proxy temp files under /tmp, so it boots as # any (arbitrary, non-root) uid; `securityContext.readOnlyRootFilesystem: - # true` here needs an emptyDir volume mounted over /tmp. Images before - # the /tmp move instead need emptyDirs over /var/cache/nginx and /run to - # run as a non-root uid at all. + # true` here needs an emptyDir volume mounted over /tmp. podLabels: {} podSecurityContext: {} securityContext: {} extraContainers: [] + extraInitContainers: [] lifecycle: {} terminationGracePeriodSeconds: "" nodeSelector: {} @@ -653,6 +777,24 @@ ui: # Same shape as gateway.topologySpreadConstraints. topologySpreadConstraints: [] +# Optional: bundled LiteAdmin worker. Renders one Deployment, Service and PVC +# running the shared image with `args: ["--admin-agent"]`, and points the pod +# that serves the /liteadmin routes (the backend in componentized mode, the +# monolith in monolith mode) at it via LITELLM_ADMIN_AGENT_URL. +liteadmin: + enabled: false + existingSecret: "" + gatewayUrl: "" + model: "" + readOnly: false + storageSize: 1Gi + storageClassName: "" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi lens: enabled: false url: "" diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261029000000_add_background_response_settlement/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261029000000_add_background_response_settlement/migration.sql new file mode 100644 index 00000000000..4965ed3cfc4 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261029000000_add_background_response_settlement/migration.sql @@ -0,0 +1,9 @@ +SET LOCAL lock_timeout = '10s'; + +-- CreateTable +CREATE TABLE IF NOT EXISTS "LiteLLM_BackgroundResponseSettlement" ( + "response_id" TEXT NOT NULL, + "settled_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "LiteLLM_BackgroundResponseSettlement_pkey" PRIMARY KEY ("response_id") +); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql new file mode 100644 index 00000000000..b00cf892b68 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql @@ -0,0 +1,19 @@ +SET LOCAL lock_timeout = '10s'; +CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogDelivery" ( + "delivery_id" TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text, + "request_id" TEXT NOT NULL, + "state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending', + "attempts" INTEGER NOT NULL DEFAULT 0, + "next_attempt_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "applied_at" TIMESTAMP(3), + "last_error" TEXT, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "accounting_payload" JSONB NOT NULL, + "is_batch" BOOLEAN NOT NULL +); +CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_state_next_attempt_at_idx" + ON "LiteLLM_SpendLogDelivery" ("state", "next_attempt_at"); +CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_state_applied_at_idx" + ON "LiteLLM_SpendLogDelivery" ("state", "applied_at"); +CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_request_id_idx" + ON "LiteLLM_SpendLogDelivery" ("request_id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261104010000_drop_accounting_event_tables/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261104010000_drop_accounting_event_tables/migration.sql new file mode 100644 index 00000000000..cba6a18368f --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261104010000_drop_accounting_event_tables/migration.sql @@ -0,0 +1,5 @@ +SET LOCAL lock_timeout = '10s'; +DROP TABLE IF EXISTS "LiteLLM_AccountingDelivery"; +DROP TABLE IF EXISTS "LiteLLM_AccountingEvent"; +DROP TYPE IF EXISTS "AccountingDeliveryTarget"; +DROP TYPE IF EXISTS "AccountingEventKind"; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_skill_version_history/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_skill_version_history/migration.sql new file mode 100644 index 00000000000..ecfa7321d87 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105000000_skill_version_history/migration.sql @@ -0,0 +1,16 @@ +SET LOCAL lock_timeout = '10s'; +CREATE TABLE IF NOT EXISTS "LiteLLM_ClaudeCodePluginVersion" ( + "id" TEXT NOT NULL, + "plugin_id" TEXT NOT NULL, + "version" TEXT, + "description" TEXT, + "manifest_json" TEXT, + "published_at" TIMESTAMP(3), + "published_by" TEXT, + "archive_available" BOOLEAN NOT NULL DEFAULT false, + "content" BYTEA, + CONSTRAINT "LiteLLM_ClaudeCodePluginVersion_pkey" PRIMARY KEY ("id"), + CONSTRAINT "LiteLLM_ClaudeCodePluginVersion_plugin_id_fkey" FOREIGN KEY ("plugin_id") REFERENCES "LiteLLM_ClaudeCodePluginTable"("id") ON DELETE CASCADE ON UPDATE CASCADE +); +CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_ClaudeCodePluginVersion_plugin_id_version_key" ON "LiteLLM_ClaudeCodePluginVersion"("plugin_id", "version"); +CREATE INDEX IF NOT EXISTS "LiteLLM_ClaudeCodePluginVersion_plugin_id_published_at_idx" ON "LiteLLM_ClaudeCodePluginVersion"("plugin_id", "published_at"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105010000_skill_submissions/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105010000_skill_submissions/migration.sql new file mode 100644 index 00000000000..e7975258ca9 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105010000_skill_submissions/migration.sql @@ -0,0 +1,27 @@ +SET LOCAL lock_timeout = '10s'; + +CREATE TABLE IF NOT EXISTS "LiteLLM_SkillSubmission" ( + "id" TEXT NOT NULL, + "plugin_name" TEXT NOT NULL, + "operation" TEXT NOT NULL, + "status" TEXT NOT NULL DEFAULT 'pending_review', + "proposal_json" TEXT NOT NULL, + "archive_content" BYTEA, + "submitted_by" TEXT NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "base_plugin_id" TEXT, + "base_updated_at" TIMESTAMP(3), + "reviewed_by" TEXT, + "reviewed_at" TIMESTAMP(3), + "rejection_reason" TEXT, + CONSTRAINT "LiteLLM_SkillSubmission_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX IF NOT EXISTS "LiteLLM_SkillSubmission_submitted_by_created_at_id_idx" + ON "LiteLLM_SkillSubmission"("submitted_by", "created_at" DESC, "id" DESC); +CREATE INDEX IF NOT EXISTS "LiteLLM_SkillSubmission_submitted_by_status_created_at_id_idx" + ON "LiteLLM_SkillSubmission"("submitted_by", "status", "created_at" DESC, "id" DESC); +CREATE INDEX IF NOT EXISTS "LiteLLM_SkillSubmission_status_created_at_id_idx" + ON "LiteLLM_SkillSubmission"("status", "created_at" DESC, "id" DESC); +CREATE INDEX IF NOT EXISTS "LiteLLM_SkillSubmission_created_at_id_idx" + ON "LiteLLM_SkillSubmission"("created_at" DESC, "id" DESC); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105020000_skill_scanning/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105020000_skill_scanning/migration.sql new file mode 100644 index 00000000000..6b70cca4a9a --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105020000_skill_scanning/migration.sql @@ -0,0 +1,3 @@ +SET LOCAL lock_timeout = '10s'; + +ALTER TABLE "LiteLLM_SkillSubmission" ADD COLUMN IF NOT EXISTS "scan_json" TEXT; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105094500_add_skill_download_history/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105094500_add_skill_download_history/migration.sql new file mode 100644 index 00000000000..43e26b324a0 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261105094500_add_skill_download_history/migration.sql @@ -0,0 +1,37 @@ +SET LOCAL lock_timeout = '10s'; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid = to_regclass('"LiteLLM_ClaudeCodePluginTable"') + AND attname = 'anonymous_download_count' AND NOT attisdropped + ) THEN + ALTER TABLE "LiteLLM_ClaudeCodePluginTable" + ADD COLUMN IF NOT EXISTS "anonymous_download_count" BIGINT NOT NULL DEFAULT 0; + END IF; + + IF to_regclass('"LiteLLM_ClaudeCodePluginDownload"') IS NULL THEN + CREATE TABLE IF NOT EXISTS "LiteLLM_ClaudeCodePluginDownload" ( + "id" BIGSERIAL NOT NULL, + "plugin_id" TEXT NOT NULL, + "downloaded_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "version" TEXT, + "digest" TEXT NOT NULL, + "authenticated" BOOLEAN NOT NULL, + "user_id" TEXT, + "user_email" TEXT, + "key_alias" TEXT, + "team_id" TEXT, + "team_alias" TEXT, + CONSTRAINT "LiteLLM_ClaudeCodePluginDownload_pkey" PRIMARY KEY ("id"), + CONSTRAINT "LiteLLM_ClaudeCodePluginDownload_plugin_id_fkey" FOREIGN KEY ("plugin_id") + REFERENCES "LiteLLM_ClaudeCodePluginTable"("id") ON DELETE CASCADE ON UPDATE CASCADE + ); + END IF; + + IF to_regclass('"LiteLLM_ClaudeCodePluginDownload_plugin_id_id_idx"') IS NULL THEN + CREATE INDEX IF NOT EXISTS "LiteLLM_ClaudeCodePluginDownload_plugin_id_id_idx" + ON "LiteLLM_ClaudeCodePluginDownload"("plugin_id", "id"); + END IF; +END $$; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261110000000_add_budget_model_tpm_rpm_limit/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261110000000_add_budget_model_tpm_rpm_limit/migration.sql new file mode 100644 index 00000000000..6b77194e3b4 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261110000000_add_budget_model_tpm_rpm_limit/migration.sql @@ -0,0 +1,7 @@ +SET LOCAL lock_timeout = '10s'; + +-- AlterTable +ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "model_tpm_limit" JSONB; + +-- AlterTable +ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "model_rpm_limit" JSONB; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261110120000_add_daily_spend_guardrail_spend/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261110120000_add_daily_spend_guardrail_spend/migration.sql new file mode 100644 index 00000000000..c2c264325a6 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261110120000_add_daily_spend_guardrail_spend/migration.sql @@ -0,0 +1,22 @@ +SET LOCAL lock_timeout = '10s'; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyGlobalSpend" ADD COLUMN IF NOT EXISTS "guardrail_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index c007b047087..5a5d56cc6e2 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -19,6 +19,8 @@ model LiteLLM_BudgetTable { rpm_limit BigInt? tpd_limit BigInt? model_max_budget Json? + model_tpm_limit Json? + model_rpm_limit Json? budget_duration String? budget_reset_at DateTime? allowed_models String[] @default([]) // per-member model scope; empty = inherit team models @@ -881,6 +883,7 @@ model LiteLLM_DailyUserSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -917,6 +920,7 @@ model LiteLLM_DailyGlobalSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -950,6 +954,7 @@ model LiteLLM_DailyOrganizationSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -988,6 +993,7 @@ model LiteLLM_DailyEndUserSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1025,6 +1031,7 @@ model LiteLLM_DailyAgentSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1062,6 +1069,7 @@ model LiteLLM_DailyTeamSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1102,6 +1110,7 @@ model LiteLLM_DailyTagSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + guardrail_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1616,10 +1625,69 @@ model LiteLLM_ClaudeCodePluginTable { created_by String? uploaded_archive LiteLLM_ClaudeCodePluginUpload? + versions LiteLLM_ClaudeCodePluginVersion[] + downloads LiteLLM_ClaudeCodePluginDownload[] + anonymous_download_count BigInt @default(0) @@map("LiteLLM_ClaudeCodePluginTable") } +model LiteLLM_ClaudeCodePluginVersion { + id String @id @default(uuid()) + plugin_id String + version String? + description String? + manifest_json String? + published_at DateTime? + published_by String? + archive_available Boolean @default(false) + content Bytes? + plugin LiteLLM_ClaudeCodePluginTable @relation(fields: [plugin_id], references: [id], onDelete: Cascade, onUpdate: Cascade) + + @@unique([plugin_id, version]) + @@index([plugin_id, published_at]) +} + + +model LiteLLM_SkillSubmission { + id String @id @default(uuid()) + plugin_name String + operation String + status String @default("pending_review") + proposal_json String + archive_content Bytes? + submitted_by String + created_at DateTime @default(now()) + base_plugin_id String? + base_updated_at DateTime? + reviewed_by String? + reviewed_at DateTime? + rejection_reason String? + scan_json String? + + @@index([submitted_by, created_at(sort: Desc), id(sort: Desc)]) + @@index([submitted_by, status, created_at(sort: Desc), id(sort: Desc)]) + @@index([status, created_at(sort: Desc), id(sort: Desc)]) + @@index([created_at(sort: Desc), id(sort: Desc)]) +} + +model LiteLLM_ClaudeCodePluginDownload { + id BigInt @id @default(autoincrement()) + plugin_id String + downloaded_at DateTime @default(now()) + version String? + digest String + authenticated Boolean + user_id String? + user_email String? + key_alias String? + team_id String? + team_alias String? + plugin LiteLLM_ClaudeCodePluginTable @relation(fields: [plugin_id], references: [id], onDelete: Cascade, onUpdate: Cascade) + + @@index([plugin_id, id]) +} + model LiteLLM_ClaudeCodePluginUpload { plugin_name String @id content Bytes @@ -1994,6 +2062,11 @@ model LiteLLM_BackgroundInteractionSettlement { @@index([settled_at], map: "idx_background_interaction_settlement_settled_at") } +model LiteLLM_BackgroundResponseSettlement { + response_id String @id + settled_at DateTime @default(now()) +} + model LiteLLM_Lens { reviews LiteLLM_LensReview[] id String @id @@ -2071,74 +2144,27 @@ model LiteLLM_TelemetryReport { @@index([created_at]) } -enum AccountingEventKind { - inbound_usage - outbound_usage - cost -} - enum AccountingDeliveryState { pending applied parked } -enum AccountingDeliveryTarget { - LiteLLM_VerificationToken - LiteLLM_UserTable - LiteLLM_TeamTable - LiteLLM_TeamMembership - LiteLLM_OrganizationTable - LiteLLM_OrganizationMembership - LiteLLM_EndUserTable - LiteLLM_AgentsTable - LiteLLM_TagTable - LiteLLM_ProjectTable - LiteLLM_ModelAccessGroupBudgetTable - LiteLLM_BudgetTable - LiteLLM_BudgetWindowSpend - LiteLLM_DailyUserSpend - LiteLLM_DailyTeamSpend - LiteLLM_DailyOrganizationSpend - LiteLLM_DailyEndUserSpend - LiteLLM_DailyAgentSpend - LiteLLM_DailyTagSpend - LiteLLM_DailyModelUsage - LiteLLM_DailyToolSpend - LiteLLM_AutoRouterDailySpend -} +model LiteLLM_SpendLogDelivery { + delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text")) + request_id String + state AccountingDeliveryState @default(pending) + accounting_payload Json + is_batch Boolean + attempts Int @default(0) + next_attempt_at DateTime @default(now()) + applied_at DateTime? + last_error String? + created_at DateTime @default(now()) -// Durable accounting ledger. One row per usage or cost event; payload holds the -// private event envelope versioned by schema_version. Usage rows retain request-scoped -// pricing context; cost rows include resolved rates and the finalized pricing snapshot. -model LiteLLM_AccountingEvent { - event_id String @id - kind AccountingEventKind - source_event_id String? - schema_version Int - payload_hash String - occurred_at DateTime - recorded_at DateTime @default(now()) - payload Json - - @@index([recorded_at]) - @@index([kind, source_event_id]) -} - -// One row per (event, target table, version): pending work for the projector and, -// once applied in the same transaction as the target write, the receipt that stops a retry. -model LiteLLM_AccountingDelivery { - event_id String - target AccountingDeliveryTarget - version Int - state AccountingDeliveryState @default(pending) - attempts Int @default(0) - next_attempt_at DateTime @default(now()) - applied_at DateTime? - last_error String? - - @@id([event_id, target, version]) - @@index([target, state, next_attempt_at]) + @@index([state, next_attempt_at]) + @@index([state, applied_at]) + @@index([request_id]) } model LiteLLM_SubscriptionAccountTable { diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index a12e1e27cfc..1c0c708ae7a 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.112" +version = "0.4.114" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -30,7 +30,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.112" +version = "0.4.114" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm-rust/.agents/skills/rust-gateway/SKILL.md b/litellm-rust/.agents/skills/rust-gateway/SKILL.md index 4fee0b54a68..923c9bf5ab6 100644 --- a/litellm-rust/.agents/skills/rust-gateway/SKILL.md +++ b/litellm-rust/.agents/skills/rust-gateway/SKILL.md @@ -7,7 +7,7 @@ metadata: # Rust gateway -`crates/gateway` mounts endpoint routers and serves the listener. `crates/gateway-inference`, `gateway-mcp` and `gateway-ui` own their endpoints, `gateway-auth` is the inbound authentication middleware and `gateway-management` holds management logic that has no router yet. A crate's `AGENTS.md`, where it has one, says what belongs in it. Read the reference for the part you are touching +`crates/gateway` mounts endpoint routers and serves the listener. `crates/gateway-inference`, `gateway-mcp` and `gateway-ui` own their endpoints, `gateway-auth` is the inbound authentication middleware and `gateway-management` holds management logic that has no router yet. `gateway-forward` is the reverse proxy that relays every request to the Python proxy and has no routes of its own. A crate's `AGENTS.md`, where it has one, says what belongs in it. Read the reference for the part you are touching - [surfaces](references/surfaces.md) - Each API surface mirrors the envelope the Python proxy returns there, so the shapes differ on purpose diff --git a/litellm-rust/.agents/skills/rust-python-boundary/SKILL.md b/litellm-rust/.agents/skills/rust-python-boundary/SKILL.md new file mode 100644 index 00000000000..5ead8f287ab --- /dev/null +++ b/litellm-rust/.agents/skills/rust-python-boundary/SKILL.md @@ -0,0 +1,53 @@ +--- +name: rust-python-boundary +description: Decide where Python-shaped input becomes typed Rust in litellm-rust. Use when reading config.yaml values, os.environ/ references, REDIS_* or other environment layering, get_secret fallbacks, or kwargs and objects handed over from Python, when coercing text to a bool, int or float the way Python does, or when adding a dependency on litellm-python-compat +metadata: + internal: true +--- + +# Python boundary + +Python-shaped input becomes typed Rust in exactly two crates. Every crate past them handles typed values, so no crate re-guesses how Python would have read a value + +## The two boundaries + +- `crates/config` reads config.yaml, which is Python-shaped: `os.environ/` references, scalars that Helm or the environment turned into text, raw kwargs maps. It also owns the environment layering and `get_secret` fallbacks Python applies while it loads that config. It has no Python runtime, so it reproduces Python value semantics with `litellm-python-compat` +- `crates/python-bridge` receives live Python objects, such as `RedisCache.redis_kwargs`. It reads them through PyO3 with the real protocol (`is_truthy()`, `str()`, `extract`), so it never imitates Python +- Both produce the same typed input, defined in the data crate the consumer already depends on (`redis-types` for Redis), so a downstream crate cannot tell which host it runs under + +## Python value semantics + +Reading a value the way Python itself reads it. This lives only at the two boundaries + +- Truthiness and `x or fallback`, `str(x)`, `int(text)` and `float(text)` +- Each surface's true and false text: `_str_to_bool`, pydantic's lax bool, `get_secret`'s `true` and `false` +- `os.environ/` references and `REDIS_` style environment layering + +A boundary knows which surface a value came from. `general_settings.coordination_redis` is a pydantic model and `cache_params` goes through `_str_to_bool`, so the boundary applies that surface's rule exactly instead of refusing the text two surfaces disagree on + +Each typed field keeps where its value came from (the key, `REDIS_` or the fallback name), so an error or warning downstream still names it + +## What stays downstream + +A crate that stands in for a Python library keeps that library's semantics, because they are its compatibility contract and not value coercion. `redis-resolve` stands in for redis-py, so it keeps + +- Which target wins, and the defaults redis-py applies +- How `parse_url` reads a url, including `int()` on `?db=` and redis-py's `to_bool` on the TLS query parameters +- Which kwargs redis-py accepts, and whether the Rust client reads, warns about or refuses each one + +Its input is typed. A rule it applies reads typed fields, never a `serde_json::Value` it has to coerce first. A value it only carries back to Python untouched, like `passthrough`, may stay untyped + +## litellm-python-compat + +The rule is about where config and kwargs get coerced, not about who may depend on the crate + +- Its data formats (`pickle`, `json.dumps`, `repr`) are fine anywhere Rust reads or writes state Python also reads or writes, such as cache values in a shared Redis +- Its coercions (`truthy`, `number`) belong at the two boundaries, and in a crate that stands in for a Python library when that library calls them itself + +## Existing code + +Some code still coerces Python-shaped values past the boundary, for example the `REDIS_*` merge in `gateway/src/redis/layers.rs` and the flag and number readers in `redis-resolve/src/options.rs`. Do not add more. When you change one, move the coercion to its boundary instead of extending it in place + +## Tests + +Pin a boundary's coercion with named cases next to the Python function it mirrors, in the Rust test and in its Python twin, like the Redis contract cases in `redis-resolve/tests/interpret.rs` and `tests/unit/test_redis.py`. A downstream crate's tests build its typed input directly and never feed it raw text to coerce diff --git a/litellm-rust/AGENTS.md b/litellm-rust/AGENTS.md index 5eaf876e3b2..5fcf05f61f7 100644 --- a/litellm-rust/AGENTS.md +++ b/litellm-rust/AGENTS.md @@ -8,6 +8,10 @@ For string-valued enums and their Serde conversions, follow [.agents/skills/rust For error types, error variants and conversions between crate errors, follow [.agents/skills/rust-error/SKILL.md](.agents/skills/rust-error/SKILL.md) +For where config.yaml values, environment layering and kwargs from Python become typed Rust, follow [.agents/skills/rust-python-boundary/SKILL.md](.agents/skills/rust-python-boundary/SKILL.md) + +The process-wide rustls `CryptoProvider` (ring) is installed once by the host. rust-proxy does it at the top of `main` through `litellm_gateway::install_crypto_provider`, and python-bridge must do the same in its module init before it builds any rustls config. Library crates never install it. The AWS SDK links aws-lc-rs next to ring, so without an installed default the first rustls config a library builds, such as redis-rs TLS, panics + For fieldless enums, derive `strum::VariantArray` and use `VARIANTS` instead of a hand-listed `ALL` array. Derive Strum string conversions instead of hand-written variant-to-string matches Use the derived conversions directly (`<&'static str>::from(x)` / `.into()`, `str::parse`) with no `as_str`/`parse` wrapper that only delegates, and spell each variant with explicit `#[strum(serialize = "...")]` instead of `serialize_all` diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index b93a8827428..2cdc9ee91c8 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -8,6 +8,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "ahash" version = "0.8.12" @@ -547,7 +558,7 @@ dependencies = [ "http 1.4.2", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-rustls 0.24.2", "hyper-rustls 0.27.9", "hyper-util", @@ -736,7 +747,7 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-util", "itoa", "matchit", @@ -883,6 +894,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ "fastrand", + "tokio", ] [[package]] @@ -982,6 +994,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "bollard" version = "0.20.2" @@ -1000,7 +1021,7 @@ dependencies = [ "home", "http 1.4.2", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-named-pipe", "hyper-rustls 0.27.9", "hyper-util", @@ -1134,6 +1155,15 @@ dependencies = [ "rustversion", ] +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" version = "1.3.0" @@ -1210,6 +1240,16 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" version = "4.6.7" @@ -1774,6 +1814,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "des" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e" +dependencies = [ + "cipher", +] + [[package]] name = "digest" version = "0.10.7" @@ -2255,7 +2304,7 @@ dependencies = [ "chrono", "http 1.4.2", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-rustls 0.27.9", "hyper-util", "ring", @@ -2391,7 +2440,7 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "lazy_static", "opentelemetry", "opentelemetry-semantic-conventions", @@ -2833,9 +2882,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.10.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", @@ -2860,7 +2909,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" dependencies = [ "hex", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -2889,7 +2938,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http 1.4.2", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-util", "rustls 0.23.42", "rustls-native-certs", @@ -2904,7 +2953,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.10.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -2923,7 +2972,7 @@ dependencies = [ "futures-util", "http 1.4.2", "http-body 1.1.0", - "hyper 1.10.1", + "hyper 1.12.0", "ipnet", "libc", "percent-encoding", @@ -2942,7 +2991,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-util", "pin-project-lite", "tokio", @@ -3105,6 +3154,16 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -3389,15 +3448,19 @@ version = "0.1.0" dependencies = [ "aws-config", "aws-credential-types", + "aws-runtime", "aws-sdk-sts", "aws-sigv4", "aws-smithy-runtime-api", + "aws-smithy-types", "aws-types", "bytes", "futures-util", "litellm-auth-types", "litellm-error", "litellm-http", + "litellm-secret-value", + "litellm-secrets", "moka", "reqwest", "rstest", @@ -3406,6 +3469,7 @@ dependencies = [ "sha2 0.10.9", "thiserror 2.0.19", "tokio", + "url", "veil", ] @@ -3418,6 +3482,7 @@ dependencies = [ "litellm-auth-types", "litellm-error", "litellm-secret-value", + "litellm-secrets", "moka", "rstest", "serde_json", @@ -3431,6 +3496,7 @@ dependencies = [ name = "litellm-auth-gcp" version = "0.1.0" dependencies = [ + "axum", "base64 0.22.1", "gcp_auth", "google-cloud-auth", @@ -3440,10 +3506,12 @@ dependencies = [ "litellm-secret-value", "moka", "rstest", + "rustls 0.23.42", "serde", "serde_json", "sha2 0.10.9", "tempfile", + "time", "tokio", "veil", ] @@ -3590,19 +3658,15 @@ dependencies = [ name = "litellm-cache-s3" version = "0.1.0" dependencies = [ - "aws-credential-types", "aws-sdk-s3", - "aws-smithy-runtime-api", "aws-smithy-types", - "aws-types", "futures-util", - "http 1.4.2", "litellm-auth-aws", "litellm-cache", "litellm-cache-testing", "litellm-error", "litellm-http", - "reqwest", + "litellm-s3", "rstest", "serde_json", "tokio", @@ -3654,8 +3718,7 @@ dependencies = [ "strum", "tempfile", "thiserror 2.0.19", - "tracing", - "tracing-subscriber", + "veil", ] [[package]] @@ -3758,8 +3821,10 @@ dependencies = [ "http-body-util", "litellm-circuit-breaker", "litellm-config", + "litellm-core-utils", "litellm-error", "litellm-gateway-auth", + "litellm-gateway-forward", "litellm-gateway-inference", "litellm-gateway-mcp", "litellm-gateway-ui", @@ -3767,11 +3832,15 @@ dependencies = [ "litellm-inference", "litellm-llms", "litellm-redis-client", + "litellm-redis-connect", + "litellm-redis-resolve", "litellm-redis-testing", + "litellm-redis-types", "litellm-secret-value", "litellm-secrets", "litellm-tracing", "rstest", + "rustls 0.23.42", "serde", "serde_json", "tempfile", @@ -3782,6 +3851,7 @@ dependencies = [ "tower-sessions-moka-store", "tracing", "uuid", + "veil", ] [[package]] @@ -3806,6 +3876,29 @@ dependencies = [ "veil", ] +[[package]] +name = "litellm-gateway-forward" +version = "0.1.0" +dependencies = [ + "axum", + "bytes", + "envy", + "futures-util", + "http-body-util", + "httparse", + "hyper 1.12.0", + "hyper-util", + "proptest", + "rstest", + "serde", + "thiserror 2.0.19", + "tokio", + "tokio-tungstenite", + "tokio-util", + "tower-service", + "tracing", +] + [[package]] name = "litellm-gateway-inference" version = "0.1.0" @@ -3920,6 +4013,7 @@ dependencies = [ "litellm-coroutine", "litellm-secret-value", "rstest", + "serde", "serde_json", "thiserror 2.0.19", "tokio", @@ -4062,9 +4156,12 @@ name = "litellm-inference-messages" version = "0.1.0" dependencies = [ "bytes", + "flate2", "futures-util", + "http 1.4.2", "litellm-auth", "litellm-auth-aws", + "litellm-cache", "litellm-cache-memory", "litellm-cache-response", "litellm-core-utils", @@ -4085,6 +4182,7 @@ dependencies = [ "rstest", "serde", "serde_json", + "thiserror 2.0.19", "tokio", "tracing", "wiremock", @@ -4199,6 +4297,7 @@ dependencies = [ "base64 0.22.1", "bytes", "data-url", + "fancy-regex 0.19.2", "futures-util", "litellm-auth", "litellm-auth-aws", @@ -4214,6 +4313,7 @@ dependencies = [ "litellm-router-types", "litellm-secret-value", "litellm-secrets", + "litellm-tracing", "reqwest", "rstest", "serde", @@ -4250,12 +4350,14 @@ dependencies = [ "jsonschema", "litellm-cost", "litellm-error", + "litellm-tracing", "macro_rules_attribute", "rstest", "schemars 1.2.2", "serde", "serde_json", "serde_with", + "sha1 0.10.7", "strum", "thiserror 2.0.19", "time", @@ -4265,9 +4367,13 @@ dependencies = [ name = "litellm-model-catalog-refresh" version = "0.1.0" dependencies = [ + "aws-credential-types", + "backon", + "bytes", "litellm-error", "litellm-http", - "litellm-model-catalog", + "litellm-s3", + "litellm-tracing", "reqwest", "rstest", "thiserror 2.0.19", @@ -4279,16 +4385,19 @@ dependencies = [ name = "litellm-python-bridge" version = "0.1.0" dependencies = [ + "async-trait", "aws-sdk-secretsmanager", "bytes", "criterion", "fancy-regex 0.19.2", "futures-util", + "indexmap 2.14.0", "litellm-auth", "litellm-auth-aws", "litellm-cache", "litellm-cache-response", "litellm-callbacks-legacy-python", + "litellm-circuit-breaker", "litellm-core-utils", "litellm-error", "litellm-host", @@ -4303,6 +4412,14 @@ dependencies = [ "litellm-llms", "litellm-llms-types", "litellm-model-catalog", + "litellm-model-catalog-refresh", + "litellm-python-compat", + "litellm-redis-client", + "litellm-redis-connect", + "litellm-redis-resolve", + "litellm-redis-store", + "litellm-redis-types", + "litellm-router-core", "litellm-router-types", "litellm-secret-value", "litellm-secrets", @@ -4317,6 +4434,7 @@ dependencies = [ "pyo3-async-runtimes", "reqwest", "rstest", + "rustls 0.23.42", "serde", "serde_json", "serde_with", @@ -4352,23 +4470,110 @@ version = "0.1.0" dependencies = [ "futures-util", "litellm-circuit-breaker", + "litellm-python-compat", "litellm-redis-testing", + "litellm-redis-types", "litellm-secret-value", + "pem", + "pkcs8", "redis", "rstest", + "rustls 0.23.42", + "rustls-native-certs", + "socket2 0.6.5", "thiserror 2.0.19", "tokio", + "tracing", +] + +[[package]] +name = "litellm-redis-connect" +version = "0.1.0" +dependencies = [ + "futures-util", + "litellm-auth-aws", + "litellm-auth-azure", + "litellm-auth-gcp", + "litellm-auth-types", + "litellm-circuit-breaker", + "litellm-core-utils", + "litellm-error", + "litellm-http", + "litellm-redis-client", + "litellm-redis-resolve", + "litellm-redis-testing", + "litellm-redis-types", + "litellm-secret-value", + "rstest", + "rustls 0.23.42", + "serde", + "serde_json", + "strum", + "thiserror 2.0.19", + "time", + "tokio", + "tracing", + "url", + "veil", +] + +[[package]] +name = "litellm-redis-resolve" +version = "0.1.0" +dependencies = [ + "litellm-python-compat", + "litellm-redis-types", + "litellm-secret-value", + "rstest", + "serde", + "serde_json", + "strum", + "thiserror 2.0.19", + "tracing", + "url", +] + +[[package]] +name = "litellm-redis-store" +version = "0.1.0" +dependencies = [ + "futures-util", + "litellm-cache", + "litellm-cache-memory", + "litellm-circuit-breaker", + "litellm-python-compat", + "litellm-redis-client", + "litellm-redis-testing", + "litellm-secret-value", + "litellm-tracing", + "rstest", + "strum", + "thiserror 2.0.19", + "tokio", + "tracing", ] [[package]] name = "litellm-redis-testing" version = "0.1.0" dependencies = [ + "pkcs8", + "rcgen", + "tempfile", "testcontainers-modules", "thiserror 2.0.19", "tokio", ] +[[package]] +name = "litellm-redis-types" +version = "0.1.0" +dependencies = [ + "litellm-secret-value", + "rstest", + "url", +] + [[package]] name = "litellm-router" version = "0.1.0" @@ -4384,6 +4589,24 @@ dependencies = [ [[package]] name = "litellm-router-core" version = "0.1.0" +dependencies = [ + "arc-swap", + "async-trait", + "jiff", + "litellm-cache", + "litellm-router-types", + "parking_lot", + "rand 0.8.7", + "rstest", + "serde", + "serde_json", + "serde_with", + "sha2 0.10.9", + "strum", + "thiserror 2.0.19", + "tokio", + "tracing", +] [[package]] name = "litellm-router-types" @@ -4397,6 +4620,23 @@ dependencies = [ "serde_with", ] +[[package]] +name = "litellm-s3" +version = "0.1.0" +dependencies = [ + "aws-config", + "aws-credential-types", + "aws-sdk-s3", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "http 1.4.2", + "litellm-http", + "reqwest", + "rstest", + "thiserror 2.0.19", +] + [[package]] name = "litellm-secret-value" version = "0.1.0" @@ -5233,6 +5473,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + [[package]] name = "pem" version = "4.0.0" @@ -5343,6 +5593,22 @@ dependencies = [ "spki", ] +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "des", + "pbkdf2", + "scrypt", + "sha2 0.10.9", + "spki", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -5350,6 +5616,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der", + "pkcs5", + "rand_core 0.6.4", "spki", ] @@ -5537,6 +5805,7 @@ version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ + "indexmap 2.14.0", "libc", "once_cell", "portable-atomic", @@ -6002,7 +6271,7 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-rustls 0.27.9", "hyper-util", "js-sys", @@ -6358,6 +6627,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -6421,6 +6699,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + [[package]] name = "sct" version = "0.7.1" @@ -7483,6 +7772,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] @@ -7562,7 +7852,7 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-timeout", "hyper-util", "percent-encoding", @@ -8138,9 +8428,9 @@ dependencies = [ [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -8522,7 +8812,7 @@ dependencies = [ "futures", "http 1.4.2", "http-body-util", - "hyper 1.10.1", + "hyper 1.12.0", "hyper-util", "log", "once_cell", diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 943e8576ae6..a00a8ec2446 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -13,6 +13,7 @@ litellm-config = { path = "crates/config" } litellm-error = { path = "crates/error" } litellm-router = { path = "crates/router" } litellm-router-types = { path = "crates/router-types" } +litellm-router-core = { path = "crates/router-core" } litellm-tracing = { path = "crates/tracing" } litellm-spend-clickhouse = { path = "crates/spend-clickhouse" } litellm-storage-clickhouse = { path = "crates/storage-clickhouse" } @@ -25,6 +26,7 @@ litellm-inference-ocr = { path = "crates/inference-ocr" } litellm-inference-testing = { path = "crates/inference-testing" } litellm-gateway-mcp = { path = "crates/gateway-mcp" } litellm-gateway = { path = "crates/gateway" } +litellm-gateway-forward = { path = "crates/gateway-forward" } litellm-gateway-inference = { path = "crates/gateway-inference" } litellm-gateway-auth = { path = "crates/gateway-auth" } litellm-gateway-management = { path = "crates/gateway-management" } @@ -58,6 +60,7 @@ litellm-cache = { path = "crates/cache" } litellm-cache-azure-blob = { path = "crates/cache-azure-blob" } litellm-cache-memory = { path = "crates/cache-memory" } litellm-cache-s3 = { path = "crates/cache-s3" } +litellm-s3 = { path = "crates/s3" } litellm-cache-gcs = { path = "crates/cache-gcs" } litellm-cache-disk = { path = "crates/cache-disk" } litellm-cache-response = { path = "crates/cache-response" } @@ -71,17 +74,25 @@ litellm-host-python = { path = "crates/host-python" } litellm-python-compat = { path = "crates/python-compat" } litellm-circuit-breaker = { path = "crates/circuit-breaker" } litellm-redis-client = { path = "crates/redis-client" } +litellm-redis-connect = { path = "crates/redis-connect" } +litellm-redis-store = { path = "crates/redis-store" } litellm-redis-testing = { path = "crates/redis-testing" } +litellm-redis-resolve = { path = "crates/redis-resolve" } +litellm-redis-types = { path = "crates/redis-types" } litellm-cost = { path = "crates/cost" } litellm-model-catalog = { path = "crates/model-catalog" } litellm-model-catalog-refresh = { path = "crates/model-catalog-refresh" } +arc-swap = "1.9.2" +async-trait = "0.1.91" askama = { version = "0.16.1", default-features = false, features = ["derive", "std"] } tracing = "0.1" axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "multipart"] } axum-login = "0.18.0" tower-sessions = { version = "0.14.0", features = ["memory-store"] } bytes = "1" +backon = { version = "1.6.0", default-features = false, features = ["std", "tokio-sleep"] } +aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rt-tokio"] } http = "1" google-cloud-auth = { version = "1.16.0", default-features = false } jsonwebtoken = { version = "11.1.0", default-features = false } @@ -150,3 +161,9 @@ panic = "unwind" debug = false incremental = false strip = "symbols" + +[profile.ci] +inherits = "release" +opt-level = 0 +lto = false +codegen-units = 256 diff --git a/litellm-rust/crates/auth-aws/Cargo.toml b/litellm-rust/crates/auth-aws/Cargo.toml index e4578777a08..c8ed7bb0f14 100644 --- a/litellm-rust/crates/auth-aws/Cargo.toml +++ b/litellm-rust/crates/auth-aws/Cargo.toml @@ -11,14 +11,16 @@ litellm-error.workspace = true litellm-http.workspace = true reqwest.workspace = true -moka = { workspace = true, features = ["sync"] } +moka.workspace = true +litellm-secret-value.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } sha2.workspace = true thiserror.workspace = true +url.workspace = true veil.workspace = true -aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"] } +aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process"] } aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] } aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"] } aws-sigv4 = "1.5.1" @@ -26,6 +28,9 @@ aws-types = "1.4.0" aws-smithy-runtime-api = "1.13.0" [dev-dependencies] +litellm-secrets.workspace = true +aws-smithy-types = "1.6.4" +aws-runtime = { version = "1.9.4", default-features = false } rstest.workspace = true litellm-http = { workspace = true, features = ["test-support"] } bytes.workspace = true diff --git a/litellm-rust/crates/auth-aws/src/aws.rs b/litellm-rust/crates/auth-aws/src/aws.rs index 81aa4b068bc..e6e32754b4f 100644 --- a/litellm-rust/crates/auth-aws/src/aws.rs +++ b/litellm-rust/crates/auth-aws/src/aws.rs @@ -1,13 +1,16 @@ use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; -use moka::sync::Cache; -use sha2::{Digest, Sha256}; +use aws_credential_types::provider::{SharedCredentialsProvider, error::CredentialsError}; +use litellm_secret_value::SecretValue; +use moka::{Expiry, future::Cache}; +use std::{future::Future, pin::Pin, sync::Arc}; use aws_credential_types::Credentials; use aws_credential_types::provider::ProvideCredentials; use aws_sigv4::http_request::{ - SignableBody, SignableRequest, SigningParams, SigningSettings, sign, + SignableBody, SignableRequest, SignatureLocation, SigningInstructions, SigningParams, + SigningSettings, sign, }; use aws_sigv4::sign::v4; use aws_smithy_runtime_api::client::identity::Identity; @@ -23,72 +26,106 @@ use super::constants::{ }; use crate::error::{CredentialFlow, Error, SigningStage}; -const STATIC_CREDENTIALS_TTL: Duration = Duration::from_secs(3600 - 60); -const AMBIENT_CREDENTIALS_TTL: Duration = Duration::from_secs(600); +const CREDENTIAL_CACHE_TTL: Duration = Duration::from_secs(600); +const EXPIRY_BUFFER: Duration = Duration::from_secs(60); #[derive(Clone)] pub struct AwsAuthService { - static_credentials: Cache, - ambient_credentials: Cache, + providers: Cache, + credentials: Cache, + loader: Arc, } impl Default for AwsAuthService { fn default() -> Self { + Self::new(Arc::new(NativeProviderLoader)) + } +} + +impl AwsAuthService { + fn new(loader: Arc) -> Self { Self { - static_credentials: Cache::builder() + providers: Cache::builder().max_capacity(200).build(), + credentials: Cache::builder() .max_capacity(200) - .time_to_live(STATIC_CREDENTIALS_TTL) - .build(), - ambient_credentials: Cache::builder() - .max_capacity(200) - .time_to_live(AMBIENT_CREDENTIALS_TTL) + .expire_after(CredentialExpiry) .build(), + loader, } } } -fn credential_cache_ttl(flow: &AwsAuthFlow) -> Option { - match flow { - AwsAuthFlow::StaticKeys { .. } => Some(STATIC_CREDENTIALS_TTL), - AwsAuthFlow::DefaultChain => Some(AMBIENT_CREDENTIALS_TTL), - AwsAuthFlow::WebIdentity { .. } - | AwsAuthFlow::AssumeRole { .. } - | AwsAuthFlow::Profile { .. } - | AwsAuthFlow::SessionToken { .. } => None, +struct CredentialExpiry; + +impl Expiry for CredentialExpiry { + fn expire_after_create( + &self, + _: &AwsAuthConfig, + credentials: &Credentials, + _: std::time::Instant, + ) -> Option { + Some(credentials.expiry().map_or(CREDENTIAL_CACHE_TTL, |expiry| { + expiry + .duration_since(SystemTime::now()) + .unwrap_or_default() + .saturating_sub(EXPIRY_BUFFER) + .min(CREDENTIAL_CACHE_TTL) + })) } } -#[derive(Clone, Debug, Default, PartialEq, Eq)] +type ProviderFuture<'a> = + Pin> + Send + 'a>>; + +trait ProviderLoader: Send + Sync { + fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_>; +} + +struct NativeProviderLoader; + +impl ProviderLoader for NativeProviderLoader { + fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_> { + Box::pin(build_provider(config, flow)) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] pub struct AwsAuthConfig { - pub access_key_id: Option, - pub secret_access_key: Option, - pub session_token: Option, + pub access_key_id: Option, + pub secret_access_key: Option, + pub session_token: Option, pub region_name: Option, pub session_name: Option, pub profile_name: Option, pub role_name: Option, - pub web_identity_token: Option, + pub web_identity_token: Option, pub sts_endpoint: Option, - pub external_id: Option, + pub external_id: Option, } impl AwsAuthConfig { fn with_environment(self, env_lookup: &(dyn Fn(&str) -> Option + Sync)) -> Self { Self { - access_key_id: self.access_key_id.or_else(|| env_lookup(AWS_ACCESS_KEY_ID)), + access_key_id: self + .access_key_id + .or_else(|| env_lookup(AWS_ACCESS_KEY_ID).map(SecretValue::new)), secret_access_key: self .secret_access_key - .or_else(|| env_lookup(AWS_SECRET_ACCESS_KEY)), - session_token: self.session_token.or_else(|| env_lookup(AWS_SESSION_TOKEN)), + .or_else(|| env_lookup(AWS_SECRET_ACCESS_KEY).map(SecretValue::new)), + session_token: self + .session_token + .or_else(|| env_lookup(AWS_SESSION_TOKEN).map(SecretValue::new)), region_name: self.region_name.or_else(|| env_lookup(AWS_REGION_NAME)), session_name: self.session_name.or_else(|| env_lookup(AWS_SESSION_NAME)), profile_name: self.profile_name.or_else(|| env_lookup(AWS_PROFILE_NAME)), role_name: self.role_name.or_else(|| env_lookup(AWS_ROLE_NAME)), web_identity_token: self .web_identity_token - .or_else(|| env_lookup(AWS_WEB_IDENTITY_TOKEN)), + .or_else(|| env_lookup(AWS_WEB_IDENTITY_TOKEN).map(SecretValue::new)), sts_endpoint: self.sts_endpoint.or_else(|| env_lookup(AWS_STS_ENDPOINT)), - external_id: self.external_id.or_else(|| env_lookup(AWS_EXTERNAL_ID)), + external_id: self + .external_id + .or_else(|| env_lookup(AWS_EXTERNAL_ID).map(SecretValue::new)), } } } @@ -96,52 +133,31 @@ impl AwsAuthConfig { #[derive(Clone, Debug, PartialEq, Eq)] pub enum AwsAuthFlow { WebIdentity { - token: String, + token: SecretValue, role: String, session_name: String, }, AssumeRole { role: String, session_name: Option, + web_identity_role: Option, }, Profile { name: String, }, SessionToken { - access_key_id: String, - secret_access_key: String, - session_token: String, + access_key_id: SecretValue, + secret_access_key: SecretValue, + session_token: SecretValue, }, StaticKeys { - access_key_id: String, - secret_access_key: String, + access_key_id: SecretValue, + secret_access_key: SecretValue, region_name: String, }, DefaultChain, } -fn cache_key(config: &AwsAuthConfig, flow: &AwsAuthFlow) -> String { - let mut hasher = Sha256::new(); - hasher.update(format!("{config:?}:{flow:?}")); - format!("{:x}", hasher.finalize()) -} - -impl AwsAuthService { - fn get_cached_credentials(&self, key: &str) -> Option { - self.static_credentials - .get(key) - .or_else(|| self.ambient_credentials.get(key)) - } - - fn set_cached_credentials(&self, key: String, credentials: Credentials, ttl: Duration) { - if ttl == STATIC_CREDENTIALS_TTL { - self.static_credentials.insert(key, credentials); - } else { - self.ambient_credentials.insert(key, credentials); - } - } -} - fn role_identity(arn: &str) -> Option<(&str, &str, &str)> { let mut parts = arn.splitn(6, ':'); let ("arn", partition, _, _, account, resource) = ( @@ -170,6 +186,9 @@ pub fn classify_auth( config: AwsAuthConfig, env_lookup: &(dyn Fn(&str) -> Option + Sync), ) -> AwsAuthFlow { + let web_identity_role = env_lookup(AWS_WEB_IDENTITY_TOKEN_FILE) + .filter(|token_file| !token_file.is_empty()) + .and_then(|_| env_lookup(AWS_ROLE_ARN)); let config = config.with_environment(env_lookup); if let (Some(token), Some(role), Some(session_name)) = ( config.web_identity_token.clone(), @@ -186,6 +205,7 @@ pub fn classify_auth( return AwsAuthFlow::AssumeRole { role, session_name: config.session_name.clone(), + web_identity_role, }; } if let Some(name) = config.profile_name { @@ -216,6 +236,30 @@ pub fn classify_auth( AwsAuthFlow::DefaultChain } +async fn profile_provider( + name: &str, + region: Option, + config: aws_config::provider_config::ProviderConfig, + builder: aws_config::profile::credentials::Builder, +) -> Result { + let region = match region { + Some(region) => Some(aws_types::region::Region::new(region)), + None => { + aws_config::default_provider::region::DefaultRegionChain::builder() + .profile_name(name) + .build() + .region() + .await + } + }; + Ok(SharedCredentialsProvider::new( + builder + .configure(&config.with_region(region)) + .profile_name(name) + .build(), + )) +} + impl AwsAuthService { pub async fn resolve_credentials( &self, @@ -224,183 +268,240 @@ impl AwsAuthService { ) -> Result { let resolved = config.clone().with_environment(env_lookup); let flow = classify_auth(config, env_lookup); - match flow { - AwsAuthFlow::SessionToken { - access_key_id, - secret_access_key, - session_token, - } => Ok(Credentials::new( - access_key_id, - secret_access_key, - Some(session_token), - None, - "litellm-static-session", - )), - AwsAuthFlow::StaticKeys { - access_key_id, - secret_access_key, - region_name, - } => { - let flow = AwsAuthFlow::StaticKeys { - access_key_id: access_key_id.clone(), - secret_access_key: secret_access_key.clone(), - region_name, - }; - let key = cache_key(&resolved, &flow); - if let Some(credentials) = self.get_cached_credentials(&key) { - return Ok(credentials); - } - let credentials = Credentials::new( - access_key_id, - secret_access_key, - None, - None, - "litellm-static", - ); - self.set_cached_credentials( - key, - credentials.clone(), - credential_cache_ttl(&flow).unwrap_or(STATIC_CREDENTIALS_TTL), - ); - Ok(credentials) - } - AwsAuthFlow::Profile { name } => { - let provider = aws_config::profile::ProfileFileCredentialsProvider::builder() - .profile_name(name) - .build(); + self.fresh_credentials(&resolved, &flow).await + } + + /// The credentials botocore's default chain finds in `region`, without LiteLLM's Bedrock settings + /// (`AWS_ROLE_NAME`, `AWS_PROFILE_NAME`, `AWS_SESSION_NAME`) that `resolve_credentials` also reads. + pub async fn default_chain_credentials(&self, region: &str) -> Result { + let config = AwsAuthConfig { + region_name: Some(region.to_owned()), + ..Default::default() + }; + self.fresh_credentials(&config, &AwsAuthFlow::DefaultChain) + .await + } + + async fn fresh_credentials( + &self, + resolved: &AwsAuthConfig, + flow: &AwsAuthFlow, + ) -> Result { + let cached = self.cached_credentials(resolved, flow).await?; + if fresh(&cached) { + return Ok(cached); + } + self.credentials.invalidate(resolved).await; + let refetched = self.cached_credentials(resolved, flow).await?; + if fresh(&refetched) { + return Ok(refetched); + } + self.credentials.invalidate(resolved).await; + Err(Error::ExpiredCredentials) + } + + async fn cached_credentials( + &self, + resolved: &AwsAuthConfig, + flow: &AwsAuthFlow, + ) -> Result { + self.credentials + .try_get_with(resolved.clone(), async { + let provider = self + .providers + .try_get_with( + resolved.clone(), + self.loader.load(resolved.clone(), flow.clone()), + ) + .await + .map_err(|error| (*error).clone())?; provider .provide_credentials() .await - .map_err(Error::credentials(CredentialFlow::Profile)) - } - AwsAuthFlow::AssumeRole { role, session_name } => { - if is_already_running_as_role(&role, &resolved).await? { - let ambient_flow = AwsAuthFlow::DefaultChain; - let key = cache_key(&resolved, &ambient_flow); - if let Some(credentials) = self.get_cached_credentials(&key) { - return Ok(credentials); - } - let provider = - aws_config::default_provider::credentials::DefaultCredentialsChain::builder() - .build() - .await; - let credentials = provider - .provide_credentials() - .await - .map_err(Error::credentials(CredentialFlow::DefaultChain))?; - self.set_cached_credentials( - key, - credentials.clone(), - credential_cache_ttl(&ambient_flow).unwrap_or(AMBIENT_CREDENTIALS_TTL), - ); - return Ok(credentials); - } - let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); - if let Some(region) = resolved.region_name.clone() { - loader = loader.region(aws_types::region::Region::new(region)); - } - if let Some(endpoint) = resolved.sts_endpoint.clone() { - loader = loader.endpoint_url(endpoint); - } - if let (Some(access_key_id), Some(secret_access_key)) = - (resolved.access_key_id, resolved.secret_access_key) - { - loader = loader.credentials_provider(Credentials::new( - access_key_id, - secret_access_key, - resolved.session_token, - None, - "litellm-role-source", - )); - } - let sdk_config = loader.load().await; - let builder = aws_config::sts::AssumeRoleProvider::builder(role); - let builder = match session_name { - Some(name) => builder.session_name(name), - None => builder.session_name(default_session_name()), - }; - let builder = match resolved.external_id { - Some(id) => builder.external_id(id), - None => builder, - }; - let provider = builder.configure(&sdk_config).build().await; - provider - .provide_credentials() - .await - .map_err(Error::credentials(CredentialFlow::AssumeRole)) - } - AwsAuthFlow::WebIdentity { - token, - role, - session_name, - } => { - let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); - if let Some(region) = resolved.region_name { - loader = loader.region(aws_types::region::Region::new(region)); - } - if let Some(endpoint) = resolved.sts_endpoint { - loader = loader.endpoint_url(endpoint); - } - let sdk_config = loader.load().await; - let client = aws_sdk_sts::Client::new(&sdk_config); - let response = client - .assume_role_with_web_identity() - .role_arn(role) - .role_session_name(session_name) - .web_identity_token(token) - .send() - .await - .map_err(Error::credentials(CredentialFlow::WebIdentity))?; - let credentials = response - .credentials() - .ok_or(Error::MissingWebIdentityCredentials)?; - let expiration = - SystemTime::try_from(*credentials.expiration()).map_err(|source| { - Error::WebIdentityExpiration(litellm_error::Source::new(source)) - })?; - Ok(Credentials::new( - credentials.access_key_id(), - credentials.secret_access_key(), - Some(credentials.session_token().to_string()), - Some(expiration), - "litellm-web-identity", - )) - } - AwsAuthFlow::DefaultChain => { - let key = cache_key(&resolved, &AwsAuthFlow::DefaultChain); - if let Some(credentials) = self.get_cached_credentials(&key) { - return Ok(credentials); - } - let provider = - aws_config::default_provider::credentials::DefaultCredentialsChain::builder() - .build() - .await; - let credentials = provider - .provide_credentials() - .await - .map_err(Error::credentials(CredentialFlow::DefaultChain))?; - self.set_cached_credentials( - key, - credentials.clone(), - credential_cache_ttl(&AwsAuthFlow::DefaultChain) - .unwrap_or(AMBIENT_CREDENTIALS_TTL), - ); - Ok(credentials) + .map_err(Error::credentials(flow.credential_flow())) + }) + .await + .map_err(|error| (*error).clone()) + } +} + +fn fresh(credentials: &Credentials) -> bool { + credentials + .expiry() + .is_none_or(|expiry| expiry > SystemTime::now() + EXPIRY_BUFFER) +} + +impl AwsAuthFlow { + fn credential_flow(&self) -> CredentialFlow { + match self { + Self::Profile { .. } => CredentialFlow::Profile, + Self::AssumeRole { .. } => CredentialFlow::AssumeRole, + Self::WebIdentity { .. } => CredentialFlow::WebIdentity, + Self::DefaultChain | Self::StaticKeys { .. } | Self::SessionToken { .. } => { + CredentialFlow::DefaultChain } } } } -async fn is_already_running_as_role(role: &str, config: &AwsAuthConfig) -> Result { +async fn build_provider( + resolved: AwsAuthConfig, + flow: AwsAuthFlow, +) -> Result { + match flow { + AwsAuthFlow::SessionToken { + access_key_id, + secret_access_key, + session_token, + } => Ok(SharedCredentialsProvider::new(Credentials::new( + access_key_id.expose(), + secret_access_key.expose(), + Some(session_token.expose().to_owned()), + None, + "litellm-static-session", + ))), + AwsAuthFlow::StaticKeys { + access_key_id, + secret_access_key, + .. + } => Ok(SharedCredentialsProvider::new(Credentials::new( + access_key_id.expose(), + secret_access_key.expose(), + None, + None, + "litellm-static", + ))), + AwsAuthFlow::Profile { name } => { + profile_provider( + &name, + resolved.region_name, + aws_config::provider_config::ProviderConfig::without_region(), + aws_config::profile::ProfileFileCredentialsProvider::builder(), + ) + .await + } + AwsAuthFlow::DefaultChain => { + let builder = + aws_config::default_provider::credentials::DefaultCredentialsChain::builder(); + let builder = match resolved.region_name { + Some(region) => builder.region(aws_types::region::Region::new(region)), + None => builder, + }; + Ok(SharedCredentialsProvider::new(builder.build().await)) + } + AwsAuthFlow::AssumeRole { + role, + session_name, + web_identity_role, + } => { + if is_already_running_as_role(&role, web_identity_role.as_deref(), &resolved).await? { + return Ok(SharedCredentialsProvider::new( + aws_config::default_provider::credentials::DefaultCredentialsChain::builder() + .build() + .await, + )); + } + let sdk_config = sdk_config(resolved.clone()).await; + let builder = aws_config::sts::AssumeRoleProvider::builder(role) + .session_name(session_name.unwrap_or_else(default_session_name)); + let builder = match resolved.external_id { + Some(id) => builder.external_id(id.expose()), + None => builder, + }; + Ok(SharedCredentialsProvider::new( + builder.configure(&sdk_config).build().await, + )) + } + AwsAuthFlow::WebIdentity { + token, + role, + session_name, + } => Ok(SharedCredentialsProvider::new(WebIdentityProvider { + client: aws_sdk_sts::Client::new(&sdk_config(resolved).await), + token, + role, + session_name, + })), + } +} + +async fn sdk_config(config: AwsAuthConfig) -> aws_types::SdkConfig { + let loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); + let loader = match config.region_name { + Some(region) => loader.region(aws_types::region::Region::new(region)), + None => loader, + }; + let loader = match config.sts_endpoint { + Some(endpoint) => loader.endpoint_url(endpoint), + None => loader, + }; + let loader = match (config.access_key_id, config.secret_access_key) { + (Some(key), Some(secret)) => loader.credentials_provider(Credentials::new( + key.expose(), + secret.expose(), + config.session_token.map(|token| token.expose().to_owned()), + None, + "litellm-role-source", + )), + _ => loader, + }; + loader.load().await +} + +#[derive(Debug)] +struct WebIdentityProvider { + client: aws_sdk_sts::Client, + token: SecretValue, + role: String, + session_name: String, +} + +impl ProvideCredentials for WebIdentityProvider { + fn provide_credentials<'a>( + &'a self, + ) -> aws_credential_types::provider::future::ProvideCredentials<'a> + where + Self: 'a, + { + aws_credential_types::provider::future::ProvideCredentials::new(async { + let response = self + .client + .assume_role_with_web_identity() + .role_arn(&self.role) + .role_session_name(&self.session_name) + .web_identity_token(self.token.expose()) + .send() + .await + .map_err(CredentialsError::provider_error)?; + let credentials = response.credentials().ok_or_else(|| { + CredentialsError::provider_error(Error::MissingWebIdentityCredentials) + })?; + let expiration = SystemTime::try_from(*credentials.expiration()) + .map_err(CredentialsError::provider_error)?; + Ok(Credentials::new( + credentials.access_key_id(), + credentials.secret_access_key(), + Some(credentials.session_token().to_owned()), + Some(expiration), + "litellm-web-identity", + )) + }) + } +} + +async fn is_already_running_as_role( + role: &str, + web_identity_role: Option<&str>, + config: &AwsAuthConfig, +) -> Result { + if config.access_key_id.is_some() || config.secret_access_key.is_some() { + return Ok(false); + } if role_identity(role).is_none() { return Ok(false); } - if let (Ok(current_role), Ok(token_file)) = ( - std::env::var(AWS_ROLE_ARN), - std::env::var(AWS_WEB_IDENTITY_TOKEN_FILE), - ) && !token_file.is_empty() - { - return Ok(same_role_arns(role, ¤t_role)); + if let Some(current_role) = web_identity_role { + return Ok(same_role_arns(role, current_role)); } let mut loader = aws_config::defaults(aws_config::BehaviorVersion::latest()); @@ -473,16 +574,6 @@ pub fn sign_headers( headers: &HeaderMap, scope: SigningScope<'_>, ) -> Result { - let identity: Identity = scope.credentials.clone().into(); - let params = v4::SigningParams::builder() - .identity(&identity) - .region(scope.region) - .name(scope.service) - .time(scope.time) - .settings(SigningSettings::default()) - .build() - .map(SigningParams::from) - .map_err(Error::signing(SigningStage::Parameters))?; let header_refs = headers .iter() .map(|(name, value)| { @@ -494,16 +585,14 @@ pub fn sign_headers( }) }) .collect::, _>>()?; - let request = SignableRequest::new( + let instructions = sign_request( method, url, header_refs.into_iter(), - SignableBody::Bytes(body), - ) - .map_err(Error::signing(SigningStage::Request))?; - let (instructions, _) = sign(request, ¶ms) - .map_err(Error::signing(SigningStage::Signature))? - .into_parts(); + body, + SigningSettings::default(), + scope, + )?; instructions .headers() .map(|(name, value)| { @@ -516,6 +605,60 @@ pub fn sign_headers( .collect() } +pub fn presign_url( + url: &str, + expires_in: Duration, + scope: SigningScope<'_>, +) -> Result { + let mut signed = url::Url::parse(url).map_err(Error::signing(SigningStage::Request))?; + let mut settings = SigningSettings::default(); + settings.signature_location = SignatureLocation::QueryParams; + settings.expires_in = Some(expires_in); + let instructions = sign_request( + "GET", + signed.as_str(), + std::iter::empty(), + &[], + settings, + scope, + )?; + let (signature, auth): (Vec<_>, Vec<_>) = instructions + .params() + .iter() + .map(|(name, value)| (*name, value.as_ref())) + .partition(|(name, _)| *name == "X-Amz-Signature"); + signed + .query_pairs_mut() + .extend_pairs(auth.into_iter().chain(signature)); + Ok(signed.into()) +} + +fn sign_request<'a>( + method: &'a str, + url: &'a str, + headers: impl Iterator, + body: &'a [u8], + settings: SigningSettings, + scope: SigningScope<'_>, +) -> Result { + let identity: Identity = scope.credentials.clone().into(); + let params = v4::SigningParams::builder() + .identity(&identity) + .region(scope.region) + .name(scope.service) + .time(scope.time) + .settings(settings) + .build() + .map(SigningParams::from) + .map_err(Error::signing(SigningStage::Parameters))?; + let request = SignableRequest::new(method, url, headers, SignableBody::Bytes(body)) + .map_err(Error::signing(SigningStage::Request))?; + let (instructions, _) = sign(request, ¶ms) + .map_err(Error::signing(SigningStage::Signature))? + .into_parts(); + Ok(instructions) +} + /// Model-id and region parsing shared by every Bedrock route. pub fn bedrock_model_id_and_region(model: &str) -> (String, Option) { let mut stripped = model; @@ -586,16 +729,26 @@ pub fn aws_auth_config( env_lookup: &dyn Fn(&str) -> Option, ) -> AwsAuthConfig { AwsAuthConfig { - access_key_id: params.resolve(&AwsParams::ACCESS_KEY_ID, env_lookup), - secret_access_key: params.resolve(&AwsParams::SECRET_ACCESS_KEY, env_lookup), - session_token: params.resolve(&AwsParams::SESSION_TOKEN, env_lookup), + access_key_id: params + .resolve(&AwsParams::ACCESS_KEY_ID, env_lookup) + .map(SecretValue::new), + secret_access_key: params + .resolve(&AwsParams::SECRET_ACCESS_KEY, env_lookup) + .map(SecretValue::new), + session_token: params + .resolve(&AwsParams::SESSION_TOKEN, env_lookup) + .map(SecretValue::new), region_name: params.resolve(&AwsParams::REGION, env_lookup), session_name: params.resolve(&AwsParams::SESSION_NAME, env_lookup), profile_name: params.resolve(&AwsParams::PROFILE_NAME, env_lookup), role_name: params.resolve(&AwsParams::ROLE_NAME, env_lookup), - web_identity_token: params.resolve(&AwsParams::WEB_IDENTITY_TOKEN, env_lookup), + web_identity_token: params + .resolve(&AwsParams::WEB_IDENTITY_TOKEN, env_lookup) + .map(SecretValue::new), sts_endpoint: params.resolve(&AwsParams::STS_ENDPOINT, env_lookup), - external_id: params.resolve(&AwsParams::EXTERNAL_ID, env_lookup), + external_id: params + .resolve(&AwsParams::EXTERNAL_ID, env_lookup) + .map(SecretValue::new), } } @@ -660,10 +813,133 @@ mod tests { use crate::constants::{AWS_REGION, BEDROCK_SERVICE}; use reqwest::header::CONTENT_TYPE; + #[cfg(unix)] + #[rstest::fixture] + fn process_profile() -> aws_runtime::env_config::file::EnvConfigFiles { + let credentials = serde_json::json!({ + "Version": 1, + "AccessKeyId": "process-access", + "SecretAccessKey": "process-secret", + "SessionToken": "process-session", + }); + aws_runtime::env_config::file::EnvConfigFiles::builder() + .include_default_config_file(false) + .include_default_credentials_file(false) + .with_contents( + aws_runtime::env_config::file::EnvConfigFileKind::Config, + format!( + "[profile smoke]\ncredential_process = /usr/bin/printf '%s' '{credentials}'\n" + ), + ) + .build() + } + + #[cfg(unix)] + #[rstest::rstest] + #[tokio::test] + async fn profile_process_credentials_can_be_loaded( + process_profile: aws_runtime::env_config::file::EnvConfigFiles, + ) { + let provider = profile_provider( + "smoke", + Some("us-west-2".into()), + aws_config::provider_config::ProviderConfig::without_region(), + aws_config::profile::ProfileFileCredentialsProvider::builder() + .profile_files(process_profile), + ) + .await + .unwrap(); + let credentials = provider.provide_credentials().await.unwrap(); + + assert_eq!(credentials.access_key_id(), "process-access"); + assert_eq!(credentials.secret_access_key(), "process-secret"); + assert_eq!(credentials.session_token(), Some("process-session")); + } + + #[derive(Debug)] + struct StsConnector; + + impl aws_smithy_runtime_api::client::http::HttpConnector for StsConnector { + fn call( + &self, + request: aws_smithy_runtime_api::client::orchestrator::HttpRequest, + ) -> aws_smithy_runtime_api::client::http::HttpConnectorFuture { + assert!(request.uri().contains("sts.us-west-2.")); + aws_smithy_runtime_api::client::http::HttpConnectorFuture::new(async { + Ok( + aws_smithy_runtime_api::client::orchestrator::HttpResponse::new( + 200_u16.try_into().unwrap(), + aws_smithy_types::body::SdkBody::from( + r#" + + role-accessrole-secret + role-session2100-01-01T00:00:00Z + arn:aws:sts::123456789012:assumed-role/smoke/session + smoke:session + smoke"#, + ), + ), + ) + }) + } + } + + #[rstest::rstest] + #[tokio::test] + async fn role_profile_uses_region_when_acquiring_sts_credentials() { + use aws_smithy_runtime_api::client::http::{SharedHttpConnector, http_client_fn}; + let files = aws_runtime::env_config::file::EnvConfigFiles::builder() + .include_default_config_file(false) + .include_default_credentials_file(false) + .with_contents( + aws_runtime::env_config::file::EnvConfigFileKind::Config, + "[profile smoke]\nrole_arn = arn:aws:iam::123456789012:role/smoke\nsource_profile = source\n\n[profile source]\naws_access_key_id = source-access\naws_secret_access_key = source-secret\n", + ).build(); + let config = aws_config::provider_config::ProviderConfig::without_region() + .with_http_client(http_client_fn(|_, _| { + SharedHttpConnector::new(StsConnector) + })); + let credentials = profile_provider( + "smoke", + Some("us-west-2".into()), + config, + aws_config::profile::ProfileFileCredentialsProvider::builder().profile_files(files), + ) + .await + .unwrap(); + let credentials = credentials.provide_credentials().await.unwrap(); + assert_eq!(credentials.access_key_id(), "role-access"); + assert_eq!(credentials.secret_access_key(), "role-secret"); + assert_eq!(credentials.session_token(), Some("role-session")); + assert!(credentials.expiry().is_some()); + } + fn no_env(_: &str) -> Option { None } + #[rstest::rstest] + #[case::access_key(Some("source-access"), None)] + #[case::secret_key(None, Some("source-secret"))] + #[case::source_pair(Some("source-access"), Some("source-secret"))] + #[tokio::test] + async fn explicit_source_keys_cannot_reuse_an_ambient_role( + #[case] access_key: Option<&str>, + #[case] secret_key: Option<&str>, + ) { + let role = "arn:aws:iam::123456789012:role/target"; + let config = AwsAuthConfig { + access_key_id: access_key.map(SecretValue::new), + secret_access_key: secret_key.map(SecretValue::new), + ..Default::default() + }; + assert!( + !is_already_running_as_role(role, Some(role), &config) + .await + .unwrap() + ); + } + #[test] fn secret_names_cover_environment_reads() { let seen = std::sync::Arc::new(std::sync::Mutex::new( @@ -706,7 +982,10 @@ mod tests { }; assert_eq!( - aws_auth_config(¶ms, &env).secret_access_key.as_deref(), + aws_auth_config(¶ms, &env) + .secret_access_key + .as_ref() + .map(SecretValue::expose), expected ); assert_eq!( @@ -831,6 +1110,53 @@ mod tests { ); } + #[rstest::rstest] + #[case::running_with_web_identity("/var/run/token", Some("arn:aws:iam::1:role/current"))] + #[case::empty_token_file("", None)] + #[tokio::test] + async fn assume_role_reads_the_ambient_role_from_the_secret_snapshot( + #[case] token_file: &str, + #[case] expected: Option<&str>, + ) { + use litellm_secrets::source::SecretSource; + + struct WebIdentitySecrets<'a>(&'a str); + + impl SecretSource for WebIdentitySecrets<'_> { + fn get_secret_str<'a>( + &'a self, + name: &'a str, + ) -> futures_util::future::BoxFuture< + 'a, + Result, litellm_secrets::Error>, + > { + Box::pin(async move { + Ok(match name { + AWS_WEB_IDENTITY_TOKEN_FILE => Some(SecretValue::new(self.0)), + AWS_ROLE_ARN => Some(SecretValue::new("arn:aws:iam::1:role/current")), + _ => None, + }) + }) + } + } + + let snapshot = WebIdentitySecrets(token_file) + .resolve(AwsParams::secret_names()) + .await + .unwrap(); + let config = AwsAuthConfig { + role_name: Some("arn:aws:iam::1:role/target".into()), + ..Default::default() + }; + let AwsAuthFlow::AssumeRole { + web_identity_role, .. + } = classify_auth(config, &|name| snapshot.get(name)) + else { + panic!("a role name selects AssumeRole"); + }; + assert_eq!(web_identity_role.as_deref(), expected); + } + #[tokio::test] async fn static_credentials_do_not_use_network() { let credentials = AwsAuthService::default() @@ -849,110 +1175,231 @@ mod tests { assert_eq!(credentials.session_token(), None); } - #[test] - fn cache_policy_matches_python_flows() { - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::StaticKeys { - access_key_id: "ak".into(), - secret_access_key: "sk".into(), - region_name: "us-east-1".into(), - }), - Some(STATIC_CREDENTIALS_TTL) - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::DefaultChain), - Some(AMBIENT_CREDENTIALS_TTL) - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::SessionToken { - access_key_id: "ak".into(), - secret_access_key: "sk".into(), - session_token: "token".into(), - }), - None - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::Profile { - name: "profile".into() - }), - None - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::AssumeRole { - role: "arn:aws:iam::123456789012:role/demo".into(), - session_name: None, - }), - None - ); - assert_eq!( - credential_cache_ttl(&AwsAuthFlow::WebIdentity { - token: "token".into(), - role: "arn:aws:iam::123456789012:role/demo".into(), - session_name: "session".into(), - }), - None - ); + struct CountingLoader { + loads: Arc, + calls: Arc, + short_lifetime: Duration, + short_calls: usize, + } + + impl ProviderLoader for CountingLoader { + fn load(&self, config: AwsAuthConfig, _: AwsAuthFlow) -> ProviderFuture<'_> { + let loads = self.loads.clone(); + let provider = CountingProvider { + calls: self.calls.clone(), + short_lifetime: self.short_lifetime, + short_calls: self.short_calls, + key: config.access_key_id.unwrap(), + }; + Box::pin(async move { + loads.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(SharedCredentialsProvider::new(provider)) + }) + } + } + + #[derive(Debug)] + struct CountingProvider { + calls: Arc, + short_lifetime: Duration, + short_calls: usize, + key: SecretValue, + } + + impl ProvideCredentials for CountingProvider { + fn provide_credentials<'a>( + &'a self, + ) -> aws_credential_types::provider::future::ProvideCredentials<'a> + where + Self: 'a, + { + aws_credential_types::provider::future::ProvideCredentials::new(async move { + tokio::task::yield_now().await; + let call = self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let lifetime = if call < self.short_calls { + self.short_lifetime + } else { + Duration::from_secs(3600) + }; + Ok(Credentials::new( + self.key.expose(), + "credential-value", + Some(format!("session-{call}")), + Some(SystemTime::now() + lifetime), + "counting", + )) + }) + } + } + + #[derive(Default)] + struct RecordingLoader { + loads: std::sync::Mutex>, + } + + impl ProviderLoader for RecordingLoader { + fn load(&self, config: AwsAuthConfig, flow: AwsAuthFlow) -> ProviderFuture<'_> { + self.loads.lock().unwrap().push((config, flow)); + Box::pin(async { + Ok(SharedCredentialsProvider::new(Credentials::new( + "ak", + "sk", + None, + None, + "recording", + ))) + }) + } } #[rstest::rstest] - fn cache_round_trip_preserves_credentials() { - let auth = AwsAuthService::default(); - let key = format!("cache-test-{}", std::process::id()); - let credentials = Credentials::new("cache-ak", "cache-sk", None, None, "test"); - auth.set_cached_credentials(key.clone(), credentials.clone(), STATIC_CREDENTIALS_TTL); + #[tokio::test] + async fn default_chain_credentials_do_not_share_a_configured_role_in_the_same_region() { + let loader = Arc::new(RecordingLoader::default()); + let auth = AwsAuthService::new(loader.clone()); + let bedrock_env = |name: &str| match name { + AWS_ROLE_NAME => Some("arn:aws:iam::123456789012:role/bedrock".to_string()), + AWS_REGION_NAME => Some("us-east-1".to_string()), + _ => None, + }; + auth.resolve_credentials(AwsAuthConfig::default(), &bedrock_env) + .await + .unwrap(); + + auth.default_chain_credentials("us-east-1").await.unwrap(); + auth.default_chain_credentials("us-east-1").await.unwrap(); + + let loads = loader.loads.lock().unwrap(); + assert_eq!(loads.len(), 2); assert_eq!( - auth.get_cached_credentials(&key) - .map(|value| value.access_key_id().to_string()), - Some("cache-ak".to_string()) + loads[1], + ( + AwsAuthConfig { + region_name: Some("us-east-1".to_string()), + ..Default::default() + }, + AwsAuthFlow::DefaultChain + ) ); } #[rstest::rstest] #[tokio::test] - async fn cloned_services_reuse_credentials_but_independent_services_do_not() { - let auth = AwsAuthService::default(); + async fn near_expiry_credentials_are_refreshed_without_rebuilding_the_provider() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let loads = Arc::new(AtomicUsize::new(0)); + let calls = Arc::new(AtomicUsize::new(0)); + let auth = AwsAuthService::new(Arc::new(CountingLoader { + loads: loads.clone(), + calls: calls.clone(), + short_lifetime: Duration::from_secs(30), + short_calls: 1, + })); let config = AwsAuthConfig { - access_key_id: Some("configured-key".into()), - secret_access_key: Some("configured-secret".into()), - region_name: Some("us-east-1".into()), - ..AwsAuthConfig::default() + access_key_id: Some("identity".into()), + ..Default::default() }; - let flow = classify_auth(config.clone(), &no_env); - let cached = Credentials::new("cached-key", "cached-secret", None, None, "test"); - auth.set_cached_credentials( - cache_key(&config, &flow), - cached.clone(), - STATIC_CREDENTIALS_TTL, - ); - - let reused = auth + let first = auth + .resolve_credentials(config.clone(), &no_env) + .await + .unwrap(); + let second = auth .clone() .resolve_credentials(config.clone(), &no_env) .await .unwrap(); - let independent = AwsAuthService::default() - .resolve_credentials(config.clone(), &no_env) - .await - .unwrap(); - let different = AwsAuthConfig { - access_key_id: Some("different-key".into()), - ..config.clone() - }; - let other_identity = auth - .resolve_credentials(different.clone(), &no_env) - .await - .unwrap(); + assert_eq!(first.session_token(), Some("session-1")); + assert_eq!(first.session_token(), second.session_token()); + assert_eq!(loads.load(Ordering::SeqCst), 1); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } - assert_eq!(reused.access_key_id(), cached.access_key_id()); - assert_eq!(reused.secret_access_key(), cached.secret_access_key()); + #[rstest::rstest] + #[case::recovered_by_one_refetch(1, Some("session-1"))] + #[case::still_expired_after_refetch(2, None)] + #[tokio::test] + async fn expired_credentials_are_refetched_once( + #[case] short_calls: usize, + #[case] session: Option<&str>, + ) { + use std::sync::atomic::{AtomicUsize, Ordering}; + let calls = Arc::new(AtomicUsize::new(0)); + let auth = AwsAuthService::new(Arc::new(CountingLoader { + loads: Arc::new(AtomicUsize::new(0)), + calls: calls.clone(), + short_lifetime: Duration::ZERO, + short_calls, + })); + let config = AwsAuthConfig { + access_key_id: Some("identity".into()), + ..Default::default() + }; + let result = auth.resolve_credentials(config, &no_env).await; assert_eq!( - Some(independent.access_key_id()), - config.access_key_id.as_deref() + result.as_ref().ok().and_then(Credentials::session_token), + session ); - assert_eq!( - Some(other_identity.access_key_id()), - different.access_key_id.as_deref() + assert!(session.is_some() || matches!(result, Err(Error::ExpiredCredentials))); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[rstest::rstest] + #[tokio::test] + async fn concurrent_calls_share_acquisition_and_distinct_identities_stay_isolated() { + use std::sync::atomic::{AtomicUsize, Ordering}; + let loads = Arc::new(AtomicUsize::new(0)); + let calls = Arc::new(AtomicUsize::new(0)); + let auth = AwsAuthService::new(Arc::new(CountingLoader { + loads: loads.clone(), + calls: calls.clone(), + short_lifetime: Duration::from_secs(3600), + short_calls: 0, + })); + let first = AwsAuthConfig { + access_key_id: Some("first".into()), + ..Default::default() + }; + let second = AwsAuthConfig { + access_key_id: Some("second".into()), + ..Default::default() + }; + let (a, b, c) = tokio::join!( + auth.resolve_credentials(first.clone(), &no_env), + auth.resolve_credentials(first, &no_env), + auth.resolve_credentials(second, &no_env) ); + let a = a.unwrap(); + let b = b.unwrap(); + let c = c.unwrap(); + assert_eq!(a.session_token(), b.session_token()); + assert_ne!(a.access_key_id(), c.access_key_id()); + assert_eq!(loads.load(Ordering::SeqCst), 2); + assert_eq!(calls.load(Ordering::SeqCst), 2); + } + + #[rstest::rstest] + fn auth_config_debug_never_discloses_credentials() { + let config = AwsAuthConfig { + access_key_id: Some("credential-value".into()), + secret_access_key: Some("credential-value".into()), + session_token: Some("credential-value".into()), + web_identity_token: Some("credential-value".into()), + external_id: Some("credential-value".into()), + ..Default::default() + }; + let output = format!("{config:?}"); + assert!(!output.contains("credential-value")); + assert!(output.contains("[REDACTED]")); + } + + #[rstest::rstest] + #[case::session(AwsAuthFlow::SessionToken { access_key_id: "credential-value".into(), secret_access_key: "credential-value".into(), session_token: "credential-value".into() })] + #[case::static_keys(AwsAuthFlow::StaticKeys { access_key_id: "credential-value".into(), secret_access_key: "credential-value".into(), region_name: "region".into() })] + #[case::web_identity(AwsAuthFlow::WebIdentity { token: "credential-value".into(), role: "role".into(), session_name: "session".into() })] + fn auth_flow_debug_never_discloses_credentials(#[case] flow: AwsAuthFlow) { + let output = format!("{flow:?}"); + assert!(!output.contains("credential-value")); + assert!(output.contains("[REDACTED]")); } #[test] @@ -1142,8 +1589,8 @@ mod tests { let credentials = AwsAuthService::default() .resolve_credentials( AwsAuthConfig { - access_key_id: Some(access_key_id), - secret_access_key: Some(secret_access_key), + access_key_id: Some(access_key_id.into()), + secret_access_key: Some(secret_access_key.into()), region_name: Some("us-west-2".to_string()), ..Default::default() }, @@ -1191,4 +1638,44 @@ mod tests { failures.join("; ") ); } + + #[rstest::rstest] + #[case::provisioned( + None, + "https://cache-1/?Action=connect&User=iam-user", + "cache-1/?Action=connect&User=iam-user&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIDEXAMPLE%2F20261029%2Fus-east-1%2Felasticache%2Faws4_request&X-Amz-Date=20261029T120000Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host&X-Amz-Signature=dfa58492a6ae9ba065088724170a52437329d03f3e9bc9a4c90a80cc8a18e2f6" + )] + #[case::serverless_with_a_session_token( + Some("session-token"), + "https://cache-1/?Action=connect&User=iam-user&ResourceType=ServerlessCache", + "cache-1/?Action=connect&User=iam-user&ResourceType=ServerlessCache&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIDEXAMPLE%2F20261029%2Fus-east-1%2Felasticache%2Faws4_request&X-Amz-Date=20261029T120000Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host&X-Amz-Security-Token=session-token&X-Amz-Signature=e48bdbb3bf9d5e8f1d325bf33d5260e333df1f6b8868e8edd1e4c3cc01b4c3f9" + )] + fn presigned_urls_match_botocore_sigv4_query_auth( + #[case] session_token: Option<&str>, + #[case] url: &str, + #[case] expected: &str, + ) { + // Expected values from botocore 1.43.117 SigV4QueryAuth with get_current_datetime patched to + // 2026-10-29T12:00:00Z (generated 2026-10-29), the signer Python's ElastiCache IAM provider uses + let credentials = Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + session_token.map(str::to_owned), + None, + "test", + ); + let signed = presign_url( + url, + Duration::from_secs(900), + SigningScope { + region: "us-east-1", + service: "elasticache", + credentials: &credentials, + time: UNIX_EPOCH + Duration::from_secs(1_793_275_200), + }, + ) + .unwrap(); + + assert_eq!(signed.strip_prefix("https://"), Some(expected)); + } } diff --git a/litellm-rust/crates/auth-aws/src/constants.rs b/litellm-rust/crates/auth-aws/src/constants.rs index 789fde156bd..0e4662eda9e 100644 --- a/litellm-rust/crates/auth-aws/src/constants.rs +++ b/litellm-rust/crates/auth-aws/src/constants.rs @@ -11,8 +11,8 @@ pub const AWS_SESSION_NAME: &str = AwsParams::SESSION_NAME.env[0]; pub const AWS_PROFILE_NAME: &str = AwsParams::PROFILE_NAME.env[0]; pub const AWS_ROLE_NAME: &str = AwsParams::ROLE_NAME.env[0]; pub const AWS_WEB_IDENTITY_TOKEN: &str = AwsParams::WEB_IDENTITY_TOKEN.env[0]; -pub const AWS_ROLE_ARN: &str = "AWS_ROLE_ARN"; -pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE"; +pub const AWS_ROLE_ARN: &str = AwsParams::ROLE_ARN_ENV; +pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = AwsParams::WEB_IDENTITY_TOKEN_FILE_ENV; pub const AWS_STS_ENDPOINT: &str = AwsParams::STS_ENDPOINT.env[0]; pub const AWS_EXTERNAL_ID: &str = AwsParams::EXTERNAL_ID.env[0]; pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK"; diff --git a/litellm-rust/crates/auth-aws/src/error.rs b/litellm-rust/crates/auth-aws/src/error.rs index 5f88fc8a171..5445de1e6c3 100644 --- a/litellm-rust/crates/auth-aws/src/error.rs +++ b/litellm-rust/crates/auth-aws/src/error.rs @@ -2,6 +2,8 @@ use litellm_error::Source; #[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)] pub enum Error { + #[error("AWS credential provider returned expired credentials")] + ExpiredCredentials, #[error("could not load AWS {flow} credentials")] Credentials { flow: CredentialFlow, diff --git a/litellm-rust/crates/auth-aws/src/lib.rs b/litellm-rust/crates/auth-aws/src/lib.rs index bd0f866371f..c8c5b5e8653 100644 --- a/litellm-rust/crates/auth-aws/src/lib.rs +++ b/litellm-rust/crates/auth-aws/src/lib.rs @@ -1,9 +1,11 @@ mod aws; pub mod constants; mod error; +mod provider; mod signer; pub use aws::*; pub use aws_credential_types::Credentials; pub use error::{CredentialFlow, Error, SigningStage}; +pub use provider::AwsCredentialsProvider; pub use signer::SigV4Signer; diff --git a/litellm-rust/crates/auth-aws/src/provider.rs b/litellm-rust/crates/auth-aws/src/provider.rs new file mode 100644 index 00000000000..6cb2325fc11 --- /dev/null +++ b/litellm-rust/crates/auth-aws/src/provider.rs @@ -0,0 +1,135 @@ +use std::sync::Arc; + +use aws_credential_types::Credentials; +use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future}; + +use crate::{AwsAuthConfig, AwsAuthService}; + +type Environment = Arc Option + Send + Sync>; + +#[derive(Clone)] +pub struct AwsCredentialsProvider { + auth: AwsAuthService, + config: Box, + environment: Environment, +} + +impl AwsCredentialsProvider { + /// Explicit keys in `config` as given, otherwise LiteLLM's `AWS_*` settings read through `environment` + pub fn configured( + auth: AwsAuthService, + config: AwsAuthConfig, + environment: impl Fn(&str) -> Option + Send + Sync + 'static, + ) -> Self { + Self { + auth, + config: Box::new(config), + environment: Arc::new(environment), + } + } + + async fn credentials(&self) -> Result { + match (&self.config.access_key_id, &self.config.secret_access_key) { + (Some(access_key_id), Some(secret_access_key)) => Ok(Credentials::new( + access_key_id.expose(), + secret_access_key.expose(), + self.config + .session_token + .as_ref() + .map(|token| token.expose().to_owned()), + None, + "litellm-configured", + )), + _ => { + self.auth + .resolve_credentials( + AwsAuthConfig::clone(&self.config), + self.environment.as_ref(), + ) + .await + } + } + } +} + +impl ProvideCredentials for AwsCredentialsProvider { + fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a> + where + Self: 'a, + { + future::ProvideCredentials::new(async { + self.credentials() + .await + .map_err(CredentialsError::provider_error) + }) + } +} + +impl std::fmt::Debug for AwsCredentialsProvider { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("AwsCredentialsProvider") + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn explicit(session_token: Option<&str>) -> AwsAuthConfig { + AwsAuthConfig { + access_key_id: Some("key".into()), + secret_access_key: Some("secret".into()), + session_token: session_token.map(Into::into), + region_name: Some("us-east-1".to_string()), + ..Default::default() + } + } + + fn ambient_token(name: &str) -> Option { + (name == "AWS_SESSION_TOKEN").then(|| "ambient".to_string()) + } + + fn environment_keys(name: &str) -> Option { + match name { + "AWS_ACCESS_KEY_ID" => Some("env-key".to_string()), + "AWS_SECRET_ACCESS_KEY" => Some("env-secret".to_string()), + "AWS_SESSION_TOKEN" => Some("env-token".to_string()), + _ => None, + } + } + + #[rstest] + #[case::explicit_keys_ignore_an_ambient_session_token( + explicit(None), ambient_token, ("key", "secret", None) + )] + #[case::explicit_keys_keep_their_session_token( + explicit(Some("t")), ambient_token, ("key", "secret", Some("t")) + )] + #[case::environment_keys_resolve_with_their_session_token( + AwsAuthConfig::default(), environment_keys, ("env-key", "env-secret", Some("env-token")) + )] + #[tokio::test] + async fn configured_credentials_read_the_environment_they_are_given( + #[case] config: AwsAuthConfig, + #[case] environment: fn(&str) -> Option, + #[case] expected: (&str, &str, Option<&str>), + ) { + let credentials = + AwsCredentialsProvider::configured(AwsAuthService::default(), config, environment) + .provide_credentials() + .await + .unwrap(); + assert_eq!( + ( + credentials.access_key_id(), + credentials.secret_access_key(), + credentials.session_token(), + ), + expected + ); + } +} diff --git a/litellm-rust/crates/auth-aws/src/signer.rs b/litellm-rust/crates/auth-aws/src/signer.rs index dff917d7c13..2d9ad9ba123 100644 --- a/litellm-rust/crates/auth-aws/src/signer.rs +++ b/litellm-rust/crates/auth-aws/src/signer.rs @@ -75,7 +75,8 @@ impl RequestSigner for SigV4Signer { ) .map_err(|error| match error { Error::NonAsciiHeader { .. } => litellm_http::Error::unsignable_header(error), - Error::Credentials { .. } + Error::ExpiredCredentials + | Error::Credentials { .. } | Error::MissingWebIdentityCredentials | Error::WebIdentityExpiration(_) | Error::Signing { .. } => litellm_http::Error::signature(error), diff --git a/litellm-rust/crates/auth-azure/Cargo.toml b/litellm-rust/crates/auth-azure/Cargo.toml index 7dcb0e513a7..36c10a74c80 100644 --- a/litellm-rust/crates/auth-azure/Cargo.toml +++ b/litellm-rust/crates/auth-azure/Cargo.toml @@ -8,17 +8,19 @@ repository.workspace = true [dependencies] litellm-secret-value.workspace = true litellm-auth-types.workspace = true +litellm-error.workspace = true moka.workspace = true serde_json.workspace = true sha2.workspace = true strum.workspace = true +tokio.workspace = true url.workspace = true azure_core = "1.0.0" azure_identity = { version = "1.0.0", features = ["tokio"] } [dev-dependencies] -litellm-error.workspace = true +litellm-secrets.workspace = true rstest.workspace = true -tokio.workspace = true +tokio = { workspace = true, features = ["test-util"] } diff --git a/litellm-rust/crates/auth-azure/src/lib.rs b/litellm-rust/crates/auth-azure/src/lib.rs index 9ed505e4160..6319406ce9c 100644 --- a/litellm-rust/crates/auth-azure/src/lib.rs +++ b/litellm-rust/crates/auth-azure/src/lib.rs @@ -3,5 +3,5 @@ mod native; mod resolve; mod types; -pub use resolve::{AzureAuthService, SECRET_NAMES}; +pub use resolve::{AzureAuthService, AzureIdentity, SECRET_NAMES}; pub use types::{AzureAuthInputs, ConfigValue}; diff --git a/litellm-rust/crates/auth-azure/src/native.rs b/litellm-rust/crates/auth-azure/src/native.rs index f40745ab7a2..03021428ea7 100644 --- a/litellm-rust/crates/auth-azure/src/native.rs +++ b/litellm-rust/crates/auth-azure/src/native.rs @@ -59,6 +59,7 @@ pub(crate) enum NativeAzureRequest { pub(crate) struct ValidatedAzureRequest { request: NativeAzureRequest, credential_source: InputSource, + probe_imds: bool, } impl ValidatedAzureRequest { @@ -68,9 +69,18 @@ impl ValidatedAzureRequest { Ok(Self { request, credential_source, + probe_imds: false, }) } + pub(crate) fn with_imds_probe(self, enabled: bool) -> Self { + Self { + probe_imds: enabled + && matches!(self.request, NativeAzureRequest::ManagedIdentity { .. }), + ..self + } + } + pub(crate) fn credential_source(&self) -> InputSource { self.credential_source } @@ -87,9 +97,34 @@ impl ValidatedAzureRequest { } } +pub(crate) enum CredentialAttempt { + Resolved(ResolvedCredential), + Unavailable(Error), + Failed(Error), +} + +impl CredentialAttempt { + pub(crate) fn into_result(self) -> Result { + match self { + Self::Resolved(credential) => Ok(credential), + Self::Unavailable(error) | Self::Failed(error) => Err(error), + } + } +} + pub(crate) struct NativeAzureTokenAcquirer { cache: AzureCredentialProviderCache, transport: Option, + imds_available: tokio::sync::Mutex, +} + +enum ImdsAvailability { + Unknown, + Available, + Unavailable { + retry_at: tokio::time::Instant, + error: Error, + }, } impl Default for NativeAzureTokenAcquirer { @@ -103,6 +138,7 @@ impl NativeAzureTokenAcquirer { Self { cache: AzureCredentialProviderCache::new(cache_capacity), transport: None, + imds_available: tokio::sync::Mutex::new(ImdsAvailability::Unknown), } } @@ -114,10 +150,51 @@ impl NativeAzureTokenAcquirer { Self { cache: AzureCredentialProviderCache::new(cache_capacity), transport: Some(transport), + imds_available: tokio::sync::Mutex::new(ImdsAvailability::Unknown), } } - pub(crate) async fn acquire( + pub(crate) async fn acquire(&self, request: ValidatedAzureRequest) -> CredentialAttempt { + let discover_imds = request.probe_imds; + if discover_imds && let Err(error) = self.discover_imds().await { + return CredentialAttempt::Unavailable(error); + } + match self.acquire_token(request).await { + Ok(credential) => CredentialAttempt::Resolved(credential), + Err(error) => { + let missing_identity = discover_imds && unassigned_identity(&error); + if missing_identity { + CredentialAttempt::Unavailable(error) + } else { + CredentialAttempt::Failed(error) + } + } + } + } + + async fn discover_imds(&self) -> Result<(), Error> { + let mut availability = self.imds_available.lock().await; + match &*availability { + ImdsAvailability::Available => return Ok(()), + ImdsAvailability::Unavailable { retry_at, error } + if tokio::time::Instant::now() < *retry_at => + { + return Err(error.clone()); + } + ImdsAvailability::Unknown | ImdsAvailability::Unavailable { .. } => {} + } + let result = probe_imds(self.transport.clone().unwrap_or_default()).await; + *availability = match &result { + Ok(()) => ImdsAvailability::Available, + Err(error) => ImdsAvailability::Unavailable { + retry_at: tokio::time::Instant::now() + Duration::from_secs(30), + error: error.clone(), + }, + }; + result + } + + async fn acquire_token( &self, request: ValidatedAzureRequest, ) -> Result { @@ -148,6 +225,54 @@ impl NativeAzureTokenAcquirer { } } +fn unassigned_identity(error: &(dyn std::error::Error + 'static)) -> bool { + std::iter::successors(Some(error), |cause| cause.source()).any(|cause| { + cause + .downcast_ref::() + .or_else(|| { + cause + .downcast_ref::()? + .downcast_ref::() + }) + .is_some_and(|error| { + error.http_status() == Some(azure_core::http::StatusCode::BadRequest) + }) + }) +} + +async fn probe_imds(transport: azure_core::http::Transport) -> Result<(), Error> { + use azure_core::http::{Context, Method, Request, Url}; + + let endpoint = + Url::parse("http://169.254.169.254/metadata/identity/oauth2/token?api-version=2019-08-01") + .map_err(|error| { + Error::InvalidConfiguration(ErrorDetail::failed("Azure IMDS probe endpoint", error)) + })?; + let mut request = Request::new(endpoint, Method::Get); + let response = tokio::time::timeout( + Duration::from_secs(1), + transport.send(&Context::new(), &mut request), + ) + .await + .map_err(|error| { + Error::CredentialAcquisition(ErrorDetail::failed("Azure IMDS availability probe", error)) + })? + .map_err(|error| { + Error::CredentialAcquisition(ErrorDetail::failed("Azure IMDS availability probe", error)) + })?; + // Headerless IMDS discovery expects 400, checked 2026-11-01 + // https://learn.microsoft.com/en-us/azure/virtual-machines/instance-metadata-service#errors-and-debugging + if response.status() != azure_core::http::StatusCode::BadRequest { + return Err(Error::CredentialAcquisition( + ErrorDetail::UnexpectedStatus { + operation: "Azure IMDS availability probe", + status: u16::from(response.status()), + }, + )); + } + Ok(()) +} + impl NativeAzureRequest { fn scope(&self) -> &str { match self { @@ -493,6 +618,7 @@ impl ClientAssertion for StaticAssertion { #[cfg(test)] mod tests { use std::sync::{Arc, Mutex}; + use std::time::Duration; use azure_core::http::headers::Headers; use azure_core::http::{AsyncRawResponse, HttpClient, Request, StatusCode, Transport}; @@ -569,6 +695,296 @@ mod tests { } } + #[derive(Debug)] + struct AvailableImdsClient { + probe_status: StatusCode, + probe_delay: Duration, + recover_after: Option, + response_status: Option, + probes: std::sync::atomic::AtomicUsize, + } + + impl Default for AvailableImdsClient { + fn default() -> Self { + Self { + probe_status: StatusCode::BadRequest, + probe_delay: Duration::ZERO, + recover_after: None, + response_status: None, + probes: std::sync::atomic::AtomicUsize::new(0), + } + } + } + + impl HttpClient for AvailableImdsClient { + fn execute_request<'life0, 'life1, 'async_trait>( + &'life0 self, + request: &'life1 Request, + ) -> std::pin::Pin< + Box> + Send + 'async_trait>, + > + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + Box::pin(async move { + if request + .headers() + .get_optional_str(&azure_core::http::headers::HeaderName::from_static( + "metadata", + )) + .is_none() + { + let probe = self + .probes + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if self.recover_after.is_some_and(|failures| probe >= failures) { + return Ok(AsyncRawResponse::from_bytes( + StatusCode::BadRequest, + Headers::new(), + "metadata header required", + )); + } + tokio::time::sleep(self.probe_delay).await; + return Ok(AsyncRawResponse::from_bytes( + self.probe_status, + Headers::new(), + "metadata header required", + )); + } + if let Some(status) = self.response_status { + return Ok(AsyncRawResponse::from_bytes( + status, + Headers::new(), + "identity endpoint failure", + )); + } + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + let expires = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() + + 3600; + Ok(AsyncRawResponse::from_bytes( + StatusCode::Ok, + Headers::new(), + serde_json::json!({ + "access_token": "managed-token", "expires_on": expires.to_string(), + "token_type": "Bearer", "resource": "https://service.test", + }) + .to_string(), + )) + }) + } + } + + #[rstest::rstest] + #[tokio::test(start_paused = true)] + async fn available_imds_retains_full_sdk_token_acquisition() { + let acquirer = NativeAzureTokenAcquirer::with_transport( + 4, + Transport::new(Arc::new(AvailableImdsClient { + probe_status: StatusCode::BadRequest, + response_status: None, + ..Default::default() + })), + ); + let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity { + client_id: None, + scope: deployment("https://service.test/.default".into()), + selection_source: InputSource::Deployment, + }) + .unwrap() + .with_imds_probe(true); + let credential = acquirer.acquire(request).await.into_result().unwrap(); + assert_eq!(credential.secret().expose(), "managed-token"); + } + + #[rstest::rstest] + #[case::another_metadata_service(StatusCode::NotFound)] + #[case::token_required(StatusCode::Unauthorized)] + #[case::proxy_rejection(StatusCode::Forbidden)] + #[case::unavailable(StatusCode::ServiceUnavailable)] + #[tokio::test(start_paused = true)] + async fn unrelated_metadata_responses_do_not_start_managed_identity_acquisition( + #[case] status: StatusCode, + ) { + let client = Arc::new(AvailableImdsClient { + probe_status: status, + ..Default::default() + }); + let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone())); + let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity { + client_id: None, + scope: deployment("https://service.test/.default".into()), + selection_source: InputSource::Deployment, + }) + .unwrap() + .with_imds_probe(true); + assert!(matches!( + acquirer.acquire(request.clone()).await, + super::CredentialAttempt::Unavailable(_) + )); + assert!(matches!( + acquirer.acquire(request).await, + super::CredentialAttempt::Unavailable(_) + )); + assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 1); + } + + #[rstest::rstest] + #[tokio::test(start_paused = true)] + async fn an_imds_timeout_is_not_repeated_on_the_next_request() { + let client = Arc::new(AvailableImdsClient { + probe_delay: Duration::from_secs(10), + ..Default::default() + }); + let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone())); + let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity { + client_id: None, + scope: deployment("https://service.test/.default".into()), + selection_source: InputSource::Deployment, + }) + .unwrap() + .with_imds_probe(true); + assert!(matches!( + acquirer.acquire(request.clone()).await, + super::CredentialAttempt::Unavailable(_) + )); + let second = tokio::time::timeout(Duration::from_millis(10), acquirer.acquire(request)) + .await + .unwrap(); + assert!(matches!(second, super::CredentialAttempt::Unavailable(_))); + assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 1); + } + + #[rstest::fixture] + fn imds_request() -> ValidatedAzureRequest { + ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity { + client_id: None, + scope: deployment("https://service.test/.default".into()), + selection_source: InputSource::Deployment, + }) + .unwrap() + .with_imds_probe(true) + } + + #[rstest::rstest] + #[case::timeout(StatusCode::BadRequest, Duration::from_secs(10))] + #[case::service_unavailable(StatusCode::ServiceUnavailable, Duration::ZERO)] + #[tokio::test(start_paused = true)] + async fn failed_imds_discovery_recovers_without_restarting_the_acquirer( + imds_request: ValidatedAzureRequest, + #[case] status: StatusCode, + #[case] delay: Duration, + ) { + let client = Arc::new(AvailableImdsClient { + probe_status: status, + probe_delay: delay, + recover_after: Some(1), + ..Default::default() + }); + let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone())); + let (first, second) = tokio::join!( + acquirer.acquire(imds_request.clone()), + acquirer.acquire(imds_request.clone()), + ); + assert!(matches!(first, super::CredentialAttempt::Unavailable(_))); + assert!(matches!(second, super::CredentialAttempt::Unavailable(_))); + assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 1); + + tokio::time::advance(Duration::from_secs(30)).await; + let (first, second) = tokio::join!( + acquirer.acquire(imds_request.clone()), + acquirer.acquire(imds_request.clone()), + ); + assert_eq!( + first.into_result().unwrap().secret().expose(), + "managed-token" + ); + assert_eq!( + second.into_result().unwrap().secret().expose(), + "managed-token" + ); + assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 2); + + tokio::time::advance(Duration::from_secs(60)).await; + assert_eq!( + acquirer + .acquire(imds_request) + .await + .into_result() + .unwrap() + .secret() + .expose(), + "managed-token", + ); + assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 2); + } + + #[rstest::rstest] + #[tokio::test(start_paused = true)] + async fn a_persistent_imds_outage_reuses_each_failed_probe( + imds_request: ValidatedAzureRequest, + ) { + let client = Arc::new(AvailableImdsClient { + probe_delay: Duration::from_secs(10), + ..Default::default() + }); + let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone())); + assert!(matches!( + acquirer.acquire(imds_request.clone()).await, + super::CredentialAttempt::Unavailable(_), + )); + tokio::time::advance(Duration::from_secs(30)).await; + let (first, second) = tokio::join!( + acquirer.acquire(imds_request.clone()), + acquirer.acquire(imds_request), + ); + assert!(matches!(first, super::CredentialAttempt::Unavailable(_))); + assert!(matches!(second, super::CredentialAttempt::Unavailable(_))); + assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 2); + } + + #[rstest::rstest] + #[case::default_identity_not_assigned(true, StatusCode::BadRequest, true)] + #[case::explicit_identity_not_assigned(false, StatusCode::BadRequest, false)] + #[case::default_identity_forbidden(true, StatusCode::Forbidden, false)] + #[case::default_identity_service_failure(true, StatusCode::InternalServerError, false)] + #[tokio::test(start_paused = true)] + async fn only_unavailable_default_identities_allow_fallback( + #[case] discovery: bool, + #[case] status: StatusCode, + #[case] unavailable: bool, + ) { + let acquirer = NativeAzureTokenAcquirer::with_transport( + 4, + Transport::new(Arc::new(AvailableImdsClient { + probe_status: StatusCode::BadRequest, + response_status: Some(status), + ..Default::default() + })), + ); + let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity { + client_id: None, + scope: deployment("https://service.test/.default".into()), + selection_source: InputSource::Deployment, + }) + .unwrap() + .with_imds_probe(discovery); + let outcome = acquirer.acquire(request).await; + let is_unavailable = matches!(outcome, super::CredentialAttempt::Unavailable(_)); + let error = outcome.into_result().unwrap_err(); + assert_eq!( + is_unavailable, + unavailable, + "{}", + litellm_error::Report::new(&error) + ); + } + + #[rstest::rstest] #[tokio::test] async fn client_secret_uses_sdk_protocol_and_reuses_cached_credential() { let transport = Arc::new(RecordingTokenClient::default()); @@ -582,8 +998,12 @@ mod tests { "https://login.test", ); - let first = acquirer.acquire(request.clone()).await.unwrap(); - let second = acquirer.acquire(request).await.unwrap(); + let first = acquirer + .acquire(request.clone()) + .await + .into_result() + .unwrap(); + let second = acquirer.acquire(request).await.into_result().unwrap(); assert_eq!(first.secret().expose(), "native-token"); assert_eq!(second.secret().expose(), "native-token"); @@ -599,6 +1019,7 @@ mod tests { ); } + #[rstest::rstest] #[tokio::test] async fn credential_provider_cache_isolates_every_client_secret_identity_field() { let transport = Arc::new(RecordingTokenClient::default()); @@ -645,10 +1066,10 @@ mod tests { ), ]; - acquirer.acquire(base.clone()).await.unwrap(); - acquirer.acquire(base).await.unwrap(); + acquirer.acquire(base.clone()).await.into_result().unwrap(); + acquirer.acquire(base).await.into_result().unwrap(); for request in variants.into_iter().skip(1) { - acquirer.acquire(request).await.unwrap(); + acquirer.acquire(request).await.into_result().unwrap(); } assert_eq!(transport.requests.lock().unwrap().len(), 6); diff --git a/litellm-rust/crates/auth-azure/src/resolve.rs b/litellm-rust/crates/auth-azure/src/resolve.rs index a92630af26a..9ff4eda540a 100644 --- a/litellm-rust/crates/auth-azure/src/resolve.rs +++ b/litellm-rust/crates/auth-azure/src/resolve.rs @@ -1,3 +1,4 @@ +use crate::native::CredentialAttempt; use litellm_auth_types::{ CredentialFileRef, CredentialLookup, CredentialRef, InputSource, ResolvedCredential, Sourced, TokenProviderHandle, @@ -12,25 +13,31 @@ use std::sync::Arc; use super::native::{NativeAzureRequest, NativeAzureTokenAcquirer, ValidatedAzureRequest}; use super::types::{AzureAuthInputs, AzureCredentialType, ConfigValue, DEFAULT_AZURE_SCOPE}; -const AZURE_AD_TOKEN_ENV: &str = "AZURE_AD_TOKEN"; -const AZURE_TENANT_ID_ENV: &str = "AZURE_TENANT_ID"; -const AZURE_CLIENT_ID_ENV: &str = "AZURE_CLIENT_ID"; -const AZURE_CLIENT_SECRET_ENV: &str = "AZURE_CLIENT_SECRET"; -const AZURE_SCOPE_ENV: &str = "AZURE_SCOPE"; -const AZURE_AUTHORITY_HOST_ENV: &str = "AZURE_AUTHORITY_HOST"; -const AZURE_CREDENTIAL_ENV: &str = "AZURE_CREDENTIAL"; -const AZURE_FEDERATED_TOKEN_FILE_ENV: &str = "AZURE_FEDERATED_TOKEN_FILE"; +#[derive(strum::IntoStaticStr, strum::VariantArray, strum::VariantNames)] +enum AzureSecretName { + #[strum(serialize = "AZURE_AD_TOKEN")] + AdToken, + #[strum(serialize = "AZURE_TENANT_ID")] + TenantId, + #[strum(serialize = "AZURE_CLIENT_ID")] + ClientId, + #[strum(serialize = "AZURE_CLIENT_SECRET")] + ClientSecret, + #[strum(serialize = "AZURE_SCOPE")] + Scope, + #[strum(serialize = "AZURE_AUTHORITY_HOST")] + AuthorityHost, + #[strum(serialize = "AZURE_CREDENTIAL")] + Credential, + #[strum(serialize = "AZURE_FEDERATED_TOKEN_FILE")] + FederatedTokenFile, + #[strum(serialize = "IDENTITY_ENDPOINT")] + IdentityEndpoint, + #[strum(serialize = "MSI_ENDPOINT")] + MsiEndpoint, +} -pub const SECRET_NAMES: &[&str] = &[ - AZURE_AD_TOKEN_ENV, - AZURE_TENANT_ID_ENV, - AZURE_CLIENT_ID_ENV, - AZURE_CLIENT_SECRET_ENV, - AZURE_SCOPE_ENV, - AZURE_AUTHORITY_HOST_ENV, - AZURE_CREDENTIAL_ENV, - AZURE_FEDERATED_TOKEN_FILE_ENV, -]; +pub const SECRET_NAMES: &[&str] = ::VARIANTS; #[derive(Clone, Debug)] pub(crate) enum AzureCredentialPlan { @@ -57,14 +64,14 @@ trait AzureTokenAcquirer: Send + Sync { fn acquire( &self, request: ValidatedAzureRequest, - ) -> Pin> + Send + '_>>; + ) -> Pin + Send + '_>>; } impl AzureTokenAcquirer for NativeAzureTokenAcquirer { fn acquire( &self, request: ValidatedAzureRequest, - ) -> Pin> + Send + '_>> { + ) -> Pin + Send + '_>> { Box::pin(NativeAzureTokenAcquirer::acquire(self, request)) } } @@ -83,12 +90,36 @@ impl AzureAuthService { Self { native } } + pub async fn identity_token( + &self, + identity: &AzureIdentity, + scope: &str, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result, Error> { + let plan = identity_plan(identity, scope, env_lookup)?; + let inputs = AzureAuthInputs::default_credential_for_scope(scope); + Ok(self + .run_plan(plan, &inputs, env_lookup) + .await? + .map(Sourced::into_value)) + } + pub async fn get_azure_ad_token( &self, inputs: &AzureAuthInputs, env_lookup: &(dyn Fn(&str) -> Option + Sync), ) -> Result>, Error> { - match select_auth_plan(inputs, env_lookup)? { + self.run_plan(select_auth_plan(inputs, env_lookup)?, inputs, env_lookup) + .await + } + + async fn run_plan( + &self, + plan: AzureCredentialPlan, + inputs: &AzureAuthInputs, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result>, Error> { + match plan { AzureCredentialPlan::Supplied(credential) => Ok(Some(credential)), AzureCredentialPlan::Caller(caller) => { let credential = caller.acquire().await?; @@ -123,6 +154,7 @@ impl AzureAuthService { self.native .acquire(request) .await + .into_result() .map(|credential| Sourced::new(credential, source)) .map(Some) } @@ -131,6 +163,7 @@ impl AzureAuthService { self.native .acquire(request) .await + .into_result() .map(|credential| Some(Sourced::new(credential, source))) } AzureCredentialPlan::Chain(requests) => { @@ -138,8 +171,16 @@ impl AzureAuthService { for request in requests { let source = request.credential_source(); match self.native.acquire(request).await { - Ok(credential) => return Ok(Some(Sourced::new(credential, source))), - Err(error) => failures.push(error), + CredentialAttempt::Resolved(credential) => { + return Ok(Some(Sourced::new(credential, source))); + } + CredentialAttempt::Unavailable(error) => failures.push(error), + CredentialAttempt::Failed(error) => { + return Err(Error::CredentialChain { + last: Box::new(error), + earlier: failures, + }); + } } } match failures.pop() { @@ -155,39 +196,161 @@ impl AzureAuthService { } } +/// The identity azure-identity builds for one explicit credential class, without LiteLLM's provider settings +/// (`AZURE_AD_TOKEN`, `AZURE_CREDENTIAL`, `AZURE_SCOPE`) that `get_azure_ad_token` also reads. +#[derive(Clone, Debug)] +pub enum AzureIdentity { + ClientSecret { + tenant_id: String, + client_id: String, + client_secret: SecretValue, + }, + ManagedIdentity { + client_id: String, + }, + Default, +} + +fn identity_plan( + identity: &AzureIdentity, + scope: &str, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + let deployment = |value: &str| Sourced::new(value.to_owned(), InputSource::Deployment); + let environment = |name: &str| { + env_lookup(name) + .filter(|value| !value.trim().is_empty()) + .map(|value| Sourced::new(value, InputSource::Environment)) + }; + let scope = deployment(scope); + let authority = environment(AzureSecretName::AuthorityHost.into()); + match identity { + AzureIdentity::ClientSecret { + tenant_id, + client_id, + client_secret, + } => Ok(AzureCredentialPlan::Native(ValidatedAzureRequest::new( + NativeAzureRequest::ClientSecret { + tenant_id: deployment(tenant_id), + client_id: deployment(client_id), + client_secret: Sourced::new(client_secret.clone(), InputSource::Deployment), + scope, + authority, + }, + )?)), + AzureIdentity::ManagedIdentity { client_id } => Ok(AzureCredentialPlan::Native( + ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity { + client_id: Some(deployment(client_id)), + scope, + selection_source: InputSource::Deployment, + })?, + )), + AzureIdentity::Default => { + let tenant_id = environment(AzureSecretName::TenantId.into()); + let client_id = environment(AzureSecretName::ClientId.into()); + let from_environment = match ( + tenant_id.clone(), + client_id.clone(), + environment(AzureSecretName::ClientSecret.into()), + ) { + (Some(tenant_id), Some(client_id), Some(client_secret)) => { + Some(NativeAzureRequest::ClientSecret { + tenant_id, + client_id, + client_secret: client_secret.map(SecretValue::new), + scope: scope.clone(), + authority: authority.clone(), + }) + } + _ => None, + }; + let probe_imds = env_lookup(AzureSecretName::IdentityEndpoint.into()).is_none() + && env_lookup(AzureSecretName::MsiEndpoint.into()).is_none(); + let AzureCredentialPlan::Chain(chain) = select_native_plan( + Some(Sourced::new( + AzureCredentialType::DefaultAzureCredential, + InputSource::Deployment, + )), + tenant_id, + client_id, + environment(AzureSecretName::FederatedTokenFile.into()), + scope, + authority, + InputSource::Environment, + )? + else { + unreachable!("DefaultAzureCredential always plans a chain") + }; + let environment_request = from_environment + .map(ValidatedAzureRequest::new) + .transpose()?; + Ok(AzureCredentialPlan::Chain( + environment_request + .into_iter() + .chain(chain) + .map(|request| request.with_imds_probe(probe_imds)) + .collect(), + )) + } + } +} + pub(crate) fn select_auth_plan( inputs: &AzureAuthInputs, env_lookup: &dyn Fn(&str) -> Option, ) -> Result { - let token = configured_secret(&inputs.azure_ad_token, AZURE_AD_TOKEN_ENV, env_lookup); - let tenant_id = configured_string(&inputs.tenant_id, AZURE_TENANT_ID_ENV, env_lookup); - let client_id = configured_string(&inputs.client_id, AZURE_CLIENT_ID_ENV, env_lookup); - let client_secret = - configured_secret(&inputs.client_secret, AZURE_CLIENT_SECRET_ENV, env_lookup); - let scope = configured_string(&inputs.azure_scope, AZURE_SCOPE_ENV, env_lookup) - .unwrap_or_else(|| Sourced::new(DEFAULT_AZURE_SCOPE.to_string(), InputSource::Environment)); - let authority = configured_string( - &inputs.azure_authority_host, - AZURE_AUTHORITY_HOST_ENV, + let token = configured_secret( + &inputs.azure_ad_token, + AzureSecretName::AdToken.into(), env_lookup, ); - let selector = configured_string(&inputs.azure_credential, AZURE_CREDENTIAL_ENV, env_lookup) - .map(|value| { - value - .value() - .parse::() - .map(|selector| Sourced::new(selector, value.source())) - }) - .transpose() - .map_err(|error| { - Error::InvalidConfiguration(ErrorDetail::failed( - "Azure credential selector parsing", - error, - )) - })?; + let tenant_id = configured_string( + &inputs.tenant_id, + AzureSecretName::TenantId.into(), + env_lookup, + ); + let client_id = configured_string( + &inputs.client_id, + AzureSecretName::ClientId.into(), + env_lookup, + ); + let client_secret = configured_secret( + &inputs.client_secret, + AzureSecretName::ClientSecret.into(), + env_lookup, + ); + let scope = configured_string( + &inputs.azure_scope, + AzureSecretName::Scope.into(), + env_lookup, + ) + .unwrap_or_else(|| Sourced::new(DEFAULT_AZURE_SCOPE.to_string(), InputSource::Environment)); + let authority = configured_string( + &inputs.azure_authority_host, + AzureSecretName::AuthorityHost.into(), + env_lookup, + ); + let selector = configured_string( + &inputs.azure_credential, + AzureSecretName::Credential.into(), + env_lookup, + ) + .map(|value| { + value + .value() + .parse::() + .map(|selector| Sourced::new(selector, value.source())) + }) + .transpose() + .map_err(|error| { + Error::InvalidConfiguration(ErrorDetail::failed( + "Azure credential selector parsing", + error, + )) + })?; let federated_token_file = configured_string( &inputs.federated_token_file, - AZURE_FEDERATED_TOKEN_FILE_ENV, + AzureSecretName::FederatedTokenFile.into(), env_lookup, ); @@ -237,7 +400,12 @@ pub(crate) fn select_auth_plan( return Ok(AzureCredentialPlan::Missing); } - select_native_plan( + let probe_imds = selector + .as_ref() + .is_none_or(|selector| *selector.value() == AzureCredentialType::DefaultAzureCredential) + && env_lookup(AzureSecretName::IdentityEndpoint.into()).is_none() + && env_lookup(AzureSecretName::MsiEndpoint.into()).is_none(); + match select_native_plan( selector, tenant_id, client_id, @@ -245,7 +413,15 @@ pub(crate) fn select_auth_plan( scope, authority, inputs.enable_azure_ad_token_refresh.source(), - ) + )? { + AzureCredentialPlan::Chain(requests) => Ok(AzureCredentialPlan::Chain( + requests + .into_iter() + .map(|request| request.with_imds_probe(probe_imds)) + .collect(), + )), + plan => Ok(plan), + } } fn select_native_plan( @@ -509,7 +685,10 @@ mod tests { #[derive(Debug)] struct FileResolver; + use crate::native::CredentialAttempt; + struct ChainAcquirer { + terminal_on: Option<&'static str>, requests: Mutex>, succeed_on: Option<&'static str>, } @@ -518,20 +697,25 @@ mod tests { fn acquire( &self, request: ValidatedAzureRequest, - ) -> std::pin::Pin> + Send + '_>> - { + ) -> std::pin::Pin + Send + '_>> { let kind = request.kind(); self.requests.lock().unwrap().push(kind); Box::pin(async move { if self.succeed_on == Some(kind) { - Ok(ResolvedCredential::AccessToken { + CredentialAttempt::Resolved(ResolvedCredential::AccessToken { token: SecretValue::new("chain-token"), expires_on: None, }) } else { - Err(Error::CredentialAcquisition( - litellm_auth_types::ErrorDetail::Unresolved { subject: kind }, - )) + let error = + Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::Unresolved { + subject: kind, + }); + if self.terminal_on == Some(kind) { + CredentialAttempt::Failed(error) + } else { + CredentialAttempt::Unavailable(error) + } } }) } @@ -690,9 +874,210 @@ mod tests { assert_eq!(resolved, Some(SecretValue::new("rotated-assertion"))); } + #[derive(Debug)] + struct UnavailableImdsClient; + + impl azure_core::http::HttpClient for UnavailableImdsClient { + fn execute_request<'life0, 'life1, 'async_trait>( + &'life0 self, + _request: &'life1 azure_core::http::Request, + ) -> std::pin::Pin< + Box< + dyn Future> + + Send + + 'async_trait, + >, + > + where + 'life0: 'async_trait, + 'life1: 'async_trait, + Self: 'async_trait, + { + Box::pin(std::future::pending()) + } + } + + struct OfflineChainAcquirer(super::NativeAzureTokenAcquirer); + + impl AzureTokenAcquirer for OfflineChainAcquirer { + fn acquire( + &self, + request: ValidatedAzureRequest, + ) -> std::pin::Pin + Send + '_>> { + Box::pin(async move { + if request.kind() == "developer-tools" { + return CredentialAttempt::Resolved(ResolvedCredential::AccessToken { + token: SecretValue::new("cli-token"), + expires_on: None, + }); + } + self.0.acquire(request).await + }) + } + } + + #[rstest::rstest] + #[case::default_off_azure("DefaultAzureCredential", None, true)] + #[case::explicit_managed_identity("ManagedIdentityCredential", None, false)] + #[case::app_service("DefaultAzureCredential", Some("http://app-service.test"), false)] + #[tokio::test(start_paused = true)] + async fn only_default_imds_discovery_is_bounded( + #[case] selector: &str, + #[case] identity_endpoint: Option<&str>, + #[case] expect_cli: bool, + ) { + let native = super::NativeAzureTokenAcquirer::with_transport( + 4, + azure_core::http::Transport::new(Arc::new(UnavailableImdsClient)), + ); + let service = AzureAuthService::with_acquirer(Arc::new(OfflineChainAcquirer(native))); + let inputs = AzureAuthInputs { + azure_credential: crate::types::ConfigValue::Value(Sourced::new( + selector.into(), + InputSource::Deployment, + )), + enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment), + ..Default::default() + }; + let environment = |name: &str| { + (name == "IDENTITY_ENDPOINT") + .then_some(identity_endpoint) + .flatten() + .map(str::to_string) + }; + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(2), + service.get_azure_ad_token(&inputs, &environment), + ) + .await; + + assert_eq!(outcome.is_ok(), expect_cli); + if expect_cli { + assert_eq!( + outcome.unwrap().unwrap().unwrap().value().secret().expose(), + "cli-token" + ); + } + } + + #[rstest::rstest] + #[case::app_service(Some("IDENTITY_ENDPOINT"), false)] + #[case::legacy_app_service(Some("MSI_ENDPOINT"), false)] + #[case::off_azure(None, true)] + #[tokio::test(start_paused = true)] + async fn default_chain_uses_endpoints_from_the_secret_snapshot( + #[case] endpoint: Option<&str>, + #[case] expect_cli: bool, + #[values(false, true)] consumer_identity: bool, + ) { + use litellm_secrets::source::SecretSource; + + struct EndpointSecrets<'a>(Option<&'a str>); + + impl SecretSource for EndpointSecrets<'_> { + fn get_secret_str<'a>( + &'a self, + name: &'a str, + ) -> std::pin::Pin< + Box< + dyn Future, litellm_secrets::Error>> + + Send + + 'a, + >, + > { + Box::pin(async move { + Ok((self.0 == Some(name)).then(|| SecretValue::new("http://app-service.test"))) + }) + } + } + + let snapshot = EndpointSecrets(endpoint) + .resolve(SECRET_NAMES) + .await + .unwrap(); + let native = super::NativeAzureTokenAcquirer::with_transport( + 4, + azure_core::http::Transport::new(Arc::new(UnavailableImdsClient)), + ); + let service = AzureAuthService::with_acquirer(Arc::new(OfflineChainAcquirer(native))); + let lookup = |name: &str| snapshot.get(name); + let scope = "https://service.test/.default"; + let outcome = tokio::time::timeout(std::time::Duration::from_secs(2), async { + if consumer_identity { + service + .identity_token(&super::AzureIdentity::Default, scope, &lookup) + .await + } else { + service + .get_azure_ad_token( + &AzureAuthInputs::default_credential_for_scope(scope), + &lookup, + ) + .await + .map(|token| token.map(Sourced::into_value)) + } + }) + .await; + + assert_eq!(outcome.is_ok(), expect_cli); + if expect_cli { + assert_eq!( + outcome.unwrap().unwrap().unwrap().secret().expose(), + "cli-token" + ); + } + } + + #[rstest::rstest] + #[case::litellm_provider_settings_are_ignored( + &[("AZURE_AD_TOKEN", "provider-token"), ("AZURE_CREDENTIAL", "ClientSecretCredential")], + &["managed-identity", "developer-tools"] + )] + #[case::environment_client_secret_comes_first( + &[("AZURE_TENANT_ID", "tenant"), ("AZURE_CLIENT_ID", "client"), ("AZURE_CLIENT_SECRET", "secret")], + &["client-secret", "managed-identity", "developer-tools"] + )] + #[case::aks_workload_identity( + &[("AZURE_TENANT_ID", "tenant"), ("AZURE_CLIENT_ID", "client"), ("AZURE_FEDERATED_TOKEN_FILE", "/var/run/token")], + &["workload-identity", "managed-identity", "developer-tools"] + )] + #[tokio::test] + async fn the_default_identity_follows_azure_identity_defaults( + #[case] environment: &[(&str, &str)], + #[case] expected: &[&str], + ) { + let acquirer = Arc::new(ChainAcquirer { + terminal_on: None, + requests: Mutex::new(Vec::new()), + succeed_on: Some("developer-tools"), + }); + let service = AzureAuthService::with_acquirer(acquirer.clone()); + let lookup = |name: &str| { + environment + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| (*value).to_owned()) + }; + + let credential = service + .identity_token( + &super::AzureIdentity::Default, + "https://redis.azure.com/.default", + &lookup, + ) + .await + .unwrap() + .unwrap(); + + assert_eq!(credential.secret().expose(), "chain-token"); + assert_eq!(*acquirer.requests.lock().unwrap(), expected); + } + + #[rstest::rstest] #[tokio::test] async fn default_chain_uses_declared_order_and_stops_after_success() { let acquirer = Arc::new(ChainAcquirer { + terminal_on: None, requests: Mutex::new(Vec::new()), succeed_on: Some("developer-tools"), }); @@ -715,9 +1100,13 @@ mod tests { ); } + #[rstest::rstest] + #[case::all_unavailable(false)] + #[case::terminal_failure_after_unavailable(true)] #[tokio::test] - async fn chain_reports_each_acquisition_failure() { + async fn chain_reports_each_acquisition_failure(#[case] terminal: bool) { let acquirer = Arc::new(ChainAcquirer { + terminal_on: terminal.then_some("developer-tools"), requests: Mutex::new(Vec::new()), succeed_on: None, }); @@ -744,6 +1133,27 @@ mod tests { ); } + #[rstest::rstest] + #[tokio::test] + async fn a_configured_identity_failure_does_not_fall_back_to_a_developer_identity() { + let acquirer = Arc::new(ChainAcquirer { + terminal_on: Some("managed-identity"), + requests: Mutex::new(Vec::new()), + succeed_on: Some("developer-tools"), + }); + let service = AzureAuthService::with_acquirer(acquirer.clone()); + let inputs = AzureAuthInputs { + enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment), + ..Default::default() + }; + let error = service + .get_azure_ad_token(&inputs, &|_| None) + .await + .unwrap_err(); + assert_eq!(*acquirer.requests.lock().unwrap(), ["managed-identity"]); + assert!(matches!(error, Error::CredentialChain { earlier, .. } if earlier.is_empty())); + } + #[derive(Debug)] struct CallerToken(&'static str); diff --git a/litellm-rust/crates/auth-gcp/Cargo.toml b/litellm-rust/crates/auth-gcp/Cargo.toml index ad60f0db817..90aab8d281c 100644 --- a/litellm-rust/crates/auth-gcp/Cargo.toml +++ b/litellm-rust/crates/auth-gcp/Cargo.toml @@ -6,7 +6,7 @@ license.workspace = true repository.workspace = true [features] -google-sdk = ["dep:google-cloud-auth", "dep:http"] +google-sdk = ["dep:http"] [dependencies] litellm-secret-value.workspace = true @@ -21,10 +21,14 @@ tokio.workspace = true veil.workspace = true gcp_auth = "0.12.7" -google-cloud-auth = { workspace = true, optional = true } +google-cloud-auth.workspace = true http = { workspace = true, optional = true } [dev-dependencies] +time = { workspace = true, features = ["formatting"] } +axum = { workspace = true, features = ["json"] } litellm-error.workspace = true rstest.workspace = true +rustls.workspace = true tempfile.workspace = true +tokio = { workspace = true, features = ["test-util"] } diff --git a/litellm-rust/crates/auth-gcp/src/auth.rs b/litellm-rust/crates/auth-gcp/src/auth.rs index 650d5831796..5976ef82344 100644 --- a/litellm-rust/crates/auth-gcp/src/auth.rs +++ b/litellm-rust/crates/auth-gcp/src/auth.rs @@ -1,20 +1,11 @@ -use std::{future::Future, pin::Pin, sync::Arc}; +use std::sync::Arc; -use gcp_auth::{CustomServiceAccount, TokenProvider}; -use litellm_auth_types::{CredentialPlacement, Error, ErrorDetail, http::apply_credential}; -use moka::future::Cache; -use serde_json::Value; -use sha2::{Digest, Sha256}; +use litellm_auth_types::{CredentialPlacement, Error, http::apply_credential}; use crate::{ - config::{ - CredentialSource, VertexConfig, credential_source, get_vertex_ai_project, - is_base64_encoded_json, names_a_credentials_file, non_empty_env, - }, - constants::{ - CLOUD_PLATFORM_SCOPE, GOOGLE_OAUTH_TOKEN_ENDPOINT, VERTEX_AI_API_KEY_ENV, - VERTEXAI_API_KEY_ENV, - }, + config::{VertexConfig, credential_source, get_vertex_ai_project, non_empty_env}, + constants::{VERTEX_AI_API_KEY_ENV, VERTEXAI_API_KEY_ENV}, + identity::{GcpAuth, GcpProviderLoader, GcpTokenSource}, }; pub struct VertexEnvironment { @@ -22,28 +13,15 @@ pub struct VertexEnvironment { pub project_id: String, } -struct VertexAccessToken { - token: String, - project_id: String, -} - -#[derive(Clone)] +#[derive(Clone, Default)] pub struct VertexAuth { - providers: Cache>, - loader: Arc, -} - -impl Default for VertexAuth { - fn default() -> Self { - Self::new(Arc::new(GcpProviderLoader)) - } + identity: GcpAuth, } impl VertexAuth { - pub fn new(loader: Arc) -> Self { + pub fn new(loader: Arc) -> Self { Self { - providers: Cache::builder().max_capacity(64).build(), - loader, + identity: GcpAuth::new(loader), } } @@ -52,7 +30,10 @@ impl VertexAuth { config: &VertexConfig, env_lookup: &(dyn Fn(&str) -> Option + Sync), ) -> Result { - self.load_provider(config, env_lookup).await?.token().await + self.identity + .access_token(credential_source(config, env_lookup)) + .await + .map(|token| token.expose().to_owned()) } pub async fn validate_environment( @@ -77,10 +58,18 @@ impl VertexAuth { (true, _) => None, (false, Some(token)) => Some(token), (false, None) => { - let access = self.get_access_token(config, env_lookup).await?; + let provider = self.load_provider(config, env_lookup).await?; + let (token, project_id) = match project_id { + Some(project_id) => (provider.token().await?, project_id), + None => tokio::try_join!(provider.token(), provider.project_id())?, + }; return Ok(VertexEnvironment { - headers: apply_credential(headers, &access.token, CredentialPlacement::Bearer)?, - project_id: project_id.unwrap_or(access.project_id), + headers: apply_credential( + headers, + token.expose(), + CredentialPlacement::Bearer, + )?, + project_id, }); } }; @@ -104,248 +93,171 @@ impl VertexAuth { }) } - async fn get_access_token( - &self, - config: &VertexConfig, - env_lookup: &(dyn Fn(&str) -> Option + Sync), - ) -> Result { - let provider = self.load_provider(config, env_lookup).await?; - let (token, project_id) = tokio::try_join!(provider.token(), provider.project_id())?; - Ok(VertexAccessToken { token, project_id }) - } - async fn load_provider( &self, config: &VertexConfig, env_lookup: &(dyn Fn(&str) -> Option + Sync), - ) -> Result, Error> { - let source = credential_source(config, env_lookup); - let key = source.cache_key(); - self.providers - .try_get_with(key, self.loader.load(source)) + ) -> Result, Error> { + self.identity + .load_provider(credential_source(config, env_lookup)) .await - .map_err(|error| (*error).clone()) } } -pub trait VertexTokenSource: Send + Sync { - fn project_id(&self) -> VertexAuthFuture<'_, String>; - fn token(&self) -> VertexAuthFuture<'_, String>; -} - -pub trait VertexProviderLoader: Send + Sync { - fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc>; -} - -pub type VertexAuthFuture<'a, T> = Pin> + Send + 'a>>; - -struct GcpTokenSource(Arc); - -impl VertexTokenSource for GcpTokenSource { - fn project_id(&self) -> VertexAuthFuture<'_, String> { - Box::pin(async move { - self.0 - .project_id() - .await - .map(|project| project.to_string()) - .map_err(auth_acquisition_error) - }) - } - - fn token(&self) -> VertexAuthFuture<'_, String> { - Box::pin(async move { - self.0 - .token(&[CLOUD_PLATFORM_SCOPE]) - .await - .map(|token| token.as_str().to_string()) - .map_err(auth_acquisition_error) - }) - } -} - -struct GcpProviderLoader; - -impl VertexProviderLoader for GcpProviderLoader { - fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc> { - Box::pin(async move { - let provider: Arc = match source { - CredentialSource::Inline(configured) => Arc::new( - CustomServiceAccount::from_json(validate_request_credentials( - configured.expose(), - )?) - .map_err(auth_acquisition_error)?, - ), - CredentialSource::Trusted(configured) => { - Arc::new(load_trusted_service_account(configured.expose())?) - } - CredentialSource::ApplicationCredentials(path) => { - Arc::new(CustomServiceAccount::from_file(path).map_err(auth_acquisition_error)?) - } - CredentialSource::Adc => { - gcp_auth::provider().await.map_err(auth_acquisition_error)? - } - }; - Ok(Arc::new(GcpTokenSource(provider)) as Arc) - }) - } -} - -fn validate_request_credentials(configured: &str) -> Result<&str, Error> { - let token_uri = serde_json::from_str::(configured) - .ok() - .and_then(|credentials| { - credentials - .get("token_uri") - .and_then(Value::as_str) - .map(str::to_string) - }); - if token_uri.as_deref() != Some(GOOGLE_OAUTH_TOKEN_ENDPOINT) { - return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType { - field: "request-controlled vertex_credentials token_uri".into(), - expected: "the canonical Google OAuth token endpoint", - })); - } - Ok(configured) -} - -impl CredentialSource { - fn cache_key(&self) -> CredentialCacheKey { - match self { - Self::Inline(configured) => { - CredentialCacheKey::Inline(Sha256::digest(configured.expose()).into()) - } - Self::Trusted(configured) => { - CredentialCacheKey::Trusted(Sha256::digest(configured.expose()).into()) - } - Self::ApplicationCredentials(path) => { - CredentialCacheKey::ApplicationCredentials(path.clone()) - } - Self::Adc => CredentialCacheKey::Adc, - } - } -} - -#[derive(Clone, Debug, Hash, PartialEq, Eq)] -enum CredentialCacheKey { - Inline([u8; 32]), - Trusted([u8; 32]), - ApplicationCredentials(String), - Adc, -} - -fn load_trusted_service_account(configured: &str) -> Result { - if is_base64_encoded_json(configured) { - return Err(Error::InvalidConfiguration( - ErrorDetail::Base64EncodedJson { - field: "vertex_credentials", - }, - )); - } - if names_a_credentials_file(configured) { - CustomServiceAccount::from_file(configured) - } else { - CustomServiceAccount::from_json(configured) - } - .map_err(auth_acquisition_error) -} - -fn auth_acquisition_error(error: gcp_auth::Error) -> Error { - Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::failed( - "Vertex AI credentials", - error, - )) -} - #[cfg(test)] mod tests { + use std::{ + collections::{BTreeMap, BTreeSet}, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + }; + + use crate::{ + CredentialSource, GcpAuthFuture, GcpProviderLoader, GcpTokenSource, VertexAuth, + VertexConfig, get_vertex_ai_location, secret_names, + }; use litellm_secret_value::SecretValue; + use rstest::rstest; + use serde_json::{Value, json}; - use super::*; - - #[rstest::rstest] - #[case::canonical_endpoint(r#"{"token_uri":"https://oauth2.googleapis.com/token"}"#, true)] - #[case::noncanonical_endpoint(r#"{"token_uri":"http://127.0.0.1/token"}"#, false)] - #[case::missing_endpoint("{}", false)] - fn request_credentials_require_canonical_token_endpoint( - #[case] credentials: &str, - #[case] accepted: bool, - ) { - assert_eq!(validate_request_credentials(credentials).is_ok(), accepted); + struct FakeProvider { + project: Option<&'static str>, + calls: Arc, } - #[rstest::rstest] - #[case::missing_file( - "/nonexistent/vertex-key.json", - "failed to open application credentials file" - )] - #[case::malformed_json( - " {\"type\": \"service_account\",", - "failed to deserialize ApplicationCredentials" - )] - #[case::quoted_json( - "'{\"type\": \"service_account\"}'", - "failed to deserialize ApplicationCredentials" - )] - #[tokio::test] - async fn trusted_credentials_are_read_as_a_file_unless_they_are_json( - #[case] configured: &str, - #[case] expected: &str, - ) { - let Err(error) = GcpProviderLoader - .load(CredentialSource::Trusted(SecretValue::new(configured))) - .await - else { - panic!("{configured:?} should not load"); - }; - let report = litellm_error::Report::new(&error).to_string(); - assert!(report.contains(expected), "{report}"); - } + impl GcpTokenSource for FakeProvider { + fn project_id(&self) -> GcpAuthFuture<'_, String> { + self.calls.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { + self.project.map(str::to_owned).ok_or( + litellm_auth_types::Error::CredentialAcquisition( + litellm_auth_types::ErrorDetail::Unresolved { + subject: "credential project", + }, + ), + ) + }) + } - #[rstest::rstest] - #[case::padded("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9")] - #[case::quoted_prefixed_unpadded("\"base64:eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9\"")] - #[case::url_safe_wrapped("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsICJr\nZXkiOiAiPz8_In0=")] - #[tokio::test] - async fn base64_encoded_credentials_are_named_as_base64(#[case] configured: &str) { - assert!( - trusted_load_error(configured) - .await - .contains("is base64-encoded JSON, not the JSON itself") - ); - } - - async fn trusted_load_error(configured: &str) -> String { - match GcpProviderLoader - .load(CredentialSource::Trusted(SecretValue::new(configured))) - .await - { - Ok(_) => panic!("{configured:?} should not load"), - Err(error) => litellm_error::Report::new(&error).to_string(), + fn token(&self) -> GcpAuthFuture<'_, SecretValue> { + self.calls.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok(SecretValue::new("adc-token")) }) } } - #[rstest::rstest] - #[tokio::test] - async fn a_relative_key_file_named_like_json_is_read_as_a_file() { - let key = tempfile::Builder::new() - .prefix("{vertex") - .suffix(".json") - .tempfile_in(".") - .unwrap(); - std::fs::write(key.path(), r#"{"project_id":"file-project"}"#).unwrap(); - let relative_name = key.path().file_name().unwrap().to_str().unwrap(); - let absolute_path = std::fs::canonicalize(key.path()).unwrap(); - assert_eq!( - trusted_load_error(relative_name).await, - trusted_load_error(absolute_path.to_str().unwrap()).await - ); + struct FakeLoader { + loads: Arc, + provider: Arc, } - #[rstest::rstest] - fn inline_and_trusted_credentials_never_share_a_cache_entry() { - assert_ne!( - CredentialSource::Inline(SecretValue::new("same-value")).cache_key(), - CredentialSource::Trusted(SecretValue::new("same-value")).cache_key() - ); + impl GcpProviderLoader for FakeLoader { + fn load(&self, _source: CredentialSource) -> GcpAuthFuture<'_, Arc> { + let loads = self.loads.clone(); + let provider = self.provider.clone(); + Box::pin(async move { + loads.fetch_add(1, Ordering::SeqCst); + Ok(provider) + }) + } + } + + fn config(value: Value) -> VertexConfig { + VertexConfig::from_sourced_optional_params(value.as_object().unwrap(), &BTreeMap::new()) + .unwrap() + } + + fn auth(calls: Arc, loads: Arc) -> VertexAuth { + let provider: Arc = Arc::new(FakeProvider { + calls, + project: Some("adc-project"), + }); + VertexAuth::new(Arc::new(FakeLoader { loads, provider })) + } + + #[rstest] + #[tokio::test] + async fn secret_names_cover_environment_reads() { + let seen = Arc::new(std::sync::Mutex::new(BTreeSet::::new())); + let recorded = seen.clone(); + let env = |name: &str| { + recorded.lock().unwrap().insert(name.to_string()); + None + }; + let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); + auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env) + .await + .unwrap(); + get_vertex_ai_location(&VertexConfig::default(), &env); + let allowed = secret_names() + .iter() + .map(|name| (*name).to_owned()) + .collect::>(); + assert!(seen.lock().unwrap().is_subset(&allowed)); + } + + #[rstest] + #[tokio::test] + async fn explicit_token_and_header_do_not_acquire_adc() { + let loads = Arc::new(AtomicUsize::new(0)); + let auth = auth(Arc::new(AtomicUsize::new(0)), loads.clone()); + let configured = config(json!({"vertex_project":"project-1"})); + let explicit = auth + .validate_environment(Vec::new(), Some("access-token"), &configured, &|_| None) + .await + .unwrap(); + assert_eq!(explicit.headers[0].1, "Bearer access-token"); + let existing = auth + .validate_environment( + vec![("authorization".into(), "Bearer existing".into())], + None, + &configured, + &|_| None, + ) + .await + .unwrap(); + assert_eq!(existing.headers[0].1, "Bearer existing"); + assert_eq!(loads.load(Ordering::SeqCst), 0); + } + + #[rstest] + #[tokio::test] + async fn provider_is_reused_across_authentication_calls() { + let calls = Arc::new(AtomicUsize::new(0)); + let loads = Arc::new(AtomicUsize::new(0)); + let auth = auth(calls.clone(), loads.clone()); + let config = VertexConfig::default(); + let environment = |_: &str| None; + let (first, second) = tokio::try_join!( + auth.validate_environment(Vec::new(), None, &config, &environment), + auth.validate_environment(Vec::new(), None, &config, &environment), + ) + .unwrap(); + assert_eq!(first.project_id, "adc-project"); + assert_eq!(first.headers[0].1, "Bearer adc-token"); + assert_eq!(first.project_id, second.project_id); + assert_eq!(first.headers, second.headers); + assert_eq!(loads.load(Ordering::SeqCst), 1); + assert_eq!(calls.load(Ordering::SeqCst), 4); + } + + #[rstest] + #[tokio::test] + async fn an_explicit_project_does_not_require_project_metadata_in_the_credentials() { + let auth = VertexAuth::new(Arc::new(FakeLoader { + loads: Arc::new(AtomicUsize::new(0)), + provider: Arc::new(FakeProvider { + calls: Arc::new(AtomicUsize::new(0)), + project: None, + }), + })); + let configured = VertexConfig::new(None, Some("configured-project".into()), None); + let environment = auth + .validate_environment(Vec::new(), None, &configured, &|_| None) + .await + .unwrap(); + assert_eq!(environment.project_id, "configured-project"); + assert_eq!(environment.headers[0].1, "Bearer adc-token"); } } diff --git a/litellm-rust/crates/auth-gcp/src/config.rs b/litellm-rust/crates/auth-gcp/src/config.rs index 133e1c26903..048e40b0a2f 100644 --- a/litellm-rust/crates/auth-gcp/src/config.rs +++ b/litellm-rust/crates/auth-gcp/src/config.rs @@ -146,6 +146,20 @@ pub enum CredentialSource { Adc, } +impl CredentialSource { + pub fn from_vertex_environment(env_lookup: &dyn Fn(&str) -> Option) -> Self { + VertexParams::CREDENTIALS + .resolve(&|_| None, env_lookup) + .map(|configured| Self::Trusted(SecretValue::new(configured))) + .unwrap_or_else(|| Self::application_default(env_lookup)) + } + + pub fn application_default(env_lookup: &dyn Fn(&str) -> Option) -> Self { + non_empty_env(env_lookup, GOOGLE_APPLICATION_CREDENTIALS_ENV) + .map_or(Self::Adc, Self::ApplicationCredentials) + } +} + pub(crate) fn credential_source( config: &VertexConfig, env_lookup: &dyn Fn(&str) -> Option, @@ -158,12 +172,7 @@ pub(crate) fn credential_source( } }; } - if let Some(configured) = VertexParams::CREDENTIALS.resolve(&|_| None, env_lookup) { - return CredentialSource::Trusted(SecretValue::new(configured)); - } - non_empty_env(env_lookup, GOOGLE_APPLICATION_CREDENTIALS_ENV) - .map(CredentialSource::ApplicationCredentials) - .unwrap_or(CredentialSource::Adc) + CredentialSource::from_vertex_environment(env_lookup) } fn optional_credentials( @@ -319,6 +328,24 @@ mod tests { )); } + #[rstest::rstest] + fn application_default_ignores_vertex_credentials() { + let env = |name: &str| match name { + VERTEXAI_CREDENTIALS_ENV => Some("environment-json".to_string()), + GOOGLE_APPLICATION_CREDENTIALS_ENV => Some("adc.json".to_string()), + _ => None, + }; + assert!( + matches!(CredentialSource::application_default(&env), CredentialSource::ApplicationCredentials(path) if path == "adc.json") + ); + assert!(matches!( + CredentialSource::application_default( + &|name| (name == VERTEXAI_CREDENTIALS_ENV).then(|| "environment-json".into()) + ), + CredentialSource::Adc + )); + } + #[rstest::rstest] fn params_become_a_deployment_sourced_config() { let params = VertexParams { diff --git a/litellm-rust/crates/auth-gcp/src/identity.rs b/litellm-rust/crates/auth-gcp/src/identity.rs new file mode 100644 index 00000000000..5144221b218 --- /dev/null +++ b/litellm-rust/crates/auth-gcp/src/identity.rs @@ -0,0 +1,445 @@ +use crate::{ + config::{CredentialSource, is_base64_encoded_json, names_a_credentials_file}, + constants::{CLOUD_PLATFORM_SCOPE, GOOGLE_OAUTH_TOKEN_ENDPOINT}, +}; +use gcp_auth::{CustomServiceAccount, TokenProvider}; +use litellm_auth_types::{Error, ErrorDetail}; +use litellm_secret_value::SecretValue; +use moka::future::Cache; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::{future::Future, pin::Pin, sync::Arc}; + +#[derive(Clone)] +pub struct GcpAuth { + providers: Cache>, + loader: Arc, +} + +impl Default for GcpAuth { + fn default() -> Self { + Self::new(Arc::new(NativeProviderLoader)) + } +} + +impl GcpAuth { + pub fn new(loader: Arc) -> Self { + Self { + providers: Cache::builder().max_capacity(64).build(), + loader, + } + } + + pub async fn access_token(&self, source: CredentialSource) -> Result { + self.load_provider(source).await?.token().await + } + + pub(crate) async fn load_provider( + &self, + source: CredentialSource, + ) -> Result, Error> { + self.providers + .try_get_with(source.cache_key(), self.loader.load(source)) + .await + .map_err(|error| (*error).clone()) + } +} + +pub trait GcpTokenSource: Send + Sync { + fn project_id(&self) -> GcpAuthFuture<'_, String>; + fn token(&self) -> GcpAuthFuture<'_, SecretValue>; +} + +pub trait GcpProviderLoader: Send + Sync { + fn load(&self, source: CredentialSource) -> GcpAuthFuture<'_, Arc>; +} + +pub type GcpAuthFuture<'a, T> = Pin> + Send + 'a>>; + +struct ServiceAccountTokenSource(Arc); + +impl GcpTokenSource for ServiceAccountTokenSource { + fn project_id(&self) -> GcpAuthFuture<'_, String> { + Box::pin(async move { + self.0 + .project_id() + .await + .map(|project| project.to_string()) + .map_err(auth_acquisition_error) + }) + } + + fn token(&self) -> GcpAuthFuture<'_, SecretValue> { + Box::pin(async move { + self.0 + .token(&[CLOUD_PLATFORM_SCOPE]) + .await + .map(|token| SecretValue::new(token.as_str())) + .map_err(auth_acquisition_error) + }) + } +} + +struct SdkTokenSource { + credentials: google_cloud_auth::credentials::AccessTokenCredentials, + project_id: Option, +} + +impl GcpTokenSource for SdkTokenSource { + fn project_id(&self) -> GcpAuthFuture<'_, String> { + Box::pin(async move { + self.project_id + .clone() + .ok_or(Error::CredentialAcquisition(ErrorDetail::Unresolved { + subject: "GCP resource project", + })) + }) + } + + fn token(&self) -> GcpAuthFuture<'_, SecretValue> { + Box::pin(async move { + self.credentials + .access_token() + .await + .map(|token| SecretValue::new(token.token)) + .map_err(auth_acquisition_error) + }) + } +} + +async fn load_application_credentials(path: &str) -> Result, Error> { + let contents = tokio::fs::read_to_string(path) + .await + .map_err(auth_acquisition_error)?; + load_credentials_json(&contents) +} + +fn load_credentials_json(contents: &str) -> Result, Error> { + let configured: Value = serde_json::from_str(contents).map_err(auth_acquisition_error)?; + let kind = configured.get("type").and_then(Value::as_str); + let project_id = configured + .get("project_id") + .and_then(Value::as_str) + .map(str::to_string); + let credentials = match kind { + Some("authorized_user") => { + google_cloud_auth::credentials::user_account::Builder::new(configured) + .with_scopes([CLOUD_PLATFORM_SCOPE]) + .build_access_token_credentials() + } + Some("impersonated_service_account") => { + google_cloud_auth::credentials::impersonated::Builder::new(configured) + .with_scopes([CLOUD_PLATFORM_SCOPE]) + .build_access_token_credentials() + } + Some("external_account") => { + google_cloud_auth::credentials::external_account::Builder::new(configured) + .with_scopes([CLOUD_PLATFORM_SCOPE]) + .build_access_token_credentials() + } + Some("service_account") | None => { + let provider = + CustomServiceAccount::from_json(contents).map_err(auth_acquisition_error)?; + return Ok(Arc::new(ServiceAccountTokenSource(Arc::new(provider)))); + } + Some(_) => { + return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType { + field: "Google application credentials type".into(), + expected: "a supported Google credential type", + })); + } + } + .map_err(auth_acquisition_error)?; + Ok(Arc::new(SdkTokenSource { + credentials, + project_id, + })) +} + +struct NativeProviderLoader; + +impl GcpProviderLoader for NativeProviderLoader { + fn load(&self, source: CredentialSource) -> GcpAuthFuture<'_, Arc> { + Box::pin(async move { + let provider: Arc = match source { + CredentialSource::Inline(configured) => Arc::new( + CustomServiceAccount::from_json(validate_request_credentials( + configured.expose(), + )?) + .map_err(auth_acquisition_error)?, + ), + CredentialSource::Trusted(configured) => { + return load_trusted_credentials(configured.expose()).await; + } + CredentialSource::ApplicationCredentials(path) => { + return load_application_credentials(&path).await; + } + CredentialSource::Adc => { + gcp_auth::provider().await.map_err(auth_acquisition_error)? + } + }; + Ok(Arc::new(ServiceAccountTokenSource(provider)) as Arc) + }) + } +} + +fn validate_request_credentials(configured: &str) -> Result<&str, Error> { + let token_uri = serde_json::from_str::(configured) + .ok() + .and_then(|credentials| { + credentials + .get("token_uri") + .and_then(Value::as_str) + .map(str::to_string) + }); + if token_uri.as_deref() != Some(GOOGLE_OAUTH_TOKEN_ENDPOINT) { + return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType { + field: "request-controlled vertex_credentials token_uri".into(), + expected: "the canonical Google OAuth token endpoint", + })); + } + Ok(configured) +} + +impl CredentialSource { + fn cache_key(&self) -> CredentialCacheKey { + match self { + Self::Inline(configured) => { + CredentialCacheKey::Inline(Sha256::digest(configured.expose()).into()) + } + Self::Trusted(configured) => { + CredentialCacheKey::Trusted(Sha256::digest(configured.expose()).into()) + } + Self::ApplicationCredentials(path) => { + CredentialCacheKey::ApplicationCredentials(path.clone()) + } + Self::Adc => CredentialCacheKey::Adc, + } + } +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +enum CredentialCacheKey { + Inline([u8; 32]), + Trusted([u8; 32]), + ApplicationCredentials(String), + Adc, +} + +async fn load_trusted_credentials(configured: &str) -> Result, Error> { + if is_base64_encoded_json(configured) { + return Err(Error::InvalidConfiguration( + ErrorDetail::Base64EncodedJson { + field: "vertex_credentials", + }, + )); + } + if names_a_credentials_file(configured) { + load_application_credentials(configured).await + } else { + load_credentials_json(configured) + } +} + +fn auth_acquisition_error(error: impl std::error::Error + Send + Sync + 'static) -> Error { + Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::failed( + "Google credentials", + error, + )) +} + +#[cfg(test)] +mod tests { + use litellm_secret_value::SecretValue; + + use super::*; + + #[rstest::rstest] + #[case::authorized_user("authorized_user", 2)] + #[case::external_account("external_account", 2)] + #[case::impersonated_service_account("impersonated_service_account", 4)] + #[tokio::test] + async fn explicit_application_credentials_refresh_expired_tokens( + #[case] kind: &str, + #[case] expected_requests: usize, + #[values(false, true)] inline: bool, + ) { + use axum::{Json, Router, routing::post}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + let _ = rustls::crypto::ring::default_provider().install_default(); + let requests = Arc::new(AtomicUsize::new(0)); + let recorded = requests.clone(); + let impersonations = requests.clone(); + let app = Router::new().route("/token", post(move || { + let recorded = recorded.clone(); + async move { + Json(serde_json::json!({ + "access_token": format!("token-{}", recorded.fetch_add(1, Ordering::SeqCst)), + "expires_in": 3600, + "token_type": "Bearer", + "issued_token_type": "urn:ietf:params:oauth:token-type:access_token", + })) + } + })); + let app = app.route("/v1/projects/-/serviceAccounts/test-principal:generateAccessToken", post(move || { + let recorded = impersonations.clone(); + async move { + let expiration = (time::OffsetDateTime::now_utc() + time::Duration::hours(1)).format(&time::format_description::well_known::Rfc3339).unwrap(); + Json(serde_json::json!({ + "accessToken": format!("impersonated-{}", recorded.fetch_add(1, Ordering::SeqCst)), + "expireTime": expiration, + })) + } + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let file = tempfile::NamedTempFile::new().unwrap(); + let subject_token = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(subject_token.path(), "subject-token").unwrap(); + let configured = if kind == "external_account" { + serde_json::json!({ + "type": "external_account", "audience": "//iam.googleapis.com/projects/test/locations/global/workloadIdentityPools/test/providers/test", + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", "token_url": format!("http://{address}/token"), + "credential_source": { "file": subject_token.path().to_str().unwrap() }, "quota_project_id": "quota-project" + }) + } else { + let source = serde_json::json!({ + "type": "authorized_user", "client_id": "user-client", "client_secret": "user-secret", + "refresh_token": "user-refresh", "quota_project_id": "quota-project", "token_uri": format!("http://{address}/token") + }); + if kind == "impersonated_service_account" { + serde_json::json!({ + "type": kind, "source_credentials": source, "quota_project_id": "quota-project", + "service_account_impersonation_url": format!("http://{address}/v1/projects/-/serviceAccounts/test-principal:generateAccessToken") + }) + } else { + source + } + }; + std::fs::write(file.path(), configured.to_string()).unwrap(); + let source = if inline { + CredentialSource::Trusted(SecretValue::new(configured.to_string())) + } else { + CredentialSource::ApplicationCredentials(file.path().to_str().unwrap().into()) + }; + let provider = NativeProviderLoader.load(source).await.unwrap(); + + assert!(matches!( + provider.project_id().await, + Err(Error::CredentialAcquisition(ErrorDetail::Unresolved { + subject: "GCP resource project" + })) + )); + let original = provider.token().await.unwrap(); + tokio::time::pause(); + tokio::time::advance(std::time::Duration::from_secs(3601)).await; + tokio::time::resume(); + let refreshed = provider.token().await.unwrap(); + assert_ne!(original, refreshed); + assert_eq!(requests.load(Ordering::SeqCst), expected_requests); + server.abort(); + } + + #[rstest::rstest] + #[case::canonical_endpoint(r#"{"token_uri":"https://oauth2.googleapis.com/token"}"#, true)] + #[case::noncanonical_endpoint(r#"{"token_uri":"http://127.0.0.1/token"}"#, false)] + #[case::missing_endpoint("{}", false)] + fn request_credentials_require_canonical_token_endpoint( + #[case] credentials: &str, + #[case] accepted: bool, + ) { + assert_eq!(validate_request_credentials(credentials).is_ok(), accepted); + } + + #[rstest::rstest] + #[case::missing_file("/nonexistent/vertex-key.json", "Google credentials")] + #[case::malformed_json(" {\"type\": \"service_account\",", "Google credentials")] + #[case::quoted_json("'{\"type\": \"service_account\"}'", "Google credentials")] + #[tokio::test] + async fn trusted_credentials_are_read_as_a_file_unless_they_are_json( + #[case] configured: &str, + #[case] expected: &str, + ) { + let Err(error) = NativeProviderLoader + .load(CredentialSource::Trusted(SecretValue::new(configured))) + .await + else { + panic!("{configured:?} should not load"); + }; + let report = litellm_error::Report::new(&error).to_string(); + assert!(report.contains(expected), "{report}"); + } + + #[rstest::rstest] + #[case::padded("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9")] + #[case::quoted_prefixed_unpadded("\"base64:eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9\"")] + #[case::url_safe_wrapped("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsICJr\nZXkiOiAiPz8_In0=")] + #[tokio::test] + async fn base64_encoded_credentials_are_named_as_base64(#[case] configured: &str) { + assert!( + trusted_load_error(configured) + .await + .contains("is base64-encoded JSON, not the JSON itself") + ); + } + + async fn trusted_load_error(configured: &str) -> String { + match NativeProviderLoader + .load(CredentialSource::Trusted(SecretValue::new(configured))) + .await + { + Ok(_) => panic!("{configured:?} should not load"), + Err(error) => litellm_error::Report::new(&error).to_string(), + } + } + + #[rstest::rstest] + #[tokio::test] + async fn a_relative_key_file_named_like_json_is_read_as_a_file() { + let key = tempfile::Builder::new() + .prefix("{vertex") + .suffix(".json") + .tempfile_in(".") + .unwrap(); + std::fs::write(key.path(), r#"{"project_id":"file-project"}"#).unwrap(); + let relative_name = key.path().file_name().unwrap().to_str().unwrap(); + let absolute_path = std::fs::canonicalize(key.path()).unwrap(); + assert_eq!( + trusted_load_error(relative_name).await, + trusted_load_error(absolute_path.to_str().unwrap()).await + ); + } + + #[rstest::rstest] + #[tokio::test] + async fn unknown_application_credential_types_are_rejected_without_service_account_fallback() { + let file = tempfile::NamedTempFile::new().unwrap(); + std::fs::write( + file.path(), + r#"{"type":"unsupported","private_key":"credential-value"}"#, + ) + .unwrap(); + let result = GcpAuth::default() + .access_token(CredentialSource::ApplicationCredentials( + file.path().to_str().unwrap().into(), + )) + .await; + let error = result.unwrap_err(); + assert!(matches!(error, Error::InvalidConfiguration(_))); + assert!( + !litellm_error::Report::new(&error) + .to_string() + .contains("credential-value") + ); + } + + #[rstest::rstest] + fn inline_and_trusted_credentials_never_share_a_cache_entry() { + assert_ne!( + CredentialSource::Inline(SecretValue::new("same-value")).cache_key(), + CredentialSource::Trusted(SecretValue::new("same-value")).cache_key() + ); + } +} diff --git a/litellm-rust/crates/auth-gcp/src/lib.rs b/litellm-rust/crates/auth-gcp/src/lib.rs index 2b389d45926..03c660e6eb8 100644 --- a/litellm-rust/crates/auth-gcp/src/lib.rs +++ b/litellm-rust/crates/auth-gcp/src/lib.rs @@ -1,16 +1,16 @@ mod auth; mod config; pub mod constants; +mod identity; #[cfg(feature = "google-sdk")] mod sdk; -pub use auth::{ - VertexAuth, VertexAuthFuture, VertexEnvironment, VertexProviderLoader, VertexTokenSource, -}; +pub use auth::{VertexAuth, VertexEnvironment}; pub use config::{ CredentialSource, VertexConfig, get_vertex_ai_location, get_vertex_ai_project, get_vertex_ai_project_from_credentials, }; pub use constants::secret_names; +pub use identity::{GcpAuth, GcpAuthFuture, GcpProviderLoader, GcpTokenSource}; #[cfg(feature = "google-sdk")] pub use sdk::GoogleCredentials; diff --git a/litellm-rust/crates/auth-gcp/tests/vertex_auth.rs b/litellm-rust/crates/auth-gcp/tests/vertex_auth.rs deleted file mode 100644 index 5991749ab8a..00000000000 --- a/litellm-rust/crates/auth-gcp/tests/vertex_auth.rs +++ /dev/null @@ -1,120 +0,0 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - }, -}; - -use litellm_auth_gcp::{ - CredentialSource, VertexAuth, VertexAuthFuture, VertexConfig, VertexProviderLoader, - VertexTokenSource, get_vertex_ai_location, secret_names, -}; -use rstest::rstest; -use serde_json::{Value, json}; - -struct FakeProvider { - calls: Arc, -} - -impl VertexTokenSource for FakeProvider { - fn project_id(&self) -> VertexAuthFuture<'_, String> { - self.calls.fetch_add(1, Ordering::SeqCst); - Box::pin(async { Ok("adc-project".into()) }) - } - - fn token(&self) -> VertexAuthFuture<'_, String> { - self.calls.fetch_add(1, Ordering::SeqCst); - Box::pin(async { Ok("adc-token".into()) }) - } -} - -struct FakeLoader { - loads: Arc, - provider: Arc, -} - -impl VertexProviderLoader for FakeLoader { - fn load(&self, _source: CredentialSource) -> VertexAuthFuture<'_, Arc> { - let loads = self.loads.clone(); - let provider = self.provider.clone(); - Box::pin(async move { - loads.fetch_add(1, Ordering::SeqCst); - Ok(provider) - }) - } -} - -fn config(value: Value) -> VertexConfig { - VertexConfig::from_sourced_optional_params(value.as_object().unwrap(), &BTreeMap::new()) - .unwrap() -} - -fn auth(calls: Arc, loads: Arc) -> VertexAuth { - let provider: Arc = Arc::new(FakeProvider { calls }); - VertexAuth::new(Arc::new(FakeLoader { loads, provider })) -} - -#[rstest] -#[tokio::test] -async fn secret_names_cover_environment_reads() { - let seen = Arc::new(std::sync::Mutex::new(BTreeSet::::new())); - let recorded = seen.clone(); - let env = |name: &str| { - recorded.lock().unwrap().insert(name.to_string()); - None - }; - let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); - auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env) - .await - .unwrap(); - get_vertex_ai_location(&VertexConfig::default(), &env); - assert!( - seen.lock() - .unwrap() - .iter() - .all(|name| secret_names().contains(&name.as_str())) - ); -} - -#[rstest] -#[tokio::test] -async fn explicit_token_and_header_do_not_acquire_adc() { - let loads = Arc::new(AtomicUsize::new(0)); - let auth = auth(Arc::new(AtomicUsize::new(0)), loads.clone()); - let configured = config(json!({"vertex_project":"project-1"})); - let explicit = auth - .validate_environment(Vec::new(), Some("access-token"), &configured, &|_| None) - .await - .unwrap(); - assert_eq!(explicit.headers[0].1, "Bearer access-token"); - let existing = auth - .validate_environment( - vec![("authorization".into(), "Bearer existing".into())], - None, - &configured, - &|_| None, - ) - .await - .unwrap(); - assert_eq!(existing.headers[0].1, "Bearer existing"); - assert_eq!(loads.load(Ordering::SeqCst), 0); -} - -#[rstest] -#[tokio::test] -async fn provider_is_reused_across_authentication_calls() { - let calls = Arc::new(AtomicUsize::new(0)); - let loads = Arc::new(AtomicUsize::new(0)); - let auth = auth(calls.clone(), loads.clone()); - for _ in 0..2 { - let environment = auth - .validate_environment(Vec::new(), None, &VertexConfig::default(), &|_| None) - .await - .unwrap(); - assert_eq!(environment.project_id, "adc-project"); - assert_eq!(environment.headers[0].1, "Bearer adc-token"); - } - assert_eq!(loads.load(Ordering::SeqCst), 1); - assert_eq!(calls.load(Ordering::SeqCst), 4); -} diff --git a/litellm-rust/crates/auth-types/src/error.rs b/litellm-rust/crates/auth-types/src/error.rs index a8ba7b64e03..a2e5629bf3f 100644 --- a/litellm-rust/crates/auth-types/src/error.rs +++ b/litellm-rust/crates/auth-types/src/error.rs @@ -92,6 +92,11 @@ impl Error { #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum ErrorDetail { + #[error("{operation} returned HTTP {status}")] + UnexpectedStatus { + operation: &'static str, + status: u16, + }, #[error("{field} must be {expected}")] InvalidType { field: String, diff --git a/litellm-rust/crates/auth-types/src/params/aws.rs b/litellm-rust/crates/auth-types/src/params/aws.rs index 8075fc15e9f..de21fbf94c9 100644 --- a/litellm-rust/crates/auth-types/src/params/aws.rs +++ b/litellm-rust/crates/auth-types/src/params/aws.rs @@ -53,6 +53,9 @@ pub struct AwsParams { } impl AwsParams { + pub const ROLE_ARN_ENV: &str = "AWS_ROLE_ARN"; + pub const WEB_IDENTITY_TOKEN_FILE_ENV: &str = "AWS_WEB_IDENTITY_TOKEN_FILE"; + pub const ACCESS_KEY_ID: ParamSpec = spec( "access key id", ParamKind::Credential, @@ -149,6 +152,10 @@ impl AwsParams { AwsParams::SPECS .iter() .flat_map(|spec| spec.env.iter().copied()) + .chain([ + AwsParams::ROLE_ARN_ENV, + AwsParams::WEB_IDENTITY_TOKEN_FILE_ENV, + ]) .fold(Vec::new(), |mut names, name| { if !names.contains(&name) { names.push(name); diff --git a/litellm-rust/crates/auth-types/tests/aws_params.rs b/litellm-rust/crates/auth-types/tests/aws_params.rs index ed9a4f78ec2..b438984c199 100644 --- a/litellm-rust/crates/auth-types/tests/aws_params.rs +++ b/litellm-rust/crates/auth-types/tests/aws_params.rs @@ -3,21 +3,13 @@ use rstest::rstest; use serde_json::{Map, Value}; #[rstest] -fn fields_name_every_param_once_and_in_declaration_order() { +fn every_named_param_roundtrips_without_loss() { let filled: Map = AwsParams::fields() .map(|name| (name.to_string(), Value::from(format!("value-of-{name}")))) .collect(); let typed = AwsParams::from_optional_params(&filled); let serialized = serde_json::to_value(&typed).unwrap(); - assert_eq!( - serialized - .as_object() - .unwrap() - .keys() - .map(String::as_str) - .collect::>(), - AwsParams::fields().collect::>() - ); + assert_eq!(filled.len(), AwsParams::fields().count()); assert_eq!(serialized, Value::Object(filled.clone())); assert_eq!( serde_json::from_value::(Value::Object(filled)).unwrap(), diff --git a/litellm-rust/crates/cache-gcs/src/cache.rs b/litellm-rust/crates/cache-gcs/src/cache.rs index a6b08445bc8..b8c6720c234 100644 --- a/litellm-rust/crates/cache-gcs/src/cache.rs +++ b/litellm-rust/crates/cache-gcs/src/cache.rs @@ -8,7 +8,7 @@ use litellm_cache::{ use litellm_http::Client; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode}; -use crate::{GcpTokenSource, TokenSource}; +use crate::{GcsTokenSource, TokenSource}; pub const DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com"; @@ -62,7 +62,7 @@ pub struct GcsCache { impl GcsCache { pub fn new(config: GcsConfig, client: Client, codec: S) -> Self { - let token = Arc::new(GcpTokenSource::new(config.path_service_account.clone())); + let token = Arc::new(GcsTokenSource::new(config.path_service_account.clone())); Self::with_token_source(config, client, codec, token) } diff --git a/litellm-rust/crates/cache-gcs/src/lib.rs b/litellm-rust/crates/cache-gcs/src/lib.rs index cbb61cf0685..d40bca081a4 100644 --- a/litellm-rust/crates/cache-gcs/src/lib.rs +++ b/litellm-rust/crates/cache-gcs/src/lib.rs @@ -2,4 +2,4 @@ mod cache; mod token; pub use cache::{DEFAULT_ENDPOINT, GcsCache, GcsConfig, key_prefix}; -pub use token::{GcpTokenSource, StaticTokenSource, TokenSource}; +pub use token::{GcsTokenSource, StaticTokenSource, TokenSource}; diff --git a/litellm-rust/crates/cache-gcs/src/token.rs b/litellm-rust/crates/cache-gcs/src/token.rs index 852d8bbf035..4f639e7b865 100644 --- a/litellm-rust/crates/cache-gcs/src/token.rs +++ b/litellm-rust/crates/cache-gcs/src/token.rs @@ -1,7 +1,6 @@ use std::{future::Future, pin::Pin}; -use litellm_auth_gcp::{VertexAuth, VertexConfig}; -use litellm_auth_types::{InputSource, Sourced}; +use litellm_auth_gcp::{CredentialSource, GcpAuth}; use litellm_secret_value::SecretValue; use litellm_cache::Error; @@ -10,28 +9,30 @@ pub trait TokenSource: Send + Sync + 'static { fn bearer_token(&self) -> Pin> + Send + '_>>; } -pub struct GcpTokenSource { - auth: VertexAuth, - config: VertexConfig, +pub struct GcsTokenSource { + auth: GcpAuth, + credentials: Option, } -impl GcpTokenSource { +impl GcsTokenSource { pub fn new(path_service_account: Option) -> Self { - let credentials = path_service_account - .map(|path| Sourced::new(SecretValue::new(path), InputSource::Deployment)); Self { - auth: VertexAuth::default(), - config: VertexConfig::new(credentials, None, None), + auth: GcpAuth::default(), + credentials: path_service_account.filter(|path| !path.trim().is_empty()), } } } -impl TokenSource for GcpTokenSource { +impl TokenSource for GcsTokenSource { fn bearer_token(&self) -> Pin> + Send + '_>> { Box::pin(async move { self.auth - .access_token(&self.config, &|name| std::env::var(name).ok()) + .access_token(match &self.credentials { + Some(path) => CredentialSource::Trusted(SecretValue::new(path.clone())), + None => CredentialSource::application_default(&|name| std::env::var(name).ok()), + }) .await + .map(|token| token.expose().to_owned()) .map_err(Error::unavailable) }) } diff --git a/litellm-rust/crates/cache-memory/src/cache.rs b/litellm-rust/crates/cache-memory/src/cache.rs index 73fbf14b07a..809ffe3044e 100644 --- a/litellm-rust/crates/cache-memory/src/cache.rs +++ b/litellm-rust/crates/cache-memory/src/cache.rs @@ -2,7 +2,10 @@ use std::{ cmp::Reverse, collections::{BinaryHeap, HashMap, HashSet}, hash::Hash, - sync::{Arc, Mutex, MutexGuard}, + sync::{ + Arc, Mutex, MutexGuard, + atomic::{AtomicUsize, Ordering}, + }, time::{Duration, SystemTime, UNIX_EPOCH}, }; @@ -31,7 +34,7 @@ struct CacheState { pub struct InMemoryCache { state: Mutex>, - max_size_in_memory: usize, + max_size_in_memory: AtomicUsize, default_ttl: Duration, max_entry_bytes: Option, measure_value: Option>, @@ -74,7 +77,9 @@ impl InMemoryCache { expirations: HashMap::new(), expiration_heap: BinaryHeap::new(), }), - max_size_in_memory: max_size_in_memory.unwrap_or(DEFAULT_MAX_SIZE_IN_MEMORY), + max_size_in_memory: AtomicUsize::new( + max_size_in_memory.unwrap_or(DEFAULT_MAX_SIZE_IN_MEMORY), + ), default_ttl: default_ttl .filter(|ttl| !ttl.is_zero()) .unwrap_or(DEFAULT_TTL), @@ -94,7 +99,7 @@ impl InMemoryCache { value: V, ttl: Option, ) -> Result { - if self.max_size_in_memory == 0 { + if self.max_size_in_memory() == 0 { return Ok(CacheWrite::Disabled); } let now = (self.now)(); @@ -129,7 +134,7 @@ impl InMemoryCache { pub fn evict_cache(&self) -> Result<(), Error> { let now = (self.now)(); let mut state = self.lock()?; - Self::evict(&mut state, self.max_size_in_memory, now, None); + Self::evict(&mut state, self.max_size_in_memory(), now, None); Ok(()) } @@ -170,7 +175,16 @@ impl InMemoryCache { } pub fn max_size_in_memory(&self) -> usize { - self.max_size_in_memory + self.max_size_in_memory.load(Ordering::Relaxed) + } + + /// `DualCache.update_in_memory_max_size`: `None` restores the default. Entries already held stay + /// until the next write evicts down to the new size. + pub fn update_max_size_in_memory(&self, max_size_in_memory: Option) { + self.max_size_in_memory.store( + max_size_in_memory.unwrap_or(DEFAULT_MAX_SIZE_IN_MEMORY), + Ordering::Relaxed, + ); } pub fn max_entry_bytes(&self) -> Option { @@ -200,6 +214,23 @@ impl InMemoryCache { .collect()) } + /// Every live entry, for callers that inspect the whole tier. + pub fn entries(&self) -> Result, Error> { + let now = (self.now)(); + let state = self.lock()?; + Ok(state + .values + .iter() + .filter(|(key, _)| { + state + .expirations + .get(*key) + .is_none_or(|expiration| *expiration >= now) + }) + .map(|(key, value)| (key.clone(), value.clone())) + .collect()) + } + pub fn delete_cache(&self, key: &str) -> Result<(), Error> { let mut state = self.lock()?; Self::remove(&mut state, key); @@ -276,7 +307,7 @@ impl InMemoryCache { ttl: Option, now: Duration, ) -> Result { - Self::evict(state, self.max_size_in_memory, now, None); + Self::evict(state, self.max_size_in_memory(), now, None); if !self.check_value_size(&value)? { return Ok(CacheWrite::TooLarge); } @@ -300,12 +331,12 @@ where eligible: &[V], context: ExactCacheContext, ) -> Result { - if self.max_size_in_memory == 0 { + if self.max_size_in_memory() == 0 { return Ok(candidate); } let now = (self.now)(); let mut state = self.lock()?; - Self::evict(&mut state, self.max_size_in_memory, now, Some(key)); + Self::evict(&mut state, self.max_size_in_memory(), now, Some(key)); let existing = state .values .get(key) @@ -335,7 +366,7 @@ impl CounterCache for InMemoryCache { amount: f64, context: ExactCacheContext, ) -> Result { - if self.max_size_in_memory == 0 { + if self.max_size_in_memory() == 0 { return Ok(amount); } let now = (self.now)(); @@ -408,7 +439,7 @@ where values: Vec, ttl: Option, ) -> Result { - if self.max_size_in_memory == 0 { + if self.max_size_in_memory() == 0 { return Ok(values); } let now = (self.now)(); diff --git a/litellm-rust/crates/cache-response/src/service.rs b/litellm-rust/crates/cache-response/src/service.rs index 0bdf948ec48..ed4dccdc355 100644 --- a/litellm-rust/crates/cache-response/src/service.rs +++ b/litellm-rust/crates/cache-response/src/service.rs @@ -27,6 +27,10 @@ impl Default for ResponseCacheConfig { pub trait ResponseCacheService: Send + Sync { fn config(&self) -> &ResponseCacheConfig; + fn keyed_by_caller(&self) -> bool { + false + } + fn lookup<'a>( &'a self, request: &'a ResponseCacheRequest, diff --git a/litellm-rust/crates/cache-s3/Cargo.toml b/litellm-rust/crates/cache-s3/Cargo.toml index d51c282d4d7..b0382d62b44 100644 --- a/litellm-rust/crates/cache-s3/Cargo.toml +++ b/litellm-rust/crates/cache-s3/Cargo.toml @@ -9,14 +9,10 @@ repository.workspace = true litellm-http.workspace = true litellm-cache.workspace = true litellm-auth-aws.workspace = true -aws-sdk-s3 = { version = "1.146.1", default-features = false, features = ["rustls", "rt-tokio"] } -aws-credential-types = "1.3.0" -aws-smithy-runtime-api = { version = "1.16.2", features = ["client", "http-1x"] } -aws-smithy-types = { version = "1.6.0", features = ["http-body-1-x"] } -aws-types = "1.6.0" +litellm-s3.workspace = true +aws-sdk-s3.workspace = true +aws-smithy-types = "1.6.0" futures-util.workspace = true -http.workspace = true -reqwest.workspace = true tokio.workspace = true [dev-dependencies] diff --git a/litellm-rust/crates/cache-s3/src/auth.rs b/litellm-rust/crates/cache-s3/src/auth.rs deleted file mode 100644 index 659390a5659..00000000000 --- a/litellm-rust/crates/cache-s3/src/auth.rs +++ /dev/null @@ -1,58 +0,0 @@ -use aws_credential_types::{ - Credentials as AwsCredentials, - provider::{ProvideCredentials, error::CredentialsError, future}, -}; -use litellm_auth_aws::{AwsAuthConfig, AwsAuthService}; - -#[derive(Clone)] -pub struct S3Credentials { - auth: AwsAuthService, - config: AwsAuthConfig, - env: fn(&str) -> Option, -} - -impl S3Credentials { - pub fn new(config: AwsAuthConfig) -> Self { - Self::with_env(config, |name| std::env::var(name).ok()) - } - - pub fn with_env(config: AwsAuthConfig, env: fn(&str) -> Option) -> Self { - Self { - auth: AwsAuthService::default(), - config, - env, - } - } -} - -impl ProvideCredentials for S3Credentials { - fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a> - where - Self: 'a, - { - future::ProvideCredentials::new(async { - if let (Some(access_key_id), Some(secret_access_key)) = ( - self.config.access_key_id.clone(), - self.config.secret_access_key.clone(), - ) { - return Ok(AwsCredentials::new( - access_key_id, - secret_access_key, - self.config.session_token.clone(), - None, - "litellm-s3-cache", - )); - } - self.auth - .resolve_credentials(self.config.clone(), &self.env) - .await - .map_err(CredentialsError::provider_error) - }) - } -} - -impl std::fmt::Debug for S3Credentials { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("S3Credentials").finish_non_exhaustive() - } -} diff --git a/litellm-rust/crates/cache-s3/src/cache.rs b/litellm-rust/crates/cache-s3/src/cache.rs index 5369bca0f18..7d5f3f50418 100644 --- a/litellm-rust/crates/cache-s3/src/cache.rs +++ b/litellm-rust/crates/cache-s3/src/cache.rs @@ -5,20 +5,18 @@ use std::{ }; use aws_sdk_s3::{ - config::{BehaviorVersion, Region, RequestChecksumCalculation, ResponseChecksumValidation}, + config::{RequestChecksumCalculation, ResponseChecksumValidation}, error::SdkError, primitives::ByteStream, }; use aws_smithy_types::{DateTime, date_time::Format}; use futures_util::future::try_join_all; -use litellm_auth_aws::AwsAuthConfig; +use litellm_auth_aws::AwsCredentialsProvider; use litellm_cache::{ BaseCache, BatchCache, CacheCodec, DisconnectCache, Error, ExactCacheContext, FlushCache, }; use tokio::runtime::Handle; -use crate::{auth::S3Credentials, transport::ReqwestHttpClient}; - pub struct S3Endpoint { pub url: String, } @@ -28,7 +26,7 @@ pub struct S3CacheConfig { pub key_prefix: String, pub region: String, pub endpoint: Option, - pub auth: AwsAuthConfig, + pub credentials: AwsCredentialsProvider, } pub struct S3Cache { @@ -49,17 +47,14 @@ impl S3Cache { runtime: Handle, ) -> Self { let endpoint_url: Option = config.endpoint.map(|endpoint| endpoint.url); - let base = aws_sdk_s3::Config::builder() - .behavior_version(BehaviorVersion::latest()) - .region(Region::new(config.region.clone())) - .http_client(ReqwestHttpClient(http)) - .credentials_provider(S3Credentials::new(config.auth)) - .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) - .response_checksum_validation(ResponseChecksumValidation::WhenRequired); - let builder = match &endpoint_url { - Some(url) => base.endpoint_url(url).force_path_style(true), - None => base, - }; + let builder = litellm_s3::config( + config.region.clone(), + endpoint_url.as_deref(), + http, + config.credentials, + ) + .request_checksum_calculation(RequestChecksumCalculation::WhenRequired) + .response_checksum_validation(ResponseChecksumValidation::WhenRequired); Self { client: aws_sdk_s3::Client::from_conf(builder.build()), codec, diff --git a/litellm-rust/crates/cache-s3/src/lib.rs b/litellm-rust/crates/cache-s3/src/lib.rs index 9be0210aad3..43006532441 100644 --- a/litellm-rust/crates/cache-s3/src/lib.rs +++ b/litellm-rust/crates/cache-s3/src/lib.rs @@ -1,6 +1,3 @@ -mod auth; mod cache; -mod transport; -pub use auth::S3Credentials; pub use cache::{S3Cache, S3CacheConfig, S3Endpoint}; diff --git a/litellm-rust/crates/cache-s3/tests/auth.rs b/litellm-rust/crates/cache-s3/tests/auth.rs deleted file mode 100644 index 59c5aa2e091..00000000000 --- a/litellm-rust/crates/cache-s3/tests/auth.rs +++ /dev/null @@ -1,57 +0,0 @@ -use aws_credential_types::provider::ProvideCredentials; -use litellm_auth_aws::AwsAuthConfig; -use litellm_cache_s3::S3Credentials; -use rstest::rstest; - -fn explicit(session_token: Option<&str>) -> AwsAuthConfig { - AwsAuthConfig { - access_key_id: Some("key".to_string()), - secret_access_key: Some("secret".to_string()), - session_token: session_token.map(str::to_string), - region_name: Some("us-east-1".to_string()), - ..Default::default() - } -} - -fn ambient_token(name: &str) -> Option { - (name == "AWS_SESSION_TOKEN").then(|| "ambient".to_string()) -} - -fn environment_keys(name: &str) -> Option { - match name { - "AWS_ACCESS_KEY_ID" => Some("env-key".to_string()), - "AWS_SECRET_ACCESS_KEY" => Some("env-secret".to_string()), - "AWS_SESSION_TOKEN" => Some("env-token".to_string()), - _ => None, - } -} - -#[rstest] -#[case::explicit_keys_ignore_an_ambient_session_token( - explicit(None), ambient_token, ("key", "secret", None) -)] -#[case::explicit_keys_keep_their_session_token( - explicit(Some("t")), ambient_token, ("key", "secret", Some("t")) -)] -#[case::environment_keys_resolve_with_their_session_token( - AwsAuthConfig::default(), environment_keys, ("env-key", "env-secret", Some("env-token")) -)] -#[tokio::test] -async fn credentials_resolve( - #[case] config: AwsAuthConfig, - #[case] env: fn(&str) -> Option, - #[case] expected: (&str, &str, Option<&str>), -) { - let credentials = S3Credentials::with_env(config, env) - .provide_credentials() - .await - .unwrap(); - assert_eq!( - ( - credentials.access_key_id(), - credentials.secret_access_key(), - credentials.session_token(), - ), - expected - ); -} diff --git a/litellm-rust/crates/cache-s3/tests/support/mod.rs b/litellm-rust/crates/cache-s3/tests/support/mod.rs index 7378dde21bf..7b62d0f7208 100644 --- a/litellm-rust/crates/cache-s3/tests/support/mod.rs +++ b/litellm-rust/crates/cache-s3/tests/support/mod.rs @@ -5,7 +5,7 @@ use std::{ sync::{Arc, Mutex}, }; -use litellm_auth_aws::AwsAuthConfig; +use litellm_auth_aws::{AwsAuthConfig, AwsAuthService, AwsCredentialsProvider}; use litellm_cache::JsonCodec; use litellm_cache_s3::{S3Cache, S3CacheConfig, S3Endpoint}; use serde_json::Value; @@ -22,12 +22,16 @@ pub fn config(endpoint: &str) -> S3CacheConfig { endpoint: Some(S3Endpoint { url: endpoint.to_string(), }), - auth: AwsAuthConfig { - access_key_id: Some("key".to_string()), - secret_access_key: Some("secret".to_string()), - region_name: Some("us-east-1".to_string()), - ..Default::default() - }, + credentials: AwsCredentialsProvider::configured( + AwsAuthService::default(), + AwsAuthConfig { + access_key_id: Some("key".into()), + secret_access_key: Some("secret".into()), + region_name: Some("us-east-1".to_string()), + ..Default::default() + }, + |_| None, + ), } } diff --git a/litellm-rust/crates/callbacks-legacy-python/python_contract.json b/litellm-rust/crates/callbacks-legacy-python/python_contract.json index ab887bbc014..a96e9f04bdc 100644 --- a/litellm-rust/crates/callbacks-legacy-python/python_contract.json +++ b/litellm-rust/crates/callbacks-legacy-python/python_contract.json @@ -11,7 +11,8 @@ "logger", "kwargs", "start_time", - "end_time" + "end_time", + "cache_hit" ], "update_logging": [ "logger", @@ -27,6 +28,10 @@ "api_key", "additional_args" ], + "messages_input": [ + "body", + "kwargs" + ], "post_call": [ "logger", "original_response", @@ -37,19 +42,39 @@ "logger", "status", "headers", - "body" + "body", + "sent" + ], + "retried_request": [ + "logger", + "logged_body", + "body", + "reasons" ], "record_api_window": [ "logger", "start", "end" ], + "times_provider_exchange": [ + "client", + "timeout" + ], "defers_async_logging": [ "logger" ], + "claim_async_success_log": [ + "logger" + ], + "mark_async_success_enqueued": [ + "logger" + ], "defer_success": [ "logger", - "pending" + "pending", + "response", + "start", + "end" ], "sync_success_for_async_call": [ "logger", @@ -76,6 +101,12 @@ "start", "end" ], + "cache_hit_success": [ + "logger", + "response", + "start", + "end" + ], "enqueue_logging": [ "coroutine" ], @@ -100,14 +131,16 @@ ], "stream_opened": [ "logger", - "head" + "head", + "keeps_requested" ], "stamp_stream_head": [ "head", "logger", "kwargs", "start_time", - "end_time" + "end_time", + "cache_hit" ], "stream_success": [ "logger", @@ -117,7 +150,8 @@ "chunks", "start", "end", - "first_chunk" + "first_chunk", + "stream_teardown" ], "stream_failure": [ "logger", diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index 1bb84184e4c..71ca89b0790 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -6,23 +6,23 @@ use crate::LoggingOperation; use litellm_host_python::PythonOwned; use litellm_host::{ - interceptors::{RawResponse, RequestContext, WireRequest}, + interceptors::{RawResponse, RequestContext, RetriedRequest, RetryReason, WireRequest}, lifecycle::{FailureOrigin, Timing, epoch_seconds}, }; -use litellm_host_python::{Error as HostError, HookStep, from_py, to_py}; +use litellm_host_python::{Error as HostError, HookStep, from_py, json_from_py, json_to_py, to_py}; use pyo3::{ exceptions::{PyBaseException, PyException}, gc::{PyTraverseError, PyVisit}, prelude::*, - types::{PyDateTime, PyDict, PyList}, + types::{PyDateTime, PyDict, PyList, PyString}, }; use serde_json::Value; use crate::{ - DeploymentHooks, LegacyCallbacks, PublicCall, PythonLogger, + DeploymentHooks, LegacyCallbacks, PublicCall, PythonLogger, SentRequest, deferred::{PendingLogging, PendingSuccess}, finalize, is_internal_call, - python::Streaming, + python::{Logging, Streaming}, setup, }; @@ -41,6 +41,7 @@ struct DeliveredStream { struct LoggedRequest { body: Py, headers: Py, + url: String, context: RequestContext, } @@ -59,6 +60,16 @@ pub struct LegacyLogging { cache_key: Option, /// When the provider request went out, until its window is recorded (Messages only). api_started: Option, + host_answered: bool, +} + +fn retry_reason(reason: &RetryReason) -> Value { + match reason { + RetryReason::ThinkingSignature => serde_json::json!({"kind": "thinking_signature"}), + RetryReason::StrictTools { dropped } => { + serde_json::json!({"kind": "strict_tools", "dropped": dropped}) + } + } } fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult> { @@ -90,6 +101,7 @@ impl LegacyLogging { internal: false, cache_key: None, api_started: None, + host_answered: false, } } @@ -137,6 +149,48 @@ impl LegacyLogging { Some(params) } + // Compared as Python compares them, so a caller's `0` is kept over a typed float's `0.0`. + fn caller_value<'py>( + &self, + py: Python<'py>, + name: &str, + sent: &Value, + ) -> PyResult>> { + match self.call.lookup(py, name)? { + Some(value) if from_py::(&value).is_ok() && value.eq(to_py(py, sent)?)? => { + Ok(Some(value)) + } + _ => Ok(None), + } + } + + // Python's Messages handler logs the caller's own optional params, not their typed copies. + fn as_called(&self, py: Python<'_>, context: &RequestContext) -> PyResult { + let mut context = context.clone(); + if self.operation != LoggingOperation::Messages { + return Ok(context); + } + for (name, sent) in context + .optional_params + .as_object_mut() + .into_iter() + .flatten() + { + if let Some(value) = self.caller_value(py, name, sent)? { + *sent = from_py(&value)?; + } + } + Ok(context) + } + + fn times_provider_exchange(&self, py: Python<'_>) -> PyResult { + let client = self.call.lookup(py, "client")?; + let timeout = self.call.lookup(py, "timeout")?; + Logging::TimesProviderExchange + .call(py, (client, timeout))? + .extract() + } + /// Python times the provider exchange inside the async `AsyncHTTPHandler.post`; its sync /// client is untimed. fn close_api_window(&mut self, py: Python<'_>, end: f64) -> PyResult<()> { @@ -158,13 +212,20 @@ impl LegacyLogging { } } - fn input_description(&self) -> &'static str { - match self.operation { + fn logged_input<'py>( + &self, + py: Python<'py>, + body: &Bound<'py, PyDict>, + ) -> PyResult> { + let description = match self.operation { LoggingOperation::Completion => "Chat completions", LoggingOperation::Responses => "Responses", - LoggingOperation::Messages => "Messages", + LoggingOperation::Messages => { + return Logging::MessagesInput.call(py, (body, self.call.kwargs().bind(py))); + } LoggingOperation::Ocr => "OCR document processing", - } + }; + Ok(PyString::new(py, description).into_any()) } /// The key `pre_call` logs. Python's Messages handler logs an empty string, never the key. @@ -223,6 +284,7 @@ impl LegacyLogging { self.call.kwargs(), &self.start, &self.end, + self.messages_cache_hit(), )?; self.response .as_ref() @@ -230,6 +292,60 @@ impl LegacyLogging { .ok_or_else(|| HostError::MissingState.into()) } + fn messages_cache_hit(&self) -> bool { + self.operation == LoggingOperation::Messages && self.cache_key.is_some() + } + + fn flag_unsent_messages_call(&self, py: Python<'_>) -> PyResult<()> { + if self.operation != LoggingOperation::Messages + || self.request.is_some() + || self.cache_key.is_some() + || self.host_answered + { + return Ok(()); + } + let details = self.logger()?.object(py).getattr("model_call_details")?; + details + .call_method1("setdefault", ("litellm_params", PyDict::new(py)))? + .set_item("aanthropic_messages", self.asynchronous) + } + + fn unstreamed_stream_request(&self, py: Python<'_>) -> PyResult { + if self.operation != LoggingOperation::Messages + || self.stream.is_some() + || self.messages_cache_hit() + { + return Ok(false); + } + let Some(stream) = self.call.kwargs().bind(py).get_item("stream")? else { + return Ok(false); + }; + if !stream.is_exact_instance_of::() || !stream.is_truthy()? { + return Ok(false); + } + let logger = self.logger()?.object(py); + logger.setattr("stream", true)?; + logger + .getattr("model_call_details")? + .set_item("stream", true)?; + Ok(true) + } + + fn dispatch_cache_hit(&self, py: Python<'_>) -> PyResult<()> { + let logger = self.logger()?; + let logged = Logging::CacheHitSuccess.call( + py, + (logger.object(py), &self.response, &self.start, &self.end), + ); + match logged { + Err(error) if error.is_instance_of::(py) => { + error.write_unraisable(py, Some(logger.object(py))); + Ok(()) + } + result => result.map(|_| ()), + } + } + fn dispatch_success(&self, py: Python<'_>) -> PyResult<()> { match self.try_dispatch_success(py) { Err(error) if error.is_instance_of::(py) => { @@ -251,30 +367,49 @@ impl LegacyLogging { if !self.asynchronous { return pending().sync(py); } - if !self.internal - && self + if self.internal { + return Ok(()); + } + let defers = logger.defers_async_logging(py); + if logger.claim_async_success_log(py)? { + let enqueues = self .call .kwargs() .bind(py) .get_item("fallbacks")? - .is_none_or(|value| value.is_none()) - { - if logger.defers_async_logging(py) { + .is_none_or(|value| value.is_none()); + if defers { let pending = Py::new( py, PendingLogging { pending: Some(pending()), + enqueues, }, )?; - logger.defer_success(py, pending.bind(py).as_any())?; - } else { + return logger.defer_success( + py, + pending.bind(py).as_any(), + &self.response, + &self.start, + &self.end, + ); + } + if enqueues { pending().asynchronous(py)?; } } + if defers { + return Ok(()); + } logger.sync_success_for_async_call(py, &self.response, &self.start, &self.end) } - fn stream_success(&self, py: Python<'_>, stream: &DeliveredStream) -> PyResult<()> { + fn stream_success( + &self, + py: Python<'_>, + stream: &DeliveredStream, + teardown: bool, + ) -> PyResult<()> { let logger = self.logger()?; let billing = self.stream_billing().ok_or(HostError::MissingState)?; let billed = Streaming::Success.call( @@ -291,6 +426,7 @@ impl LegacyLogging { &self.start, &self.end, &stream.first_chunk, + teardown, ), ); match billed { @@ -428,6 +564,10 @@ impl LegacyLogging { self.logger = Some(result.logger()?); self.call.set_kwargs(result.kwargs()?); if self.runs_deployment_hooks() { + self.call + .kwargs() + .bind(py) + .set_item("litellm_logging_obj", self.logger()?.object(py))?; return Ok(HookStep::Await( DeploymentHooks::before_call(py, self.call.kwargs(), self.call_type())?, Self::resume_begin, @@ -442,6 +582,7 @@ impl LegacyLogging { wire: Box, context: &RequestContext, ) -> PyResult>> { + let context = &self.as_called(py, context)?; let logger = self.logger()?; logger.update_from_kwargs( py, @@ -450,16 +591,25 @@ impl LegacyLogging { context, self.logged_request_params(context).as_ref(), )?; - let body = to_py(py, &wire.body)? - .into_bound(py) - .cast_into::()?; + let lossless = self.operation == LoggingOperation::Messages; + let body = if lossless { + json_to_py(py, &wire.body)? + } else { + to_py(py, &wire.body)?.into_bound(py) + } + .cast_into::()?; for (name, sent) in wire.body.as_object().into_iter().flatten() { - if let Some(value) = self.call.lookup(py, name)? - && from_py::(&value).is_ok_and(|caller| caller == *sent) - { + if let Some(value) = self.caller_value(py, name, sent)? { body.set_item(name, value)?; } } + if self.operation == LoggingOperation::Messages { + // Python's Messages handler merges its transformed request body into the call details. + logger + .object(py) + .getattr("model_call_details")? + .call_method1("update", (&body,))?; + } let headers = PyDict::new(py); for (name, value) in &wire.headers { headers.set_item(name, value)?; @@ -467,17 +617,22 @@ impl LegacyLogging { self.request = Some(LoggedRequest { body: body.clone().unbind(), headers: headers.clone().unbind(), + url: wire.url.clone(), context: context.clone(), }); + let input = self.logged_input(py, &body)?; self.logger()?.pre_call( py, - self.input_description(), + &input, self.logged_api_key(context), &body, &headers, &wire.url, )?; - if self.operation == LoggingOperation::Messages && self.asynchronous { + if self.operation == LoggingOperation::Messages + && self.asynchronous + && self.times_provider_exchange(py)? + { self.api_started = Some(epoch_seconds()); } let headers = headers @@ -485,7 +640,11 @@ impl LegacyLogging { .map(|(name, value)| Ok((name.extract::()?, value.extract::()?))) .collect::>>()?; Ok(HookStep::Ready(Box::new(WireRequest { - body: from_py(&body)?, + body: if lossless { + json_from_py(body.as_any())? + } else { + from_py(&body)? + }, headers, ..*wire }))) @@ -499,7 +658,11 @@ impl LegacyLogging { ) -> PyResult>> { self.end = Some(datetime(py, timing.end_time)?); self.response = Some(response); - if self.runs_deployment_hooks() { + self.flag_unsent_messages_call(py)?; + if self.unstreamed_stream_request(py)? { + return self.finalize(py); + } + if self.runs_deployment_hooks() && !self.messages_cache_hit() { return Ok(HookStep::Await( DeploymentHooks::after_success( py, @@ -520,12 +683,30 @@ impl LegacyLogging { ) -> PyResult> { use litellm_host::interceptors::{Cost, ResultSource}; + self.cache_key = match &facts.source { + ResultSource::Provider => None, + ResultSource::Cache { key } => Some(key.clone()), + ResultSource::Host => { + self.host_answered = true; + return Ok(HookStep::Ready(())); + } + }; let logger = self.logger()?.object(py); + if self.messages_cache_hit() { + logger + .getattr("model_call_details")? + .set_item("cache_hit", true)?; + return Ok(HookStep::Ready(())); + } let params = logger .getattr("litellm_params")? .cast_into::()? .copy()?; - params.set_item("custom_llm_provider", &facts.provider.provider)?; + if self.operation != LoggingOperation::Messages { + // Python's Messages handler hands the provider to the logger beside its params, never + // in them, and the proxy's logger carries over to the next fallback hop. + params.set_item("custom_llm_provider", &facts.provider.provider)?; + } crate::python::Logging::Update.call( py, ( @@ -544,12 +725,17 @@ impl LegacyLogging { }; logger.setattr("_provider_reported_cost", reported_cost)?; details.call_method1("pop", ("response_cost", py.None()))?; - self.cache_key = match &facts.source { - ResultSource::Provider => None, - ResultSource::Cache { key } => Some(key.clone()), - }; - details.set_item("cache_hit", self.cache_key.is_some())?; + if self.cache_key.is_some() { + details.set_item("cache_hit", true)?; + } else { + details.call_method1("pop", ("cache_hit", py.None()))?; + } details.set_item("cache_key", self.cache_key.as_deref())?; + if let (LoggingOperation::Messages, ResultSource::Provider, Some(request)) = + (self.operation, &facts.source, &self.request) + { + details.call_method1("update", (request.body.bind(py),))?; + } Ok(HookStep::Ready(())) } @@ -563,8 +749,13 @@ impl LegacyLogging { // response, and logging replays it through `AnthropicConfig.transform_response`. self.close_api_window(py, epoch_seconds())?; if let Some(head) = &raw.head { + let sent = self.request.as_ref().map(|request| SentRequest { + url: &request.url, + headers: &request.headers, + body: &request.body, + }); self.logger()? - .record_response(py, head.status, &head.headers, &raw.body)?; + .record_response(py, head.status, &head.headers, &raw.body, sent)?; } return Ok(HookStep::Ready(())); } @@ -583,6 +774,40 @@ impl LegacyLogging { Ok(HookStep::Ready(())) } + pub(crate) fn request_retried( + &mut self, + py: Python<'_>, + retry: &RetriedRequest, + ) -> PyResult> { + let Some(request) = &self.request else { + return Ok(HookStep::Ready(())); + }; + let reasons: Vec = retry.reasons.iter().map(retry_reason).collect(); + let retried = Logging::RetriedRequest.call( + py, + ( + self.logger()?.object(py), + request.body.bind(py), + to_py(py, &retry.body)?, + to_py(py, &reasons)?, + ), + )?; + let now = epoch_seconds(); + if self.api_started.is_some() { + self.close_api_window(py, now)?; + self.api_started = Some(now); + } + Ok(HookStep::Await(retried.unbind(), Self::resume_retried)) + } + + fn resume_retried( + &mut self, + _py: Python<'_>, + result: PyResult>, + ) -> PyResult> { + result.map(|_| HookStep::Ready(())) + } + pub(crate) fn succeeded( &mut self, py: Python<'_>, @@ -592,7 +817,9 @@ impl LegacyLogging { self.end = Some(datetime(py, timing.end_time)?); self.response = Some(response.clone_ref(py)); match &self.stream { - Some(stream) => self.stream_success(py, stream)?, + Some(stream) => self.stream_success(py, stream, false)?, + None if self.messages_cache_hit() => self.dispatch_cache_hit(py)?, + None if self.unstreamed_stream_request(py)? => {} None => self.dispatch_success(py)?, } Ok(HookStep::Ready(())) @@ -627,7 +854,11 @@ impl LegacyLogging { if self.stream_billing().is_none() { return Err(HostError::MissingState.into()); } - if let Some(key) = &self.cache_key { + if let Some(key) = self + .cache_key + .as_ref() + .filter(|_| !self.messages_cache_hit()) + { head.bind(py).set_item("cache_key", key)?; head.bind(py).set_item("cache_hit", true)?; } @@ -642,10 +873,18 @@ impl LegacyLogging { self.call.kwargs(), &self.start, datetime(py, epoch_seconds())?, + self.messages_cache_hit(), ), )?; } - Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?; + Streaming::Opened.call( + py, + ( + self.logger()?.object(py), + head.bind(py), + self.operation == LoggingOperation::Messages, + ), + )?; self.stream = Some(DeliveredStream { chunks: PyList::empty(py).unbind(), first_chunk: None, @@ -662,8 +901,28 @@ impl LegacyLogging { } } +impl LegacyLogging { + // A caller that goes away mid-stream cancels the call before it ends: Python's Messages + // stream iterator still bills the chunks it read, straight away rather than deferred. + fn bill_abandoned_stream(&mut self, py: Python<'_>) -> PyResult<()> { + if !self.asynchronous || self.end.is_some() || self.messages_cache_hit() { + return Ok(()); + } + let Some(stream) = self.stream.take() else { + return Ok(()); + }; + self.end = Some(datetime(py, epoch_seconds())?); + self.stream_success(py, &stream, true) + } +} + impl PythonOwned for LegacyLogging { fn close(&mut self, py: Python<'_>) { + if self.logger.is_some() + && let Err(error) = self.bill_abandoned_stream(py) + { + error.write_unraisable(py, None); + } if let Some(logger) = self.logger.take() && let Err(error) = logger.restore_context(py) { @@ -817,6 +1076,24 @@ if asynchronous: }); } + #[rstest::rstest] + fn the_deployment_pre_call_hook_receives_the_logger_as_client_injects_it() { + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c" +kwargs = {'logger': logger} +logger.hooks['pre'] = lambda value: seen.append(value.get('litellm_logging_obj')) or 'awaitable' +seen = [] +", + ); + let (_, step) = begin(py, &locals, true); + assert!(awaits_deployment_hook(&step)); + run(py, &locals, c"assert seen == [logger]"); + }); + } + #[rstest::rstest] fn kwargs_returned_by_the_pre_call_hook_are_what_the_call_prepares() { Python::initialize(); @@ -937,6 +1214,82 @@ assert finalized is replacement }); } + #[rstest] + #[case::unary(false)] + #[case::stream(true)] + fn a_host_answer_keeps_the_logger_as_set_up_and_a_streamed_one_is_finalized_unstreamed( + #[case] stream: bool, + ) { + use litellm_host::interceptors::{Cost, ExecutionFacts, ProviderIdentity, ResultSource}; + + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c" +kwargs = {'logger': logger} +response = object() +logger.hooks = {'pre': lambda kwargs: kwargs} +", + ); + local(&locals, "kwargs").set_item("stream", stream).unwrap(); + let (mut logging, step) = begin(py, &locals, true); + logging.operation = crate::LoggingOperation::Messages; + resume( + &mut logging, + step, + py, + Ok(local(&locals, "kwargs").unbind()), + ) + .unwrap(); + let facts = ExecutionFacts { + provider: ProviderIdentity { + model: "claude-sonnet-4-5".into(), + provider: "anthropic".into(), + }, + cost: Cost::Deferred, + source: ResultSource::Host, + }; + assert!(matches!( + logging.result_ready(py, &facts).unwrap(), + HookStep::Ready(()) + )); + let response = local(&locals, "response").unbind(); + let step = logging + .transform_response(py, response.clone_ref(py), TIMING) + .unwrap(); + assert_eq!(awaits_deployment_hook(&step), !stream); + if !stream { + return; + } + let HookStep::Ready(returned) = step else { + panic!("a host-answered stream is returned as is"); + }; + assert!(returned.bind(py).is(response.bind(py))); + logging + .on_event( + py, + PythonCallEvent::Succeeded { + timing: TIMING, + response: &response, + }, + ) + .unwrap(); + run( + py, + &locals, + c" +names = logger.names() +assert ('finalize', response) in logger.calls, names +assert 'update_logging' not in names, names +assert not [name for name in names if 'success' in name], names +assert logger.stream is True +assert logger.model_call_details['stream'] is True +", + ); + }); + } + #[rstest] #[case::pre_call(false)] #[case::post_call(true)] @@ -1028,7 +1381,9 @@ mod payload_tests { use std::ffi::CStr; use litellm_host::hooks::CallHooks; - use litellm_host::interceptors::{RawResponse, RequestContext, WireRequest}; + use litellm_host::interceptors::{ + RawResponse, RequestContext, RetriedRequest, RetryReason, WireRequest, + }; use litellm_host::lifecycle::ExecutionEvent; use litellm_host_python::{HookStep, PythonCallEvent, PythonOwned, to_py}; use litellm_secret_value::SecretValue; @@ -1114,10 +1469,10 @@ logger._provider_reported_cost = None }; let previous = litellm_host::interceptors::ExecutionFacts { provider: facts.provider.clone(), - cost: litellm_host::interceptors::Cost::Reported { - amount: serde_json::Number::from_f64(9.0).unwrap(), + cost: litellm_host::interceptors::Cost::CacheHit, + source: litellm_host::interceptors::ResultSource::Cache { + key: "earlier-hit".into(), }, - source: litellm_host::interceptors::ResultSource::Provider, }; logging.result_ready(py, &previous).unwrap(); local(&locals, "logger") @@ -1135,13 +1490,59 @@ logger._provider_reported_cost = None c" assert logger.model_call_details.get('response_cost') is None, logger.model_call_details assert logger._provider_reported_cost == reported, logger._provider_reported_cost -assert logger.model_call_details['cache_hit'] is cache_hit, logger.model_call_details +assert logger.model_call_details.get('cache_hit') is (True if cache_hit else None), logger.model_call_details assert logger.update['custom_llm_provider'] == 'edenai', logger.update ", ); }); } + #[rstest] + #[case::messages(crate::LoggingOperation::Messages, false)] + #[case::ocr(crate::LoggingOperation::Ocr, true)] + fn result_ready_puts_the_provider_in_the_logged_params_except_for_messages( + #[case] operation: crate::LoggingOperation, + #[case] in_params: bool, + ) { + Python::initialize(); + Python::attach(|py| { + let locals = namespace(py, PAYLOAD_LOGGER); + run( + py, + &locals, + c" +logger.litellm_params = {} +logger.optional_params = {} +logger.model_call_details = {} +logger._provider_reported_cost = None +", + ); + let mut logging = LegacyLogging { + logger: Some(PythonLogger::new(local(&locals, "logger").unbind())), + operation, + ..legacy_call(py, &locals, false) + }; + let facts = litellm_host::interceptors::ExecutionFacts { + provider: litellm_host::interceptors::ProviderIdentity { + model: "native-test".into(), + provider: "anthropic".into(), + }, + cost: litellm_host::interceptors::Cost::Deferred, + source: litellm_host::interceptors::ResultSource::Provider, + }; + logging.result_ready(py, &facts).unwrap(); + locals.set_item("in_params", in_params).unwrap(); + run( + py, + &locals, + c" +assert logger.update['custom_llm_provider'] == 'anthropic', logger.update +assert ('custom_llm_provider' in logger.update['litellm_params']) is in_params, logger.update +", + ); + }); + } + const DOCUMENT: &str = "data:application/pdf;base64,YWJj"; const EDITED: &str = "data:application/pdf;base64,ZWRpdGVk"; @@ -1149,6 +1550,61 @@ assert logger.update['custom_llm_provider'] == 'edenai', logger.update json!({"type": "document_url", "document_url": source}) } + #[test] + fn a_retried_request_hands_the_logged_body_the_retried_body_and_the_reasons_to_logging() { + Python::initialize(); + Python::attach(|py| { + let locals = namespace(py, PAYLOAD_LOGGER); + let mut logging = LegacyLogging { + logger: Some(PythonLogger::new(local(&locals, "logger").unbind())), + ..legacy_call(py, &locals, false) + }; + let context = RequestContext { + model: "model".into(), + custom_llm_provider: "provider".into(), + optional_params: json!({}), + secret_fields: Vec::new(), + api_key: None, + }; + let wire = WireRequest { + url: "https://provider.invalid/v1/messages".into(), + headers: Vec::new(), + body: json!({"messages": [{"role": "user", "content": "hi"}], "thinking": {"type": "enabled"}}), + }; + logging + .before_provider_request(py, Box::new(wire.clone()), &context) + .unwrap(); + let retried = RetriedRequest { + reasons: vec![ + RetryReason::ThinkingSignature, + RetryReason::StrictTools { dropped: 2 }, + ], + body: json!({"messages": [{"role": "user", "content": "hi"}]}), + }; + + let step = logging + .on_event( + py, + PythonCallEvent::Execution(ExecutionEvent::ProviderRequestRetried { + retry: retried, + }), + ) + .unwrap(); + + assert!(matches!(step, HookStep::Await(_, _))); + run( + py, + &locals, + c" +[(logged, body, reasons)] = [value for name, value in logger.calls if name == 'retried_request'] +assert logged is logger.pre['complete_input_dict'], logged +assert body == {'messages': [{'role': 'user', 'content': 'hi'}]}, body +assert reasons == [{'kind': 'thinking_signature'}, {'kind': 'strict_tools', 'dropped': 2}], reasons +", + ); + }); + } + fn before_provider_request(script: &CStr, body: Value) -> WireRequest { before_send_with_secrets(script, json!({}), body, &[]) } @@ -1276,7 +1732,10 @@ assert logger.update['custom_llm_provider'] == 'edenai', logger.update #[rstest] #[case::completion(crate::LoggingOperation::Completion, "Chat completions")] #[case::responses(crate::LoggingOperation::Responses, "Responses")] - #[case::messages(crate::LoggingOperation::Messages, "Messages")] + #[case::messages( + crate::LoggingOperation::Messages, + r#"[{'role': 'user', 'content': '{"pages": [1]}'}]"# + )] #[case::ocr(crate::LoggingOperation::Ocr, "OCR document processing")] fn prepared_arguments_replace_the_legacy_view_without_losing_callback_aliases( #[case] operation: crate::LoggingOperation, @@ -1317,7 +1776,7 @@ prepared = {'pages': replacement} &locals, c" assert logger.pre['complete_input_dict']['pages'] is replacement -assert logger.pre_input == description +assert str(logger.pre_input) == description assert original == [0] ", ); @@ -1868,15 +2327,67 @@ mod terminal_tests { .unwrap() } + #[rstest] + #[case::requested_stream( + c"kwargs = {'logger': logger, 'stream': True}", + true, + &["finalize"] + )] + #[case::unstreamed(c"kwargs = {'logger': logger, 'stream': False}", false, &["success_hook"])] + #[case::stream_flag_not_true(c"kwargs = {'logger': logger, 'stream': 'true'}", false, &["success_hook"])] + fn a_whole_messages_result_for_a_requested_stream_is_finalized_without_success_logging( + #[case] script: &CStr, + #[case] handed_back: bool, + #[case] expected: &[&str], + ) { + Python::initialize(); + Python::attach(|py| { + let locals = namespace(py, c"response = object()"); + run(py, &locals, script); + let mut logging = LegacyLogging { + operation: crate::LoggingOperation::Messages, + ..logged(py, &locals, true) + }; + let step = logging + .transform_public_response(py, local(&locals, "response").unbind(), TIMING) + .unwrap(); + assert_eq!(matches!(step, HookStep::Ready(_)), handed_back); + if handed_back { + assert!(matches!( + succeed(py, &locals, &mut logging), + HookStep::Ready(()) + )); + } + let names: Vec = local(&locals, "logger") + .call_method0("names") + .unwrap() + .extract() + .unwrap(); + assert_eq!(names, expected); + if handed_back { + run( + py, + &locals, + c"assert logger.stream is True and logger.model_call_details['stream'] is True", + ); + } + }); + } + #[rstest] #[case::sync_listened(false, c"", &["submit"])] #[case::async_listened( true, c"", - &["async_success_handler", "enqueued", "sync_success_for_async_call"] + &["async_success_handler", "mark_async_success_enqueued", "enqueued", "sync_success_for_async_call"] )] - #[case::async_deferred(true, c"logger._defer_async_logging = True", &["sync_success_for_async_call"])] + #[case::async_deferred(true, c"logger._defer_async_logging = True", &[])] #[case::async_with_fallbacks(true, c"kwargs = {'fallbacks': ['other']}", &["sync_success_for_async_call"])] + #[case::async_deferred_with_fallbacks( + true, + c"logger._defer_async_logging = True\nkwargs = {'fallbacks': ['other']}", + &[] + )] fn success_reaches_the_logging_handlers( #[case] asynchronous: bool, #[case] script: &CStr, @@ -1902,12 +2413,88 @@ mod terminal_tests { &locals, c" assert all(value is response for name, value in logger.calls if name.endswith('_handler')) -assert hasattr(logger, '_native_pending_logging') == getattr(logger, '_defer_async_logging', False) +assert hasattr(logger, 'enqueue_deferred_logging') == getattr(logger, '_defer_async_logging', False) +assert getattr(logger, 'deferred_success_result', (response,))[0] is response ", ); }); } + #[rstest] + #[case::accepted(c"", &["sync_success_for_async_call", "async_success_handler", "mark_async_success_enqueued", "enqueued"])] + #[case::accepted_with_fallbacks(c"kwargs = {'fallbacks': ['other']}", &["sync_success_for_async_call"])] + fn a_deferred_success_runs_sync_callbacks_only_once_released( + #[case] script: &CStr, + #[case] expected: &[&str], + ) { + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c"response = object()\nlogger._defer_async_logging = True", + ); + run(py, &locals, script); + let mut logging = logged(py, &locals, true); + succeed(py, &locals, &mut logging); + run( + py, + &locals, + c" +assert logger.names() == [], logger.calls +logger.enqueue_deferred_logging() +logger.enqueue_deferred_logging() +", + ); + let names: Vec = local(&locals, "logger") + .call_method0("names") + .unwrap() + .extract() + .unwrap(); + assert_eq!(names, expected); + }); + } + + #[rstest] + #[case::deferred( + c"logger._defer_async_logging = True", + c"logger.enqueue_deferred_logging()", + &["sync_success_for_async_call", "async_success_handler", "mark_async_success_enqueued", "enqueued"] + )] + #[case::deferred_and_blocked(c"logger._defer_async_logging = True", c"", &[])] + #[case::immediate( + c"", + c"", + &["async_success_handler", "mark_async_success_enqueued", "enqueued", "sync_success_for_async_call", "sync_success_for_async_call"] + )] + fn a_fallback_attempt_on_the_same_logger_leaves_the_first_success_claimed( + #[case] script: &CStr, + #[case] release: &CStr, + #[case] expected: &[&str], + ) { + Python::initialize(); + Python::attach(|py| { + let locals = namespace(py, c"refused = object()\nresponse = refused"); + run(py, &locals, script); + let mut first = logged(py, &locals, true); + succeed(py, &locals, &mut first); + run(py, &locals, c"response = object()"); + let mut fallback = logged(py, &locals, true); + succeed(py, &locals, &mut fallback); + run(py, &locals, release); + let names: Vec = local(&locals, "logger") + .call_method0("names") + .unwrap() + .extract() + .unwrap(); + assert_eq!(names, expected); + run( + py, + &locals, + c"assert all(value is refused for name, value in logger.calls if name == 'async_success_handler')", + ); + }); + } + #[rstest] fn dropped_observations_preserve_deferred_success_and_response_identity() { Python::initialize(); @@ -1933,10 +2520,10 @@ assert hasattr(logger, '_native_pending_logging') == getattr(logger, '_defer_asy )); assert_eq!(sender.dropped_events(), 1); run(py, &locals, c" -assert logger.names() == ['sync_success_for_async_call'], logger.calls -logger._native_pending_logging.release(True) -logger._native_pending_logging.release(True) -assert logger.names() == ['sync_success_for_async_call', 'async_success_handler', 'enqueued'], logger.calls +assert logger.names() == [], logger.calls +logger.enqueue_deferred_logging() +logger.enqueue_deferred_logging() +assert logger.names() == ['sync_success_for_async_call', 'async_success_handler', 'mark_async_success_enqueued', 'enqueued'], logger.calls assert logger.calls[0][1] is response assert logger.calls[1][1] is response "); @@ -1981,6 +2568,45 @@ assert chunks[1] is last }); } + #[rstest] + #[case::cancelled_mid_stream(None, &["stamp_stream_head", "stream_opened", "stream_success", "restore"])] + #[case::after_success(Some(true), &["stamp_stream_head", "stream_opened", "stream_success", "restore"])] + #[case::after_failure(Some(false), &["stamp_stream_head", "stream_opened", "stream_failure", "restore"])] + fn closing_an_unfinished_stream_bills_its_delivered_chunks_once( + #[case] ended: Option, + #[case] expected: &[&str], + ) { + Python::initialize(); + Python::attach(|py| { + let locals = namespace( + py, + c"first = b'first'\nresponse = None\nfailure = ValueError('provider')\nhead = {}", + ); + let mut logging = LegacyLogging { + operation: crate::LoggingOperation::Messages, + ..logged(py, &locals, true) + }; + logging + .on_stream_open(py, &local(&locals, "head").unbind()) + .unwrap(); + logging + .on_stream_chunk(py, &local(&locals, "first").unbind()) + .unwrap(); + match ended { + Some(true) => drop(succeed(py, &locals, &mut logging)), + Some(false) => drop(fail(py, &locals, &mut logging)), + None => {} + } + litellm_host_python::PythonOwned::close(&mut logging, py); + let names: Vec = local(&locals, "logger") + .call_method0("names") + .unwrap() + .extract() + .unwrap(); + assert_eq!(names, expected); + }); + } + #[rstest] #[case::synchronous(false, &["failure_handler"])] #[case::asynchronous(true, &[])] @@ -2009,7 +2635,7 @@ assert chunks[1] is last } #[rstest::rstest] - fn internal_async_calls_skip_the_async_success_fan_out() { + fn internal_async_calls_skip_success_logging() { Python::initialize(); Python::attach(|py| { let locals = namespace(py, c"response = object()"); @@ -2018,11 +2644,7 @@ assert chunks[1] is last ..logged(py, &locals, true) }; succeed(py, &locals, &mut logging); - run( - py, - &locals, - c"assert logger.names() == ['sync_success_for_async_call'], logger.calls", - ); + run(py, &locals, c"assert logger.calls == [], logger.calls"); }); } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs b/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs index 1951c9ebfb4..9eed008910c 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs @@ -3,13 +3,19 @@ //! duplication. All of it expires with the legacy callback contract. use litellm_host::interceptors::{RequestContext, WireRequest}; -use litellm_host_python::to_py; +use litellm_host_python::json_to_py; use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict}; use serde_json::Value; use crate::logger::PythonLogger; use crate::python::{Logging, Wrapper}; +pub struct SentRequest<'a> { + pub url: &'a str, + pub headers: &'a Py, + pub body: &'a Py, +} + pub trait LegacyCallbacks { /// `Logging.update_from_kwargs`: what the logger is told about the request it is /// about to see, with consumed credentials redacted. @@ -26,7 +32,7 @@ pub trait LegacyCallbacks { fn pre_call( &self, py: Python<'_>, - input: &str, + input: &Bound<'_, PyAny>, api_key: Option<&str>, body: &Bound<'_, PyDict>, headers: &Bound<'_, PyDict>, @@ -41,6 +47,7 @@ pub trait LegacyCallbacks { status: u16, headers: &[(String, String)], body: &str, + request: Option>, ) -> PyResult<()>; /// Records one provider HTTP exchange's window, as Python's timed `AsyncHTTPHandler.post` @@ -59,7 +66,16 @@ pub trait LegacyCallbacks { fn defers_async_logging(&self, py: Python<'_>) -> bool; - fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()>; + fn claim_async_success_log(&self, py: Python<'_>) -> PyResult; + + fn defer_success( + &self, + py: Python<'_>, + pending: &Bound<'_, PyAny>, + response: &Option>, + start: &Py, + end: &Option>, + ) -> PyResult<()>; fn sync_success_for_async_call( &self, @@ -108,9 +124,7 @@ impl LegacyCallbacks for PythonLogger { let redacted_kwargs = redact(py, kwargs, &secret_fields)?; let optional_params = redact( py, - &to_py(py, &context.optional_params)? - .into_bound(py) - .cast_into::()?, + &json_to_py(py, &context.optional_params)?.cast_into::()?, &secret_fields, )?; let params = PyDict::new(py); @@ -129,7 +143,7 @@ impl LegacyCallbacks for PythonLogger { } } for (name, value) in request_params.into_iter().flatten() { - params.set_item(name, to_py(py, value)?)?; + params.set_item(name, json_to_py(py, value)?)?; } Logging::Update.call( py, @@ -148,7 +162,7 @@ impl LegacyCallbacks for PythonLogger { fn pre_call( &self, py: Python<'_>, - input: &str, + input: &Bound<'_, PyAny>, api_key: Option<&str>, body: &Bound<'_, PyDict>, headers: &Bound<'_, PyDict>, @@ -168,8 +182,13 @@ impl LegacyCallbacks for PythonLogger { status: u16, headers: &[(String, String)], body: &str, + request: Option>, ) -> PyResult<()> { - Logging::RecordResponse.call(py, (self.object(py), status, headers.to_vec(), body))?; + let request = request.map(|sent| (sent.url, sent.headers.bind(py), sent.body.bind(py))); + Logging::RecordResponse.call( + py, + (self.object(py), status, headers.to_vec(), body, request), + )?; Ok(()) } @@ -203,8 +222,21 @@ impl LegacyCallbacks for PythonLogger { .unwrap_or(false) } - fn defer_success(&self, py: Python<'_>, pending: &Bound<'_, PyAny>) -> PyResult<()> { - Logging::DeferSuccess.call(py, (self.object(py), pending))?; + fn claim_async_success_log(&self, py: Python<'_>) -> PyResult { + Logging::ClaimAsyncSuccess + .call(py, (self.object(py),))? + .extract() + } + + fn defer_success( + &self, + py: Python<'_>, + pending: &Bound<'_, PyAny>, + response: &Option>, + start: &Py, + end: &Option>, + ) -> PyResult<()> { + Logging::DeferSuccess.call(py, (self.object(py), pending, response, start, end))?; Ok(()) } @@ -252,6 +284,7 @@ impl LegacyCallbacks for PythonLogger { ) -> PyResult<()> { let coroutine = Logging::AsyncSuccessHandler.call(py, (self.object(py), response, start, end))?; + Logging::MarkAsyncSuccessEnqueued.call(py, (self.object(py),))?; let enqueue = Logging::Enqueue.call(py, (&coroutine,)); if enqueue.is_err() && let Err(error) = coroutine.call_method0("close") diff --git a/litellm-rust/crates/callbacks-legacy-python/src/deferred.rs b/litellm-rust/crates/callbacks-legacy-python/src/deferred.rs index 292648ba69c..ea72191ddb2 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/deferred.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/deferred.rs @@ -22,21 +22,35 @@ impl PendingSuccess { self.logger .enqueue_success(py, &self.response, &self.start, &self.end) } + + pub(crate) fn sync_for_async_call(&self, py: Python<'_>) -> PyResult<()> { + self.logger + .sync_success_for_async_call(py, &self.response, &self.start, &self.end) + } } #[pyclass] pub(crate) struct PendingLogging { pub(crate) pending: Option, + pub(crate) enqueues: bool, } #[pymethods] impl PendingLogging { - fn release(slf: &Bound<'_, Self>, py: Python<'_>, success: bool) -> PyResult<()> { - let pending = slf.borrow_mut().pending.take(); - if let Some(pending) = pending - && success - { - match pending.asynchronous(py) { + fn __call__(slf: &Bound<'_, Self>, py: Python<'_>) -> PyResult<()> { + let (pending, enqueues) = { + let mut slot = slf.borrow_mut(); + (slot.pending.take(), slot.enqueues) + }; + if let Some(pending) = pending { + let released = pending.sync_for_async_call(py).and_then(|()| { + if enqueues { + pending.asynchronous(py) + } else { + Ok(()) + } + }); + match released { Err(error) if error.is_instance_of::(py) => { error.write_unraisable(py, Some(pending.logger.object(py))); } @@ -87,6 +101,7 @@ mod tests { start: py.None(), end: Some(py.None()), }), + enqueues: true, }, ) .unwrap(); @@ -108,7 +123,7 @@ observed = [] def on_enqueue(coroutine): observed.append(marker.get()) - pending.release(True) + pending() logger.on_enqueue = on_enqueue ", @@ -118,28 +133,12 @@ logger.on_enqueue = on_enqueue &locals, c" marker.set('release') -pending.release(True) -pending.release(True) +pending() +pending() assert observed == ['release'], observed -assert logger.names() == ['async_success_handler', 'enqueued'], logger.calls +assert logger.names() == ['sync_success_for_async_call', 'async_success_handler', 'mark_async_success_enqueued', 'enqueued'], logger.calls assert logger.calls[0][1] is response -", - ); - }); - } - - #[test] - fn a_blocked_release_drops_the_success_for_good() { - Python::initialize(); - Python::attach(|py| { - let locals = defer(py, c""); - run( - py, - &locals, - c" -pending.release(False) -pending.release(True) -assert logger.calls == [], logger.calls +assert logger.calls[1][1] is response ", ); }); @@ -168,7 +167,7 @@ logger.on_enqueue = on_enqueue locals .set_item("failure", py.eval(failure, None, Some(&locals)).unwrap()) .unwrap(); - let released = local(&locals, "pending").call_method1("release", (true,)); + let released = local(&locals, "pending").call0(); match released { Ok(_) => assert!(!propagates), Err(error) => { @@ -181,8 +180,8 @@ logger.on_enqueue = on_enqueue py, &locals, c" -pending.release(True) -assert logger.names() == ['async_success_handler', 'enqueued', 'closed'], logger.calls +pending() +assert logger.names() == ['sync_success_for_async_call', 'async_success_handler', 'mark_async_success_enqueued', 'enqueued', 'closed'], logger.calls assert unraisable_from(logger) == ([] if propagates else [failure]) ", ); diff --git a/litellm-rust/crates/callbacks-legacy-python/src/lib.rs b/litellm-rust/crates/callbacks-legacy-python/src/lib.rs index f453237b469..3a8c15ae54c 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/lib.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/lib.rs @@ -17,7 +17,7 @@ mod mapping; mod python; pub use adapter::LegacyLogging; pub use call::PublicCall; -pub(crate) use callbacks::{LegacyCallbacks, is_internal_call}; +pub(crate) use callbacks::{LegacyCallbacks, SentRequest, is_internal_call}; pub(crate) use logger::{DeploymentHooks, PythonLogger, finalize, setup}; pub use mapping::{CallBoundary, CallbackMapping, Dispatch, callback_mappings}; diff --git a/litellm-rust/crates/callbacks-legacy-python/src/logger.rs b/litellm-rust/crates/callbacks-legacy-python/src/logger.rs index 38f3bf29828..3019d87a20b 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/logger.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/logger.rs @@ -77,8 +77,12 @@ pub fn finalize( kwargs: &Py, start: &Py, end: &Option>, + cache_hit: bool, ) -> PyResult<()> { - Wrapper::Finalize.call(py, (response, logger.object(py), kwargs, start, end))?; + Wrapper::Finalize.call( + py, + (response, logger.object(py), kwargs, start, end, cache_hit), + )?; Ok(()) } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs b/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs index cfa9e53aa7b..e321b93b03d 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs @@ -179,6 +179,9 @@ impl CallHooks for LegacyLogging { PythonCallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw }) => { (AFTER.invoke)(self, py, raw) } + PythonCallEvent::Execution(ExecutionEvent::ProviderRequestRetried { retry }) => { + self.request_retried(py, &retry) + } PythonCallEvent::Succeeded { timing, response } => { (SUCCESS.invoke)(self, py, timing, response) } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/python.rs b/litellm-rust/crates/callbacks-legacy-python/src/python.rs index 6d910d35f53..58ddf7c708a 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/python.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/python.rs @@ -42,14 +42,22 @@ pub(crate) enum Logging { Update, #[strum(serialize = "pre_call")] PreCall, + #[strum(serialize = "messages_input")] + MessagesInput, #[strum(serialize = "post_call")] PostCall, #[strum(serialize = "record_response")] RecordResponse, #[strum(serialize = "record_api_window")] RecordApiWindow, + #[strum(serialize = "times_provider_exchange")] + TimesProviderExchange, + #[strum(serialize = "retried_request")] + RetriedRequest, #[strum(serialize = "defers_async_logging")] DefersAsync, + #[strum(serialize = "claim_async_success_log")] + ClaimAsyncSuccess, #[strum(serialize = "defer_success")] DeferSuccess, #[strum(serialize = "sync_success_for_async_call")] @@ -58,6 +66,10 @@ pub(crate) enum Logging { SubmitSuccess, #[strum(serialize = "async_success_handler")] AsyncSuccessHandler, + #[strum(serialize = "cache_hit_success")] + CacheHitSuccess, + #[strum(serialize = "mark_async_success_enqueued")] + MarkAsyncSuccessEnqueued, #[strum(serialize = "enqueue_logging")] Enqueue, #[strum(serialize = "failure_handler")] diff --git a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs index 8acfa7a898b..5b8b2bc4f96 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs @@ -53,7 +53,7 @@ def setup(call_type, args, kwargs, start, asynchronous): FAKES = { 'setup': setup, - 'finalize': lambda response, logger, kwargs, start, end: logger.record('finalize', response), + 'finalize': lambda response, logger, kwargs, start, end, cache_hit: logger.record('finalize', response), 'update_logging': lambda logger, kwargs, model, optional_params, litellm_params, provider: logger.update_from_kwargs( kwargs=kwargs, model=model, @@ -62,17 +62,27 @@ FAKES = { custom_llm_provider=provider, ), 'pre_call': lambda logger, input, api_key, additional_args: logger.pre_call(input, api_key, additional_args), + 'messages_input': lambda body, kwargs: [{'role': 'user', 'content': json.dumps(body)}], 'post_call': lambda logger, original_response, api_key, additional_args: logger.post_call( original_response, api_key, additional_args ), - 'record_response': lambda logger, status, headers, body: logger.model_call_details.update( + 'record_response': lambda logger, status, headers, body, sent: logger.model_call_details.update( response_headers=dict(headers) ), + 'retried_request': lambda logger, logged_body, body, reasons: ( + logger.record('retried_request', (logged_body, body, reasons)), + 'awaitable', + )[1], 'record_api_window': lambda logger, start, end: logger.model_call_details.update( llm_api_duration_ms=(end - start) * 1000 ), + 'times_provider_exchange': lambda client, timeout: client is None, 'defers_async_logging': lambda logger: bool(getattr(logger, '_defer_async_logging', False)), - 'defer_success': lambda logger, pending: setattr(logger, '_native_pending_logging', pending), + 'claim_async_success_log': lambda logger: logger.claim_async_success_log(), + 'defer_success': lambda logger, pending, response, start, end: ( + setattr(logger, 'deferred_success_result', (response, start, end)), + setattr(logger, 'enqueue_deferred_logging', pending), + ), 'sync_success_for_async_call': lambda logger, response, start, end: logger.handle_sync_success_callbacks_for_async_calls( response, start, end ), @@ -81,6 +91,8 @@ FAKES = { )(error, ''.join(traceback.format_exception(error)), start, end), 'submit_success': lambda logger, response, start, end: logger.record('submit', (response, start, end)), 'async_success_handler': lambda logger, response, start, end: logger.async_success_handler(response, start, end), + 'cache_hit_success': lambda logger, response, start, end: logger.record('cache_hit_success', response), + 'mark_async_success_enqueued': lambda logger: logger.record('mark_async_success_enqueued', None), 'enqueue_logging': lambda coroutine: coroutine.enqueue(), 'restore_context': lambda logger: logger.record('restore', None), 'custom_pricing_fields': lambda: ('ocr_cost_per_page',), @@ -90,9 +102,9 @@ FAKES = { 'success', response, call_type ), 'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type), - 'stream_opened': lambda logger, head: logger.record('stream_opened', head), - 'stamp_stream_head': lambda head, logger, kwargs, start, end: logger.record('stamp_stream_head', head), - 'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record( + 'stream_opened': lambda logger, head, keeps_requested: logger.record('stream_opened', head), + 'stamp_stream_head': lambda head, logger, kwargs, start, end, cache_hit: logger.record('stamp_stream_head', head), + 'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk, stream_teardown: logger.record( 'stream_success', list(chunks) ), 'stream_failure': lambda logger, endpoint_type, request_body, chunks, error: logger.record('stream_failure', error), @@ -129,6 +141,7 @@ class StubCoroutine: class StubLogger: def __init__(self): self.calls = [] + self.model_call_details = {} self.hooks = {} self.on_enqueue = lambda coroutine: None @@ -156,6 +169,10 @@ class StubLogger: self.record('async_success_handler', response) return StubCoroutine(self) + def claim_async_success_log(self): + claimed, self.claimed = getattr(self, 'claimed', False), True + return not claimed + def handle_sync_success_callbacks_for_async_calls(self, response, start, end): self.record('sync_success_for_async_call', response) diff --git a/litellm-rust/crates/circuit-breaker/AGENTS.md b/litellm-rust/crates/circuit-breaker/AGENTS.md index 52ef0ca7435..852cd72664a 100644 --- a/litellm-rust/crates/circuit-breaker/AGENTS.md +++ b/litellm-rust/crates/circuit-breaker/AGENTS.md @@ -1,13 +1,14 @@ - `fsm` is a generic state machine runtime and knows nothing about breakers. `breaker` is a generic circuit breaker and knows nothing about Redis. Deciding which errors count as failures belongs to the consumer - `Machine::step` stays pure. `&self` is config and time comes in on the event, never read inside the machine. The only clock read is `Admission`'s drop, at the call-site boundary - Transition rules are one `match (state, event)` per machine, with no macro DSL and no `_` arm that would hide a new state from the compiler -- `Shared` counts a transition, bumps the generation and calls the observer only when the state's enum variant changes, matching Python bumping `generation` only when `_state` changes. Data inside a variant (failure counters, `since`) changes without a transition +- `Shared` counts a transition, bumps the generation and calls the observer only when the state's enum variant changes. Data inside a variant (failure counters, `since`) changes without a transition - The observer runs under the lock so transitions are seen in generation order. Keep it to logging or a metric - A disabled breaker is `Option::None` at the consumer, not a flag inside the machine. `Config` comes in already resolved, reading `REDIS_CIRCUIT_BREAKER_*` env is the consumer's job -## Parity with Python `RedisCircuitBreaker` (`litellm/caching/redis_cache.py`) -- The rstest table `matches_the_python_redis_circuit_breaker` was checked case by case against the Python breaker. A behavior change here needs the same check, by driving the Python class with the same steps -- Divergences, each on purpose +## The only Redis breaker +- Python's `RedisCache` admits through this breaker via `NativeRedisBreaker` (`python-bridge/src/redis/breaker.rs`), sharing one per Redis server with the native client through `redis-connect`'s `RedisHosts`, so the Python `RedisCircuitBreaker` it replaced is gone +- The rstest table `matches_the_python_redis_circuit_breaker` was checked case by case against that Python class and now pins its behavior. A behavior change here changes what every Redis caller in the proxy sees +- Divergences from the old Python class, each on purpose - monotonic `Instant` instead of `time.time()`, so a wall clock jump can't open or close the breaker - a probe `Admission` dropped without an outcome records a hard failure and reopens. Python's probe that dies on `CancelledError` (a `BaseException`) records nothing and leaves the breaker HALF_OPEN for good, and a dropped Rust future is common - `_failure_count` is dropped, it only fed a log line diff --git a/litellm-rust/crates/circuit-breaker/src/breaker.rs b/litellm-rust/crates/circuit-breaker/src/breaker.rs index c9509561b37..cf4961ef099 100644 --- a/litellm-rust/crates/circuit-breaker/src/breaker.rs +++ b/litellm-rust/crates/circuit-breaker/src/breaker.rs @@ -1,5 +1,5 @@ use std::{ - sync::Arc, + sync::{Arc, Weak}, time::{Duration, Instant}, }; @@ -128,6 +128,9 @@ impl Machine for Config { #[derive(Clone)] pub struct Breaker(Arc>); +#[derive(Clone)] +pub struct WeakBreaker(Weak>); + #[derive(Debug, PartialEq, Eq)] pub struct Rejected; @@ -165,6 +168,16 @@ impl Breaker { pub fn state(&self) -> State { self.0.state() } + + pub fn downgrade(&self) -> WeakBreaker { + WeakBreaker(Arc::downgrade(&self.0)) + } +} + +impl WeakBreaker { + pub fn upgrade(&self) -> Option { + self.0.upgrade().map(Breaker) + } } impl Admission { diff --git a/litellm-rust/crates/circuit-breaker/src/lib.rs b/litellm-rust/crates/circuit-breaker/src/lib.rs index e76228122d1..af8df68246e 100644 --- a/litellm-rust/crates/circuit-breaker/src/lib.rs +++ b/litellm-rust/crates/circuit-breaker/src/lib.rs @@ -1,5 +1,7 @@ mod breaker; mod fsm; -pub use breaker::{Admission, Breaker, Config, Event, Failure, Output, Rejected, State}; +pub use breaker::{ + Admission, Breaker, Config, Event, Failure, Output, Rejected, State, WeakBreaker, +}; pub use fsm::{Machine, Observer, Shared, Transition}; diff --git a/litellm-rust/crates/config/AGENTS.md b/litellm-rust/crates/config/AGENTS.md new file mode 100644 index 00000000000..9f6f1db9a67 --- /dev/null +++ b/litellm-rust/crates/config/AGENTS.md @@ -0,0 +1,3 @@ +## Redis settings + +- This crate only deserializes YAML. `RedisSettings` is the raw `coordination_redis` or `cache_params` map and `CacheSettings` tells a `type: redis` cache apart from the rest. Reading those maps the way `litellm/_redis.py` does belongs to `gateway/src/redis`, and turning merged kwargs into a `Resolution` belongs to `redis-resolve` diff --git a/litellm-rust/crates/config/Cargo.toml b/litellm-rust/crates/config/Cargo.toml index 278aea6200e..a433aacc1c9 100644 --- a/litellm-rust/crates/config/Cargo.toml +++ b/litellm-rust/crates/config/Cargo.toml @@ -6,17 +6,16 @@ license.workspace = true repository.workspace = true [dependencies] -litellm-secret-value.workspace = true litellm-router-types.workspace = true +litellm-secret-value.workspace = true serde.workspace = true -serde_json.workspace = true serde_yaml_ng = "0.10.0" strum.workspace = true thiserror.workspace = true -tracing.workspace = true +veil.workspace = true [dev-dependencies] litellm-error.workspace = true rstest.workspace = true +serde_json.workspace = true tempfile.workspace = true -tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] } diff --git a/litellm-rust/crates/config/src/error.rs b/litellm-rust/crates/config/src/error.rs index e5219ac992c..300f15380bd 100644 --- a/litellm-rust/crates/config/src/error.rs +++ b/litellm-rust/crates/config/src/error.rs @@ -11,11 +11,4 @@ pub enum Error { path: PathBuf, source: serde_yaml_ng::Error, }, - #[error("Redis host does not yet support configured setting {0}")] - RedisSetting(String), - #[error("invalid Redis setting {setting}: {reason}")] - RedisInvalid { - setting: &'static str, - reason: &'static str, - }, } diff --git a/litellm-rust/crates/config/src/lib.rs b/litellm-rust/crates/config/src/lib.rs index b8af6f11d2e..46657aa91e7 100644 --- a/litellm-rust/crates/config/src/lib.rs +++ b/litellm-rust/crates/config/src/lib.rs @@ -14,10 +14,7 @@ pub use error::Error; pub use litellm_router_types::{LitellmParams, Spelled}; pub use mcp::{McpAuth, McpServer, McpTransport}; pub use model::Model; -pub use redis::{ - CacheSettings, Lookup, RedisCluster, RedisEndpoint, RedisHost, RedisNode, RedisSettings, - ResolvedRedis, environment_has_target, -}; +pub use redis::{CacheSettings, RedisSettings}; pub use settings::{ ClickHouseStoreSettings, GeneralSettings, LiteLlmSettings, RouterSettings, TracingSettings, TracingStoreSettings, diff --git a/litellm-rust/crates/config/src/redis.rs b/litellm-rust/crates/config/src/redis.rs index 7a91691c54b..d0ba107242c 100644 --- a/litellm-rust/crates/config/src/redis.rs +++ b/litellm-rust/crates/config/src/redis.rs @@ -1,195 +1,21 @@ -use std::{collections::BTreeMap, fmt, time::Duration}; +use std::collections::BTreeMap; -use litellm_router_types::Spelled; -use litellm_secret_value::SecretValue; use serde::{Deserialize, Deserializer, de::Error as _}; +use veil::Redact; -use crate::{AdditionalFields, Error, Object, Value}; +use crate::{Object, Value}; -pub type Lookup<'a> = &'a dyn Fn(&str) -> Option; - -const SUPPORTED: &[&str] = &[ - "url", - "host", - "port", - "username", - "password", - "ssl", - "db", - "socket_timeout", - "startup_nodes", - "namespace", -]; - -const IGNORED: &[&str] = &[ - "max_connections", - "single_connection_client", - "health_check_interval", - "socket_keepalive", - "socket_keepalive_options", - "decode_responses", - "encoding", - "encoding_errors", - "charset", - "errors", - "client_name", - "lib_name", - "lib_version", - "sentinel_password", - "gcp_ssl_ca_certs", - "azure_client_id", - "azure_tenant_id", - "azure_client_secret", - "aws_iam_user_name", - "aws_iam_cache_name", - "aws_iam_region", - "aws_iam_serverless", -]; - -const WARN_IGNORED: &[&str] = &[ - "max_connections", - "health_check_interval", - "socket_keepalive", - "socket_keepalive_options", -]; - -const REDIS_PY_KWARGS: &[&str] = &[ - "aws_iam_auth", - "aws_iam_cache_name", - "aws_iam_region", - "aws_iam_serverless", - "aws_iam_user_name", - "azure_client_id", - "azure_client_secret", - "azure_redis_ad_token", - "azure_tenant_id", - "cache", - "cache_config", - "charset", - "client_name", - "db", - "decode_responses", - "encoding", - "encoding_errors", - "errors", - "event_dispatcher", - "gcp_service_account", - "gcp_ssl_ca_certs", - "health_check_interval", - "host", - "lib_name", - "lib_version", - "max_connections", - "password", - "port", - "protocol", - "redis_connect_func", - "retry_on_error", - "retry_on_timeout", - "single_connection_client", - "socket_connect_timeout", - "socket_keepalive", - "socket_keepalive_options", - "socket_timeout", - "ssl", - "ssl_ca_certs", - "ssl_ca_data", - "ssl_ca_path", - "ssl_cert_reqs", - "ssl_certfile", - "ssl_check_hostname", - "ssl_ciphers", - "ssl_keyfile", - "ssl_min_version", - "ssl_ocsp_context", - "ssl_ocsp_expected_cert", - "ssl_password", - "ssl_validate_ocsp", - "ssl_validate_ocsp_stapled", - "unix_socket_path", - "url", - "username", -]; - -const UNSUPPORTED_EXTRA: &[&str] = &[ - "credential_provider", - "sentinel_nodes", - "retry", - "connection_pool", -]; - -const CACHE_ONLY: &[&str] = &[ - "type", - "mode", - "ttl", - "default_in_memory_ttl", - "default_in_redis_ttl", - "supported_call_types", - "similarity_threshold", - "max_messages", - "_backend", -]; - -#[derive(Clone, Default)] +/// A Redis section as `config.yaml` spells it, before any surface reads it. +#[derive(Clone, Default, Deserialize, Redact)] +#[serde(transparent)] pub struct RedisSettings { - pub url: Option, - pub host: Option>, - pub port: Option>, - pub username: Option, - pub password: Option, - pub ssl: Option>, - pub db: Option>, - pub socket_timeout: Option>, - pub startup_nodes: Option, - pub namespace: Option>, - pub additional_fields: AdditionalFields, - unparsed: DeferredInputs, + #[redact(with = "[REDACTED]")] + fields: BTreeMap, } -impl<'de> Deserialize<'de> for RedisSettings { - fn deserialize>(deserializer: D) -> Result { - let fields = BTreeMap::::deserialize(deserializer)?; - let scalar = |key: &str| fields.get(key).and_then(as_text); - Ok(Self { - url: scalar("url").map(SecretValue::new), - host: scalar("host").map(Spelled::Value), - port: spelled(scalar("port")), - username: scalar("username").map(SecretValue::new), - password: scalar("password").map(SecretValue::new), - ssl: spelled(scalar("ssl")), - db: spelled(scalar("db")), - socket_timeout: spelled(scalar("socket_timeout")), - startup_nodes: fields - .get("startup_nodes") - .filter(|value| !value.is_null()) - .cloned(), - namespace: scalar("namespace").map(Spelled::Value), - additional_fields: cloned(&fields, |key, _| !SUPPORTED.contains(&key)), - unparsed: deferred_inputs(&fields), - }) - } -} - -impl fmt::Debug for RedisSettings { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("RedisSettings") - .field("url", &self.url) - .field("host", &self.host) - .field("port", &self.port) - .field("username", &self.username) - .field("password", &self.password) - .field("ssl", &self.ssl) - .field("db", &self.db) - .field("socket_timeout", &self.socket_timeout) - .field( - "startup_nodes", - &self.startup_nodes.as_ref().map(|_| ""), - ) - .field("namespace", &self.namespace) - .field("additional_fields", &self.additional_fields.keys()) - .field("unparsed", &self.unparsed.keys()) - .finish() +impl RedisSettings { + pub fn fields(&self) -> &BTreeMap { + &self.fields } } @@ -216,452 +42,22 @@ impl<'de> Deserialize<'de> for CacheSettings { } } -#[derive(Clone, PartialEq)] -pub struct ResolvedRedis { - pub endpoint: RedisEndpoint, - pub namespace: Option, - pub timeout: Option, -} +#[cfg(test)] +mod tests { + use super::RedisSettings; + use rstest::rstest; -impl fmt::Debug for ResolvedRedis { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("ResolvedRedis") - .field("endpoint", &self.endpoint) - .field("namespace", &self.namespace) - .field("timeout", &self.timeout) - .finish() + #[rstest] + #[case::scalar(serde_json::json!("credential-value"))] + #[case::object(serde_json::json!({"secret": "credential-value"}))] + #[case::array(serde_json::json!(["credential-value"]))] + fn credentials_are_redacted(#[case] password: serde_json::Value) { + let settings: RedisSettings = serde_json::from_value(serde_json::json!({ + "host": "redis", "password": password, + })) + .unwrap(); + let debug = format!("{settings:?}"); + assert!(!debug.contains("credential-value")); + assert!(debug.contains("[REDACTED]")); } } - -#[derive(Clone, PartialEq)] -pub enum RedisEndpoint { - Url(SecretValue), - Host(RedisHost), - Cluster(RedisCluster), -} - -impl fmt::Debug for RedisEndpoint { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Url(_) => formatter.debug_tuple("Url").field(&"").finish(), - Self::Host(host) => host.fmt(formatter), - Self::Cluster(cluster) => cluster.fmt(formatter), - } - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct RedisHost { - pub host: String, - pub port: u16, - pub username: Option, - pub password: Option, - pub tls: bool, - pub db: i64, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct RedisCluster { - pub nodes: Vec, - pub username: Option, - pub password: Option, - pub tls: bool, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct RedisNode { - pub host: String, - pub port: u16, -} - -#[derive(Clone)] -struct Entry { - value: Value, - source: String, -} - -type Kwargs = BTreeMap; - -impl RedisSettings { - pub fn resolve(&self, lookup: Lookup<'_>) -> Result { - self.resolve_with(lookup, false) - } - - pub fn resolve_cache(&self, lookup: Lookup<'_>) -> Result { - self.resolve_with(lookup, true) - } - - fn resolve_with(&self, lookup: Lookup<'_>, cache: bool) -> Result { - let block = self.kwargs(cache); - let block: Kwargs = block - .into_iter() - .filter_map(|(key, entry)| { - let value = environment_reference(&entry.value, lookup)?; - Some((key, Entry { value, ..entry })) - }) - .collect(); - let environment = environment_kwargs(lookup); - let names_a_target = ["host", "startup_nodes", "sentinel_nodes"] - .iter() - .any(|key| block.contains_key(*key)); - let drop_environment_url = names_a_target && !block.contains_key("url"); - let merged: Kwargs = environment - .into_iter() - .filter(|(key, _)| !(key == "url" && drop_environment_url)) - .chain(block) - .collect(); - reject_unsupported(&merged, lookup)?; - warn_ignored(&merged); - - let startup_nodes = match merged.get("startup_nodes") { - Some(entry) => Some(cluster_nodes(&entry.value)?), - None => lookup("REDIS_CLUSTER_NODES") - .map(|nodes| cluster_nodes(&Value::String(nodes))) - .transpose()?, - }; - let username = secret_value(&merged, "username"); - let password = secret_value(&merged, "password"); - let tls = bool_value(&merged, "ssl"); - let endpoint = match (startup_nodes, text_value(&merged, "url")) { - (Some(nodes), _) if !nodes.is_empty() => RedisEndpoint::Cluster(RedisCluster { - nodes, - username, - password, - tls, - }), - (_, Some(url)) => RedisEndpoint::Url(SecretValue::new(url)), - _ => RedisEndpoint::Host(RedisHost { - host: text_value(&merged, "host").ok_or(Error::RedisInvalid { - setting: "host", - reason: "either host or url must be specified for redis", - })?, - port: integer(&merged, "port")?.unwrap_or(6379), - username, - password, - tls, - db: integer(&merged, "db")?.unwrap_or(0), - }), - }; - let timeout = merged - .get("socket_timeout") - .and_then(|entry| as_text(&entry.value)) - .map(|seconds| { - seconds - .parse::() - .ok() - .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) - .ok_or(Error::RedisInvalid { - setting: "socket_timeout", - reason: "must be a non-negative number of seconds", - }) - }) - .transpose()?; - Ok(ResolvedRedis { - endpoint, - namespace: text_value(&merged, "namespace"), - timeout, - }) - } - - fn kwargs(&self, cache: bool) -> Kwargs { - let known = [ - ( - "url", - self.url.as_ref().map(|value| value.expose().to_owned()), - ), - ("host", spelled_text(self.host.as_ref())), - ("port", spelled_text(self.port.as_ref())), - ( - "username", - self.username - .as_ref() - .map(|value| value.expose().to_owned()), - ), - ( - "password", - self.password - .as_ref() - .map(|value| value.expose().to_owned()), - ), - ("ssl", spelled_text(self.ssl.as_ref())), - ("db", spelled_text(self.db.as_ref())), - ("socket_timeout", spelled_text(self.socket_timeout.as_ref())), - ("namespace", spelled_text(self.namespace.as_ref())), - ]; - let known = known.into_iter().filter_map(|(key, value)| { - value.map(|value| { - ( - key.to_owned(), - Entry { - value: Value::String(value), - source: key.to_owned(), - }, - ) - }) - }); - let startup_nodes = self.startup_nodes.clone().map(|value| { - ( - "startup_nodes".to_owned(), - Entry { - value, - source: "startup_nodes".to_owned(), - }, - ) - }); - let extras = self.additional_fields.iter().filter_map(|(key, value)| { - if cache && CACHE_ONLY.contains(&key.as_str()) { - return None; - } - let key = match (cache, key.as_str()) { - (true, "redis_startup_nodes") => "startup_nodes".to_owned(), - _ => key.clone(), - }; - Some(( - key.clone(), - Entry { - value: value.clone(), - source: key, - }, - )) - }); - let unparsed = self.unparsed.iter().map(|(key, value)| { - ( - key.clone(), - Entry { - value: value.0.clone(), - source: key.clone(), - }, - ) - }); - extras - .chain(unparsed) - .chain(known) - .chain(startup_nodes) - .collect() - } -} - -pub fn environment_has_target(lookup: Lookup<'_>) -> bool { - [ - "REDIS_HOST", - "REDIS_URL", - "REDIS_CLUSTER_NODES", - "REDIS_SENTINEL_NODES", - ] - .iter() - .any(|name| lookup(name).is_some()) -} - -fn reject_unsupported(merged: &Kwargs, lookup: Lookup<'_>) -> Result<(), Error> { - let active_auth = [ - merged - .get("gcp_service_account") - .map(|entry| entry.source.as_str()) - .or_else(|| lookup("REDIS_GCP_SERVICE_ACCOUNT").map(|_| "REDIS_GCP_SERVICE_ACCOUNT")), - text_value(merged, "azure_redis_ad_token") - .or_else(|| lookup("REDIS_AZURE_AD_TOKEN")) - .filter(|value| value.eq_ignore_ascii_case("true")) - .map(|_| { - merged - .get("azure_redis_ad_token") - .map_or("REDIS_AZURE_AD_TOKEN", |entry| entry.source.as_str()) - }), - bool_value(merged, "aws_iam_auth").then(|| merged["aws_iam_auth"].source.as_str()), - merged - .get("sentinel_nodes") - .map(|entry| entry.source.as_str()) - .or_else(|| lookup("REDIS_SENTINEL_NODES").map(|_| "REDIS_SENTINEL_NODES")), - ]; - if let Some(setting) = active_auth.into_iter().flatten().next() { - return Err(Error::RedisSetting(setting.to_owned())); - } - match merged - .iter() - .find(|(key, _)| is_unsupported(key)) - .map(|(_, entry)| entry.source.clone()) - { - Some(setting) => Err(Error::RedisSetting(setting)), - None => Ok(()), - } -} - -fn warn_ignored(merged: &Kwargs) { - for setting in WARN_IGNORED { - if merged.contains_key(*setting) { - tracing::warn!(setting, "Redis setting is accepted but ignored"); - } - } -} - -fn is_unsupported(key: &str) -> bool { - let known = REDIS_PY_KWARGS.contains(&key) || UNSUPPORTED_EXTRA.contains(&key); - let handled = SUPPORTED.contains(&key) - || IGNORED.contains(&key) - || matches!( - key, - "gcp_service_account" | "azure_redis_ad_token" | "aws_iam_auth" - ); - known && !handled -} - -fn environment_kwargs(lookup: Lookup<'_>) -> Kwargs { - REDIS_PY_KWARGS - .iter() - .filter_map(|key| { - let source = format!("REDIS_{}", key.to_ascii_uppercase()); - let value = lookup(&source)?; - Some(( - (*key).to_owned(), - Entry { - value: Value::String(value), - source, - }, - )) - }) - .collect() -} - -fn environment_reference(value: &Value, lookup: Lookup<'_>) -> Option { - match value { - Value::String(text) => match text.strip_prefix("os.environ/") { - Some(name) => lookup(name).map(Value::String), - None => Some(value.clone()), - }, - Value::Null => None, - _ => Some(value.clone()), - } -} - -fn as_text(value: &Value) -> Option { - match value { - Value::String(text) => Some(text.clone()), - Value::Number(number) => Some(number.to_string()), - Value::Bool(flag) => Some(flag.to_string()), - _ => None, - } -} - -fn text_value(kwargs: &Kwargs, key: &str) -> Option { - kwargs.get(key).and_then(|entry| as_text(&entry.value)) -} - -fn secret_value(kwargs: &Kwargs, key: &str) -> Option { - text_value(kwargs, key).map(SecretValue::new) -} - -fn bool_value(kwargs: &Kwargs, key: &str) -> bool { - match kwargs.get(key).map(|entry| &entry.value) { - Some(Value::Bool(flag)) => *flag, - Some(value) => as_text(value) - .is_some_and(|text| matches!(text.to_ascii_lowercase().as_str(), "true" | "1" | "yes")), - None => false, - } -} - -fn integer(kwargs: &Kwargs, key: &'static str) -> Result, Error> { - kwargs - .get(key) - .and_then(|entry| as_text(&entry.value)) - .map(|text| { - text.parse().map_err(|_| Error::RedisInvalid { - setting: key, - reason: "must be an integer", - }) - }) - .transpose() -} - -const INVALID_NODES: Error = Error::RedisInvalid { - setting: "startup_nodes", - reason: "must be a list of {host, port} objects", -}; - -fn cluster_nodes(value: &Value) -> Result, Error> { - match value { - Value::String(json) => { - let nodes: Vec> = - serde_json::from_str(json).map_err(|_| INVALID_NODES)?; - nodes - .iter() - .map(|node| { - cluster_node( - node.get("host").and_then(json_text), - node.get("port").and_then(json_text), - ) - }) - .collect() - } - Value::Sequence(nodes) => nodes - .iter() - .map(|node| { - cluster_node( - node.get("host").and_then(as_text), - node.get("port").and_then(as_text), - ) - }) - .collect(), - _ => Err(INVALID_NODES), - } -} - -fn cluster_node(host: Option, port: Option) -> Result { - match (host, port.and_then(|port| port.parse().ok())) { - (Some(host), Some(port)) => Ok(RedisNode { host, port }), - _ => Err(INVALID_NODES), - } -} - -fn json_text(value: &serde_json::Value) -> Option { - match value { - serde_json::Value::String(text) => Some(text.clone()), - serde_json::Value::Number(number) => Some(number.to_string()), - _ => None, - } -} - -fn spelled_text(value: Option<&Spelled>) -> Option { - value.map(|value| match value { - Spelled::Value(value) => value.to_string(), - Spelled::Text(value) => value.clone(), - }) -} - -fn spelled(text: Option) -> Option> { - text.map(|text| match text.parse() { - Ok(value) => Spelled::Value(value), - Err(_) => Spelled::Text(text), - }) -} - -fn cloned( - fields: &BTreeMap, - keep: impl Fn(&str, &Value) -> bool, -) -> AdditionalFields { - fields - .iter() - .filter(|(key, value)| keep(key, value)) - .map(|(key, value)| (key.clone(), value.clone())) - .collect() -} - -#[derive(Clone)] -struct DeferredInput(Value); - -type DeferredInputs = BTreeMap; - -fn deferred_inputs(fields: &BTreeMap) -> DeferredInputs { - fields - .iter() - .filter(|(key, value)| is_unparsed(key, value)) - .map(|(key, value)| (key.clone(), DeferredInput(value.clone()))) - .collect() -} - -fn is_unparsed(key: &str, value: &Value) -> bool { - SUPPORTED.contains(&key) - && match key { - "startup_nodes" => value.is_null(), - _ => as_text(value).is_none(), - } -} diff --git a/litellm-rust/crates/config/tests/config.rs b/litellm-rust/crates/config/tests/config.rs index 3c7d02203e5..fec042e41be 100644 --- a/litellm-rust/crates/config/tests/config.rs +++ b/litellm-rust/crates/config/tests/config.rs @@ -440,3 +440,29 @@ fn mcp_config_redacts_nested_credentials_and_preserves_policy_for_validation() { fn rejects_invalid_typed_mcp_settings(#[case] setting: &str) { assert!(Config::from_yaml(&format!("mcp_servers:\n docs:\n {setting}\n")).is_err()); } + +#[rstest] +fn inactive_redis_cache_keeps_malformed_known_inputs_deferred() { + let config = Config::from_yaml( + "litellm_settings:\n cache: false\n cache_params:\n type: redis\n host: []\n port: invalid\n socket_timeout: invalid", + ) + .unwrap(); + + assert!(matches!( + config.litellm_settings.cache_params, + Some(litellm_config::CacheSettings::Redis(_)) + )); +} + +#[rstest] +fn non_redis_cache_settings_remain_opaque() { + let config = Config::from_yaml( + "litellm_settings:\n cache_params:\n type: local\n custom_option:\n nested: true", + ) + .unwrap(); + + assert!(matches!( + config.litellm_settings.cache_params, + Some(litellm_config::CacheSettings::Other(_)) + )); +} diff --git a/litellm-rust/crates/config/tests/redis.rs b/litellm-rust/crates/config/tests/redis.rs deleted file mode 100644 index ab409541c8e..00000000000 --- a/litellm-rust/crates/config/tests/redis.rs +++ /dev/null @@ -1,405 +0,0 @@ -use std::{ - collections::HashMap, - fmt::Write, - sync::{Arc, Mutex}, -}; - -use litellm_config::{CacheSettings, Config, Error, RedisEndpoint, RedisSettings, ResolvedRedis}; -use rstest::rstest; -use tracing::{Event, Subscriber, field::Visit}; -use tracing_subscriber::{Layer, layer::Context, prelude::*}; - -fn resolve( - yaml: &str, - environment: &[(&str, &str)], -) -> Result { - let environment: HashMap = environment - .iter() - .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) - .collect(); - let config = Config::from_yaml(yaml).unwrap(); - config - .general_settings - .coordination_redis - .as_ref() - .unwrap_or(&RedisSettings::default()) - .resolve(&|name| environment.get(name).cloned()) -} - -fn resolve_cache(yaml: &str) -> Result { - let config = Config::from_yaml(yaml).unwrap(); - match config.litellm_settings.cache_params.unwrap() { - CacheSettings::Redis(settings) => settings.resolve_cache(&|_| None), - CacheSettings::Other(_) => panic!("expected a Redis cache"), - } -} - -fn cluster_ports(resolved: &ResolvedRedis) -> Vec { - match &resolved.endpoint { - RedisEndpoint::Cluster(cluster) => cluster.nodes.iter().map(|node| node.port).collect(), - endpoint => panic!("expected a cluster, got {endpoint:?}"), - } -} - -#[rstest] -#[case::host( - "general_settings:\n coordination_redis:\n host: redis\n port: 6380\n db: 2\n ssl: true\n namespace: app\n socket_timeout: 1.5", - &[], - "host:redis:6380:2:true:app:1500" -)] -#[case::environment_url( - "model_list: []", - &[("REDIS_URL", "redis://environment:6379")], - "url:redis://environment:6379" -)] -#[case::explicit_host_drops_environment_url( - "general_settings:\n coordination_redis:\n host: configured", - &[("REDIS_URL", "redis://environment:6379")], - "host:configured:6379:0:false::none" -)] -#[case::yaml_cluster( - "general_settings:\n coordination_redis:\n startup_nodes:\n - host: node-a\n port: 7000\n - host: node-b\n port: \"7001\"", - &[], - "cluster:node-a:7000,node-b:7001" -)] -#[case::json_environment_cluster( - "model_list: []", - &[("REDIS_CLUSTER_NODES", r#"[{"host":"node-a","port":"7000"},{"host":"node-b","port":7001}]"#)], - "cluster:node-a:7000,node-b:7001" -)] -#[case::missing_reference_keeps_environment_fallback( - "general_settings:\n coordination_redis:\n host: os.environ/MISSING_HOST", - &[("REDIS_HOST", "environment")], - "host:environment:6379:0:false::none" -)] -#[case::configured_values_beat_the_same_environment_values( - "general_settings:\n coordination_redis:\n host: configured\n port: 6380\n db: 3", - &[("REDIS_HOST", "environment"), ("REDIS_PORT", "6390"), ("REDIS_DB", "9")], - "host:configured:6380:3:false::none" -)] -#[case::environment_url_beats_the_environment_host( - "model_list: []", - &[("REDIS_URL", "redis://environment:6379"), ("REDIS_HOST", "ignored")], - "url:redis://environment:6379" -)] -#[case::startup_nodes_beat_a_url( - "general_settings:\n coordination_redis:\n url: redis://single:6379\n startup_nodes:\n - host: node-a\n port: 7000", - &[], - "cluster:node-a:7000" -)] -#[case::environment_references_resolve_the_host_and_port( - "general_settings:\n coordination_redis:\n host: os.environ/COORD_HOST\n port: os.environ/COORD_PORT", - &[("COORD_HOST", "referenced"), ("COORD_PORT", "6400")], - "host:referenced:6400:0:false::none" -)] -#[case::a_null_host_falls_back_to_the_environment( - "general_settings:\n coordination_redis:\n host: null", - &[("REDIS_HOST", "environment")], - "host:environment:6379:0:false::none" -)] -fn resolves_redis_through_the_parsed_config( - #[case] yaml: &str, - #[case] environment: &[(&str, &str)], - #[case] expected: &str, -) { - let resolved = resolve(yaml, environment).unwrap(); - let description = match resolved.endpoint { - RedisEndpoint::Url(url) => format!("url:{}", url.expose()), - RedisEndpoint::Host(host) => format!( - "host:{}:{}:{}:{}:{}:{}", - host.host, - host.port, - host.db, - host.tls, - resolved.namespace.unwrap_or_default(), - resolved - .timeout - .map(|timeout| timeout.as_millis().to_string()) - .unwrap_or_else(|| "none".to_owned()) - ), - RedisEndpoint::Cluster(cluster) => format!( - "cluster:{}", - cluster - .nodes - .iter() - .map(|node| format!("{}:{}", node.host, node.port)) - .collect::>() - .join(",") - ), - }; - - assert_eq!(description, expected); -} - -#[rstest] -#[case::custom_tls_file( - "general_settings:\n coordination_redis:\n host: redis\n ssl_ca_certs: /ca.pem", - &[], - "ssl_ca_certs" -)] -#[case::gcp_auth_from_config( - "general_settings:\n coordination_redis:\n host: redis\n gcp_service_account: account", - &[], - "gcp_service_account" -)] -#[case::gcp_auth_from_environment( - "model_list: []", - &[("REDIS_HOST", "redis"), ("REDIS_GCP_SERVICE_ACCOUNT", "account")], - "REDIS_GCP_SERVICE_ACCOUNT" -)] -#[case::azure_auth_from_config( - "general_settings:\n coordination_redis:\n host: redis\n azure_redis_ad_token: true", - &[], - "azure_redis_ad_token" -)] -#[case::azure_auth_from_environment( - "model_list: []", - &[("REDIS_HOST", "redis"), ("REDIS_AZURE_AD_TOKEN", "true")], - "REDIS_AZURE_AD_TOKEN" -)] -#[case::aws_auth_from_config( - "general_settings:\n coordination_redis:\n host: redis\n aws_iam_auth: true", - &[], - "aws_iam_auth" -)] -#[case::aws_auth_from_environment( - "model_list: []", - &[("REDIS_HOST", "redis"), ("REDIS_AWS_IAM_AUTH", "true")], - "REDIS_AWS_IAM_AUTH" -)] -#[case::sentinel_from_config( - "general_settings:\n coordination_redis:\n sentinel_nodes: [[redis, 26379]]", - &[], - "sentinel_nodes" -)] -#[case::sentinel_from_environment( - "model_list: []", - &[("REDIS_SENTINEL_NODES", "[[\"redis\",26379]]")], - "REDIS_SENTINEL_NODES" -)] -fn rejected_settings_name_their_source( - #[case] yaml: &str, - #[case] environment: &[(&str, &str)], - #[case] expected: &str, -) { - let error = resolve(yaml, environment).unwrap_err(); - - assert!(matches!(error, Error::RedisSetting(setting) if setting == expected)); -} - -#[rstest] -fn an_explicit_host_keeps_environment_credentials() { - let resolved = resolve( - "general_settings:\n coordination_redis:\n host: configured", - &[ - ("REDIS_PASSWORD", "environment-secret"), - ("REDIS_SSL", "True"), - ], - ) - .unwrap(); - - let RedisEndpoint::Host(host) = resolved.endpoint else { - panic!("expected a host endpoint"); - }; - assert_eq!( - host.password.as_ref().map(|password| password.expose()), - Some("environment-secret") - ); - assert!(host.tls); -} - -#[rstest] -#[case::only_a_port("general_settings:\n coordination_redis:\n port: 6379", &[])] -#[case::a_structured_host_with_an_environment_target( - "general_settings:\n coordination_redis:\n host: []", - &[("REDIS_HOST", "environment"), ("REDIS_URL", "redis://environment:6379")] -)] -fn a_block_without_a_usable_target_fails(#[case] yaml: &str, #[case] environment: &[(&str, &str)]) { - let error = resolve(yaml, environment).unwrap_err(); - - assert!(matches!( - error, - Error::RedisInvalid { - setting: "host", - .. - } - )); -} - -#[rstest] -fn the_cache_alias_names_cluster_nodes() { - let resolved = resolve_cache( - "litellm_settings:\n cache_params:\n type: redis\n redis_startup_nodes:\n - host: node-a\n port: 7000", - ) - .unwrap(); - - assert_eq!(cluster_ports(&resolved), vec![7000]); -} - -#[rstest] -fn canonical_startup_nodes_beat_the_cache_alias() { - let resolved = resolve_cache( - "litellm_settings:\n cache_params:\n type: redis\n startup_nodes:\n - host: canonical\n port: 7000\n redis_startup_nodes:\n - host: alias\n port: 7100\n - host: alias\n port: 7101", - ) - .unwrap(); - - assert_eq!(cluster_ports(&resolved), vec![7000]); -} - -#[rstest] -fn a_null_canonical_startup_nodes_setting_shadows_the_cache_alias() { - let resolved = resolve_cache( - "litellm_settings:\n cache_params:\n type: redis\n host: cache\n startup_nodes: null\n redis_startup_nodes:\n - host: alias\n port: 7100", - ) - .unwrap(); - - assert!(matches!(resolved.endpoint, RedisEndpoint::Host(_))); -} - -#[rstest] -fn a_missing_canonical_startup_nodes_reference_shadows_the_cache_alias() { - let resolved = resolve_cache( - "litellm_settings:\n cache_params:\n type: redis\n host: cache\n startup_nodes: os.environ/MISSING_NODES\n redis_startup_nodes:\n - host: alias\n port: 7100", - ) - .unwrap(); - - assert!(matches!(resolved.endpoint, RedisEndpoint::Host(_))); -} - -#[rstest] -fn rejected_environment_settings_never_print_the_value() { - let error = resolve( - "model_list: []", - &[ - ("REDIS_HOST", "redis"), - ("REDIS_SSL_CERT_REQS", "private-option-value"), - ], - ) - .unwrap_err(); - let message = error.to_string(); - - assert!(message.contains("REDIS_SSL_CERT_REQS")); - assert!(!message.contains("private-option-value")); -} - -#[rstest] -#[case::negative("-1")] -#[case::not_a_number("slow")] -fn rejects_bad_socket_timeout(#[case] timeout: &str) { - let yaml = format!( - "general_settings:\n coordination_redis:\n host: redis\n socket_timeout: {timeout}" - ); - let error = resolve(&yaml, &[]).unwrap_err(); - - assert!(matches!( - error, - Error::RedisInvalid { - setting: "socket_timeout", - .. - } - )); -} - -#[rstest] -fn defaults_socket_timeout_for_the_gateway() { - let resolved = resolve( - "general_settings:\n coordination_redis:\n host: redis", - &[], - ) - .unwrap(); - - assert_eq!(resolved.timeout, None); -} - -#[rstest] -fn debug_redacts_redis_secrets_and_url() { - let config = Config::from_yaml( - "general_settings:\n coordination_redis:\n url: redis://user:password@redis:6379\n username: credential-user\n password: credential-password", - ) - .unwrap(); - let settings = config.general_settings.coordination_redis.unwrap(); - let resolved = settings.resolve(&|_| None).unwrap(); - let debug = format!("{settings:?} {resolved:?}"); - - assert!(!debug.contains("redis://user:password")); - assert!(!debug.contains("credential-user")); - assert!(!debug.contains("credential-password")); - assert!(debug.contains("redacted") || debug.contains("REDACTED")); -} - -#[rstest] -fn inactive_redis_cache_keeps_malformed_known_inputs_deferred() { - let config = Config::from_yaml( - "litellm_settings:\n cache: false\n cache_params:\n type: redis\n host: []\n port: invalid\n socket_timeout: invalid", - ) - .unwrap(); - - assert!(matches!( - config.litellm_settings.cache_params, - Some(litellm_config::CacheSettings::Redis(_)) - )); -} - -#[rstest] -fn non_redis_cache_settings_remain_opaque() { - let config = Config::from_yaml( - "litellm_settings:\n cache_params:\n type: local\n custom_option:\n nested: true", - ) - .unwrap(); - - assert!(matches!( - config.litellm_settings.cache_params, - Some(litellm_config::CacheSettings::Other(_)) - )); -} - -#[rstest] -fn legacy_unknown_settings_do_not_change_resolution() { - let resolved = resolve( - "general_settings:\n coordination_redis:\n host: redis\n redis_flush_size: 42\n service_name: primary", - &[], - ) - .unwrap(); - - assert!(matches!(resolved.endpoint, RedisEndpoint::Host(_))); -} - -struct WarningCapture(Arc>>); - -impl Layer for WarningCapture { - fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) { - let mut fields = Fields::default(); - event.record(&mut fields); - self.0.lock().unwrap().push(fields.0); - } -} - -#[derive(Default)] -struct Fields(String); - -impl Visit for Fields { - fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { - write!(self.0, "{}={value:?} ", field.name()).unwrap(); - } - - fn record_str(&mut self, field: &tracing::field::Field, value: &str) { - write!(self.0, "{}={value} ", field.name()).unwrap(); - } -} - -#[rstest] -fn ignored_setting_warning_names_the_setting_without_its_value() { - let events = Arc::new(Mutex::new(Vec::new())); - let subscriber = tracing_subscriber::registry().with(WarningCapture(events.clone())); - tracing::subscriber::with_default(subscriber, || { - resolve( - "general_settings:\n coordination_redis:\n host: redis\n max_connections: 47", - &[], - ) - .unwrap(); - }); - let warnings = events.lock().unwrap().join(" "); - - assert!(warnings.contains("max_connections")); - assert!(!warnings.contains("47")); -} diff --git a/litellm-rust/crates/gateway-forward/AGENTS.md b/litellm-rust/crates/gateway-forward/AGENTS.md new file mode 100644 index 00000000000..3d07f9ee342 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/AGENTS.md @@ -0,0 +1,40 @@ +# Forwarding gateway + +`Forward` is a reverse proxy in front of the Python proxy on loopback. Python must see each request as if the client had connected to it directly, because every Python route (inference, MCP, pass-through, UI, WebSocket) is served behind it without Rust knowing which one it is + +- Serve HTTP/1.1 only, as uvicorn does. The supervisor never puts Rust in front of a Python server that speaks HTTP/2 +- Never branch on a route or path. A behavior one endpoint needs is a protocol behavior every endpoint gets +- Forward the method, request target, headers, and body byte for byte within the [URL contract](FORWARDING.md#url-fragments-and-literal-hashes). An unescaped `#` starts a fragment and is omitted with everything after it, while `%23` is preserved. Only drop hop-by-hop headers (RFC 9110 section 7.6.1) and set `x-litellm-rust-peer`, which replaces any client copy +- Stream both bodies. Never buffer, cap, or time out a body, since uploads and provider streams have no upper bound +- A client disconnect closes the upstream connection at every stage, because that is how Python learns to stop a provider stream +- An upstream that dies mid-response fails the client body. Never end it cleanly, or SSE clients treat a truncated stream as complete +- Never invent a Host. Python builds absolute URLs from it, so a missing one must never become the private upstream address +- A request Python could not have received unchanged (HTTP/1.1 without Host, a transfer coding other than chunked, a target without a path) gets the status uvicorn would have given, from Rust +- Pooled upstream connections are retired well before Python's keep-alive timeout, so a request never lands on a connection Python is closing +- Idle client connections close after Python's keep-alive timeout, as uvicorn closed them, so a client that vanishes cannot hold a socket forever. Only the wait for the next request is timed, never an exchange in progress +- WebSocket is the one protocol switch Python serves. Upgrades are relayed as raw bytes after the 101 and drained on shutdown +- Drain on EOF of stdin exactly as on SIGTERM. The supervisor holds the only write end, so EOF means it is gone and nobody is left to stop the gateway. `input_closed` reads on a plain thread, since a read blocked on tokio's blocking pool would hold up the runtime's exit + +## Layout + +Every forwarding decision has one home. `rewrite.rs` owns what Python and the client receive, as pure functions over requests and responses. `Rejection` in `error.rs` is the only place that picks a status for a request Rust answers itself. `settings.rs` parses the environment into typed values, so an `Upstream` is a loopback origin by construction. `server.rs` accepts client connections and owns their timeouts, since `axum::serve` exposes none, and `connections.rs` tracks every connection and relay for shutdown. `forward.rs` and `websocket.rs` only move requests, bodies and upgraded connections and never touch a header themselves. `connector.rs` keeps hyper reading Python's answer after Python stops reading the request + +## Environment + +`Settings` reads only these variables, and only when `LITELLM_FORWARD_UPSTREAM` is set. The Python supervisor in `litellm/proxy/rust_forward/` sets every one of them from the values Python actually runs with, so all of them are required and a missing or invalid one fails startup naming it. The crate never reads Python's own settings (`HOST`, `PORT`, `KEEPALIVE_TIMEOUT`, `GRACEFUL_SHUTDOWN_TIMEOUT`) and keeps no copy of Python's defaults, since Python can get those values from CLI flags the environment never shows + +Only the Python supervisor starts the gateway in forwarding mode, from `litellm` and from `python -m gateway.launch`, on a laptop and in every image. `supervisor/children.py` is the only writer of these variables, so adding, renaming or removing one changes `settings.rs` and `children.rust_spec` in the same change. Without `LITELLM_FORWARD_UPSTREAM` the binary runs as the standalone Rust gateway and this crate does nothing + +| Variable | Meaning | +|---|---| +| `LITELLM_FORWARD_UPSTREAM` | Python's loopback origin, e.g. `http://127.0.0.1:53211` | +| `LITELLM_FORWARD_HOST` | public bind host | +| `LITELLM_FORWARD_PORT` | public bind port | +| `LITELLM_FORWARD_UPSTREAM_KEEPALIVE` | seconds uvicorn keeps an idle connection open. Pooled connections are retired at half of it, and idle client connections are closed after it | +| `LITELLM_FORWARD_SHUTDOWN_DRAIN` | seconds to drain on shutdown, already including Python's graceful timeout and teardown headroom | + +## Tests + +Test through real sockets with `tests/support`, never `oneshot`. `Gateway` runs the forwarder on a listener like production, `recording` is a well-behaved upstream that captures what Python would receive, and `raw_upstream` and `Raw` drive bytes by hand for framing, timing and disconnects a real server would not produce. Waits measured in minutes use paused tokio time instead of real sleeps + +The forwarder sees HTTP/1.1 messages, not routes, so the tests are split into one file per traffic shape uvicorn can emit: `exchange` (plain request and response), `framing` (requests whose framing a parser could read two ways), `slow_head` (minutes before response headers), `stream` (incremental response bodies), `idle_stream` (long-lived streams that go quiet), `upload`, `download`, `websocket`, `client_identity`, `lifecycle` (disconnects, pooling, idle timeouts, shutdown) and `roundtrip` (the forwarding invariant for generated messages). A new route never gets its own test. Add its behavior to the shape it already has, and when it needs a wire behavior none of the files covers, add a new shape file for it diff --git a/litellm-rust/crates/gateway-forward/Cargo.toml b/litellm-rust/crates/gateway-forward/Cargo.toml new file mode 100644 index 00000000000..821ea91d269 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "litellm-gateway-forward" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +axum.workspace = true +envy = "0.4.2" +hyper = { version = "1.12.0", features = ["http1", "server"] } +hyper-util = { workspace = true, features = ["client-legacy", "http1", "tokio"] } +serde.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["io-util", "sync"] } +tokio-util = { version = "0.7", features = ["rt"] } +tower-service = "0.3.3" +tracing.workspace = true + +[dev-dependencies] +bytes.workspace = true +futures-util.workspace = true +http-body-util = "0.1" +httparse = "1" +hyper = { version = "1.12.0", features = ["client"] } +proptest.workspace = true +rstest.workspace = true +tokio = { workspace = true, features = ["io-util", "sync", "test-util"] } +tokio-tungstenite.workspace = true diff --git a/litellm-rust/crates/gateway-forward/FORWARDING.md b/litellm-rust/crates/gateway-forward/FORWARDING.md new file mode 100644 index 00000000000..532f341aba5 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/FORWARDING.md @@ -0,0 +1,31 @@ +# Python forwarding compatibility + +The opt-in forwarding gateway serves HTTP/1.1 and WebSockets in front of Python. Python still owns every route and policy. Its protocol contract and environment settings are in [AGENTS.md](AGENTS.md) + +## URL fragments and literal hashes + +The gateway follows Hyper's `http::Uri` parsing. If a raw client sends an unescaped `#` in the request target, only the path and query before the first `#` reach Python. For example, `/v1/files?purpose=batch#retry&limit=1` becomes `/v1/files?purpose=batch`, and `/gemini/v1beta/files/a#b?key=k` becomes `/gemini/v1beta/files/a` + +This is accepted behavior because `#` starts a URL fragment, which is separated before dereferencing ([RFC 3986 section 3.5](https://www.rfc-editor.org/rfc/rfc3986.html#section-3.5)). HTTP origin-form contains only the path and optional query, not a fragment ([RFC 9112 section 3.2.1](https://www.rfc-editor.org/rfc/rfc9112.html#section-3.2.1)). Browsers and URL-based clients omit the fragment before sending the request, so UI hash routing is unaffected + +A literal hash in path or query data must be encoded as `%23`, which the gateway forwards unchanged. Direct Python with h11 also accepts unescaped hashes as data, but that permissive raw-target behavior is outside this URL contract. Active socket tests cover both [fragment omission](tests/exchange.rs) and [encoded-hash preservation](tests/exchange.rs) + +## Known differences from uvicorn with h11 + +These cases remain executable as ignored socket tests. They are not passing parity checks + +| Input | Direct Python | Through Rust | Regression test | +|---|---|---|---| +| Header after a removed hop-by-hop field | Keeps the remaining header order | `HeaderMap::remove` can move a later header | [dropping_a_hop_by_hop_header_keeps_the_others_in_order](tests/exchange.rs) | +| Repeated header names interleaved with other names | Keeps the interleaving | Groups values by name, keeping their order within that name | [interleaved_repeated_headers_reach_python_interleaved](tests/exchange.rs) | +| Obsolete folded header, such as `x-a: b\r\n c` | Unfolds to `b c` | Rejects with 400 | [a_folded_header_reaches_python_unfolded](tests/exchange.rs) | +| Identical Content-Length list, such as `7, 7` | Accepts length 7 | Rejects with 400 | [a_content_length_list_of_one_value_reaches_python](tests/framing.rs) | +| HTTP/1.0 request with chunked transfer coding | Decodes the body | Rejects with 400 | [an_http10_chunked_upload_reaches_python](tests/framing.rs) | + +Clients can avoid the parser rejections by sending unfolded headers, one Content-Length value, and HTTP/1.1 for chunked uploads. Integrations must not depend on header ordering across different field names + +Run `cargo test -p litellm-gateway-forward --test exchange --test framing -- --ignored` from this directory to reproduce the differences + +## Connection header handling + +The gateway strips standard hop-by-hop fields and fields named by Connection, except routing, representation, credential, forwarding, and WebSocket handshake headers. A client cannot nominate those end-to-end fields for removal. The private `x-litellm-rust-peer` always replaces any client-supplied copy diff --git a/litellm-rust/crates/gateway-forward/src/connections.rs b/litellm-rust/crates/gateway-forward/src/connections.rs new file mode 100644 index 00000000000..10a4867fa51 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/connections.rs @@ -0,0 +1,58 @@ +use tokio_util::{ + sync::{CancellationToken, DropGuard}, + task::TaskTracker, +}; + +/// Every client connection and WebSocket relay the gateway runs, so shutdown can wait for them +/// and then close whatever outlives the drain. +#[derive(Clone, Default)] +pub(crate) struct Connections { + tasks: TaskTracker, + draining: CancellationToken, + closed: CancellationToken, +} + +impl Connections { + /// Runs `work` until it finishes or the gateway closes every connection. + pub fn spawn(&self, work: impl Future + Send + 'static) { + let closed = self.closed.clone(); + self.tasks.spawn(async move { + tokio::select! { + () = closed.cancelled() => {} + () = work => {} + } + }); + } + + pub fn begin_shutdown(&self) { + self.draining.cancel(); + } + + pub async fn draining(&self) { + self.draining.cancelled().await; + } + + pub fn is_draining(&self) -> bool { + self.draining.is_cancelled() + } + + pub async fn until_closed(&self, future: impl Future) -> Option { + tokio::select! { + () = self.closed.cancelled() => None, + output = future => Some(output), + } + } + + pub fn close(&self) { + self.closed.cancel(); + } + + pub fn close_on_drop(&self) -> DropGuard { + self.closed.clone().drop_guard() + } + + pub async fn wait(&self) { + self.tasks.close(); + self.tasks.wait().await; + } +} diff --git a/litellm-rust/crates/gateway-forward/src/connector.rs b/litellm-rust/crates/gateway-forward/src/connector.rs new file mode 100644 index 00000000000..5eb998db0c0 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/connector.rs @@ -0,0 +1,122 @@ +use std::{ + io::{self, IoSlice}, + pin::Pin, + task::{Context, Poll, ready}, +}; + +use axum::http::Uri; +use hyper::rt::{Read, ReadBufCursor, Write}; +use hyper_util::{ + client::legacy::connect::{Connected, Connection, HttpConnector}, + rt::TokioIo, +}; +use tokio::net::TcpStream; + +/// Connects to Python like `HttpConnector`, but keeps hyper reading Python's answer after Python +/// stops reading the request. uvicorn rejects an upload (413, 401) as soon as it reads the head +/// and closes with the body unread, so writing the rest fails, and hyper would fail the request +/// and drop the answer it already received. Like curl and httpcore, the forwarder discards the +/// rest of the request instead and reads on. A connection that never answered still fails, on the +/// read, and no request is ever sent twice. +#[derive(Clone)] +pub(crate) struct Connector(pub HttpConnector); + +impl tower_service::Service for Connector { + type Response = AnswerAfterRefusal; + type Error = >::Error; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, context: &mut Context<'_>) -> Poll> { + self.0.poll_ready(context) + } + + fn call(&mut self, uri: Uri) -> Self::Future { + let connecting = self.0.call(uri); + Box::pin(async move { + Ok(AnswerAfterRefusal { + stream: connecting.await?, + refused: false, + }) + }) + } +} + +pub(crate) struct AnswerAfterRefusal { + stream: TokioIo, + refused: bool, +} + +impl AnswerAfterRefusal { + fn written(&mut self, written: io::Result, length: usize) -> io::Result { + match written { + Err(error) + if matches!( + error.kind(), + io::ErrorKind::BrokenPipe | io::ErrorKind::ConnectionReset + ) => + { + self.refused = true; + Ok(length) + } + written => written, + } + } +} + +impl Read for AnswerAfterRefusal { + fn poll_read( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: ReadBufCursor<'_>, + ) -> Poll> { + Pin::new(&mut self.stream).poll_read(context, buffer) + } +} + +impl Write for AnswerAfterRefusal { + fn poll_write( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &[u8], + ) -> Poll> { + if self.refused { + return Poll::Ready(Ok(buffer.len())); + } + let written = ready!(Pin::new(&mut self.stream).poll_write(context, buffer)); + Poll::Ready(self.written(written, buffer.len())) + } + + fn poll_write_vectored( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffers: &[IoSlice<'_>], + ) -> Poll> { + let length = buffers.iter().map(|buffer| buffer.len()).sum(); + if self.refused { + return Poll::Ready(Ok(length)); + } + let written = ready!(Pin::new(&mut self.stream).poll_write_vectored(context, buffers)); + Poll::Ready(self.written(written, length)) + } + + fn is_write_vectored(&self) -> bool { + self.stream.is_write_vectored() + } + + fn poll_flush(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + if self.refused { + return Poll::Ready(Ok(())); + } + Pin::new(&mut self.stream).poll_flush(context) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.stream).poll_shutdown(context) + } +} + +impl Connection for AnswerAfterRefusal { + fn connected(&self) -> Connected { + self.stream.connected() + } +} diff --git a/litellm-rust/crates/gateway-forward/src/error.rs b/litellm-rust/crates/gateway-forward/src/error.rs new file mode 100644 index 00000000000..7dbdde68a85 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/error.rs @@ -0,0 +1,91 @@ +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, +}; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("invalid forwarding upstream URI")] + UpstreamUri(#[source] axum::http::uri::InvalidUri), + #[error( + "LITELLM_FORWARD_UPSTREAM must be a loopback IPv4 http origin without a path, query or credentials" + )] + Upstream, + #[error("{0} must be set in forward mode")] + MissingSetting(String), + #[error("invalid forwarding setting: {0}")] + InvalidSetting(String), + #[error("forwarding gateway I/O failed: {0}")] + Io(#[from] std::io::Error), +} + +/// Why a request got an answer from Rust instead of from Python. +#[derive(Debug, thiserror::Error)] +pub(crate) enum Rejection { + #[error("HTTP/1.1 request without a Host header")] + MissingHost, + #[error("request target has no path to forward")] + NoPath, + #[error("request uses a transfer coding other than chunked")] + UnsupportedTransferCoding, + #[error("client request body failed: {0}")] + ClientBody(#[source] hyper_util::client::legacy::Error), + #[error("python proxy unreachable: {0}")] + Unreachable(#[source] hyper_util::client::legacy::Error), + #[error("python proxy switched protocols without being asked to")] + UnrequestedUpgrade, + #[error("python proxy websocket upgrade failed: {0}")] + UpgradeFailed(#[source] hyper::Error), + #[error("forwarding gateway is shutting down")] + ShuttingDown, + #[error("could not build the request for the python proxy")] + Internal, +} + +impl Rejection { + /// A client body that ends early or breaks its framing fails the upstream request too, but + /// the fault is the client's, not Python's. + pub fn from_upstream(error: hyper_util::client::legacy::Error) -> Self { + let client_body = std::error::Error::source(&error) + .and_then(|source| source.downcast_ref::()) + .is_some_and(hyper::Error::is_user); + match client_body { + true => Self::ClientBody(error), + false => Self::Unreachable(error), + } + } + + fn status(&self) -> StatusCode { + match self { + Self::MissingHost + | Self::NoPath + | Self::UnsupportedTransferCoding + | Self::ClientBody(_) => StatusCode::BAD_REQUEST, + Self::Unreachable(_) | Self::UnrequestedUpgrade | Self::UpgradeFailed(_) => { + StatusCode::BAD_GATEWAY + } + Self::ShuttingDown => StatusCode::SERVICE_UNAVAILABLE, + Self::Internal => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn is_gateway_fault(&self) -> bool { + matches!( + self, + Self::Unreachable(_) + | Self::UnrequestedUpgrade + | Self::UpgradeFailed(_) + | Self::Internal + ) + } +} + +impl IntoResponse for Rejection { + fn into_response(self) -> Response { + match self.is_gateway_fault() { + true => tracing::warn!(error = %self, "request not forwarded"), + false => tracing::debug!(error = %self, "request not forwarded"), + } + self.status().into_response() + } +} diff --git a/litellm-rust/crates/gateway-forward/src/forward.rs b/litellm-rust/crates/gateway-forward/src/forward.rs new file mode 100644 index 00000000000..69fb63ada7d --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/forward.rs @@ -0,0 +1,169 @@ +use std::{ + pin::Pin, + task::{Context, Poll}, + time::Duration, +}; + +use axum::{ + body::Body, + extract::Request, + http::{StatusCode, Uri}, + response::{IntoResponse, Response}, +}; +use hyper::body::{Frame, Incoming, SizeHint}; +use hyper_util::{ + client::legacy::{Client, connect::HttpConnector}, + rt::{TokioExecutor, TokioTimer}, +}; +use tokio::net::TcpListener; + +use crate::{ + Rejection, Upstream, connections::Connections, connector::Connector, rewrite, server, + server::ClientSocket, websocket, +}; + +// h11 counts no headers and only caps a head at 16 KiB, so it accepts as many headers as fit in +// that many shortest lines (`a:\r\n`). Python copies every provider header into its response +// too, so hyper's default of 100 would turn ordinary traffic into 431s and 502s. +pub(crate) const MAX_HEADERS: usize = 16 * 1024 / b"a:\r\n".len(); + +#[derive(Clone)] +pub struct Forward { + upstream: Uri, + client: Client, + connections: Connections, + keepalive: Duration, +} + +impl Forward { + /// `upstream_keepalive` is how long the upstream keeps an idle connection open. Pooled + /// connections are dropped well before that, so a request never lands on one it is closing, + /// and idle client connections are closed after it, as uvicorn did. + pub fn new(upstream: &Upstream, upstream_keepalive: Duration) -> Self { + let mut connector = HttpConnector::new(); + connector.set_nodelay(true); + let client = Client::builder(TokioExecutor::new()) + .http1_max_headers(MAX_HEADERS) + .pool_timer(TokioTimer::new()) + .pool_idle_timeout(upstream_keepalive / 2) + .set_host(false) + .build(Connector(connector)); + Self { + upstream: upstream.uri().clone(), + client, + connections: Connections::default(), + keepalive: upstream_keepalive, + } + } + + /// Serves until `shutdown` resolves, then lets open requests and WebSockets finish for up to + /// `drain` before closing whatever is left. Python shuts down alongside and ends long streams + /// itself, so `drain` only backstops a Python that never does. + pub async fn serve( + self, + listener: TcpListener, + shutdown: impl Future, + drain: Duration, + ) { + server::serve(self, listener, shutdown, drain).await; + } + + pub(crate) fn connections(&self) -> &Connections { + &self.connections + } + + pub(crate) fn keepalive(&self) -> Duration { + self.keepalive + } + + pub(crate) async fn handle(&self, client: ClientSocket, request: Request) -> Response { + self.forward(client, request) + .await + .unwrap_or_else(IntoResponse::into_response) + } + + async fn forward( + &self, + client: ClientSocket, + mut request: Request, + ) -> Result { + if let Some(downstream) = websocket::take_handshake(&self.connections, &mut request)? { + return self.forward_websocket(client, request, downstream).await; + } + let request = rewrite::upstream_request(request, &self.upstream, client, false)?; + let response = self + .client + .request(request) + .await + .map_err(Rejection::from_upstream)?; + http_response(response) + } + + async fn forward_websocket( + &self, + client: ClientSocket, + request: Request, + downstream: hyper::upgrade::OnUpgrade, + ) -> Result { + let request = rewrite::upstream_request(request, &self.upstream, client, true)?; + let response = self + .connections + .until_closed(self.client.request(request)) + .await + .ok_or(Rejection::ShuttingDown)? + .map_err(Rejection::from_upstream)?; + if response.status() == StatusCode::SWITCHING_PROTOCOLS { + return websocket::switch(&self.connections, downstream, response).await; + } + http_response(response) + } +} + +fn http_response(response: Response) -> Result { + if response.status() == StatusCode::SWITCHING_PROTOCOLS { + return Err(Rejection::UnrequestedUpgrade); + } + Ok( + rewrite::downstream_response(response) + .map(|body| Body::new(Relayed { body, failed: None })), + ) +} + +/// Python's response body on its way to the client. +struct Relayed { + body: Incoming, + failed: Option, +} + +impl hyper::body::Body for Relayed { + type Data = ::Data; + type Error = hyper::Error; + + // hyper aborts a client connection whose response body fails without flushing the head it + // buffered, so a head Python sent right before dying would never arrive and SDKs would retry + // a request Python already ran. Holding the error for one poll lets hyper flush the head. + fn poll_frame( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + if let Some(error) = self.failed.take() { + return Poll::Ready(Some(Err(error))); + } + match Pin::new(&mut self.body).poll_frame(context) { + Poll::Ready(Some(Err(error))) => { + self.failed = Some(error); + context.waker().wake_by_ref(); + Poll::Pending + } + frame => frame, + } + } + + fn is_end_stream(&self) -> bool { + self.failed.is_none() && self.body.is_end_stream() + } + + fn size_hint(&self) -> SizeHint { + self.body.size_hint() + } +} diff --git a/litellm-rust/crates/gateway-forward/src/input.rs b/litellm-rust/crates/gateway-forward/src/input.rs new file mode 100644 index 00000000000..772d4610683 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/input.rs @@ -0,0 +1,29 @@ +use std::io::{self, Read}; + +use tokio::sync::oneshot; + +/// Resolves once `input` reaches end of file or fails to read. The Python supervisor holds the only +/// write end of the gateway's stdin, so EOF means it is gone and the gateway drains as on SIGTERM. +/// +/// The read runs on a plain thread because a blocked read on tokio's blocking pool would hold up +/// runtime shutdown after a signal had already stopped the gateway. +pub fn input_closed(mut input: impl Read + Send + 'static) -> impl Future + Send { + let (closed, on_closed) = oneshot::channel::<()>(); + let reader = std::thread::Builder::new() + .name("stdin-eof".into()) + .spawn(move || { + let _ = io::copy(&mut input, &mut io::sink()); + let _ = closed.send(()); + }); + async move { + match reader { + Ok(_) => { + let _ = on_closed.await; + } + Err(error) => { + tracing::warn!(%error, "could not watch stdin, so only signals stop the gateway"); + std::future::pending::<()>().await; + } + } + } +} diff --git a/litellm-rust/crates/gateway-forward/src/lib.rs b/litellm-rust/crates/gateway-forward/src/lib.rs new file mode 100644 index 00000000000..35f25d19c3c --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/lib.rs @@ -0,0 +1,31 @@ +mod connections; +mod connector; +mod error; +mod forward; +mod input; +mod rewrite; +mod server; +mod settings; +mod websocket; + +pub use error::Error; +pub(crate) use error::Rejection; +pub use forward::Forward; +pub use input::input_closed; +pub use rewrite::PEER_HEADER; +pub use settings::{Settings, Upstream}; + +/// Binds the public listener and forwards to Python until `shutdown` resolves, then drains for +/// `settings.shutdown_drain`. +pub async fn run( + settings: Settings, + shutdown: impl Future + Send + 'static, +) -> Result<(), Error> { + let forward = Forward::new(&settings.upstream, settings.upstream_keepalive); + let listener = tokio::net::TcpListener::bind((settings.host.as_str(), settings.port)).await?; + tracing::info!(address = %listener.local_addr()?, "forwarding gateway listening"); + forward + .serve(listener, shutdown, settings.shutdown_drain) + .await; + Ok(()) +} diff --git a/litellm-rust/crates/gateway-forward/src/rewrite.rs b/litellm-rust/crates/gateway-forward/src/rewrite.rs new file mode 100644 index 00000000000..fa22bf19037 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/rewrite.rs @@ -0,0 +1,161 @@ +use axum::{ + body::HttpBody, + extract::Request, + http::{HeaderMap, HeaderValue, Response, Uri, Version, header}, +}; + +use crate::{Rejection, server::ClientSocket}; + +pub const PEER_HEADER: &str = "x-litellm-rust-peer"; + +/// The request Python receives: the client's method, target, headers and body unchanged, sent to +/// `upstream` over HTTP/1.1 without the client connection's own headers, plus the client's TCP +/// peer in `PEER_HEADER`, which replaces any copy the client sent. +pub(crate) fn upstream_request( + mut request: Request, + upstream: &Uri, + client: ClientSocket, + websocket: bool, +) -> Result { + if !only_chunked(request.headers()) { + return Err(Rejection::UnsupportedTransferCoding); + } + let path_and_query = request + .uri() + .path_and_query() + .cloned() + .ok_or(Rejection::NoPath)?; + if let Some(host) = public_host(&request, client)? { + request.headers_mut().insert(header::HOST, host); + } + let mut uri = upstream.clone().into_parts(); + uri.path_and_query = Some(path_and_query); + *request.uri_mut() = Uri::from_parts(uri).map_err(|_| Rejection::Internal)?; + *request.version_mut() = Version::HTTP_11; + let chunked = request.body().size_hint().exact().is_none(); + let headers = request.headers_mut(); + strip_hop_by_hop(headers); + if chunked { + // hyper sends no body for a GET with an unknown length unless told it is chunked. + headers.insert( + header::TRANSFER_ENCODING, + HeaderValue::from_static("chunked"), + ); + } + if websocket { + restore_websocket_handshake(headers); + } + headers.insert( + PEER_HEADER, + HeaderValue::from_str(&client.peer.to_string()).map_err(|_| Rejection::Internal)?, + ); + Ok(request) +} + +// Python builds absolute URLs from Host and, without a usable one, from the address it listens +// on, which behind Rust is the private loopback port. A client that sent no usable Host gets the +// public address it connected to, which is what Python used before. h11 rejects HTTP/1.1 without +// Host. +fn public_host(request: &Request, client: ClientSocket) -> Result, Rejection> { + let host = request.headers().get(header::HOST); + if host.is_some_and(|host| !host.is_empty()) { + return Ok(None); + } + if host.is_none() && request.version() == Version::HTTP_11 { + return Err(Rejection::MissingHost); + } + HeaderValue::from_str(&client.local.to_string()) + .map(Some) + .map_err(|_| Rejection::Internal) +} + +// hyper only decodes chunked framing, so anything else would reach Python still coded but framed +// as a plain body. uvicorn rejects it with the 400 it sends for every h11 parse error. +fn only_chunked(headers: &HeaderMap) -> bool { + headers + .get_all(header::TRANSFER_ENCODING) + .iter() + .flat_map(|value| value.as_bytes().split(|byte| *byte == b',')) + .all(|coding| coding.trim_ascii().eq_ignore_ascii_case(b"chunked")) +} + +/// The response the client receives: Python's, without the upstream connection's own headers. +pub(crate) fn downstream_response(mut response: Response) -> Response { + strip_hop_by_hop(response.headers_mut()); + response +} + +/// Python's 101 as the client receives it, still naming the protocol both sides switch to. +pub(crate) fn switched_response(mut response: Response) -> Response { + strip_hop_by_hop(response.headers_mut()); + restore_websocket_handshake(response.headers_mut()); + response +} + +pub(crate) fn is_websocket_handshake(headers: &HeaderMap) -> bool { + headers + .get(header::UPGRADE) + .is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"websocket")) + && headers + .get_all(header::CONNECTION) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .any(|token| token.trim().eq_ignore_ascii_case("upgrade")) +} + +const HOP_BY_HOP: [&str; 9] = [ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "proxy-connection", +]; + +// Per-connection headers (RFC 9110 section 7.6.1) describe the client's connection to Rust, not +// Rust's connection to Python, so a proxy must drop them instead of forwarding them. +fn strip_hop_by_hop(headers: &mut HeaderMap) { + let named: Vec<_> = headers + .get_all(header::CONNECTION) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .filter_map(|name| header::HeaderName::from_bytes(name.trim().as_bytes()).ok()) + .filter(|name| !must_forward(name)) + .collect(); + for name in named { + headers.remove(name); + } + for name in HOP_BY_HOP { + headers.remove(name); + } +} + +fn must_forward(name: &header::HeaderName) -> bool { + matches!( + name.as_str(), + "host" + | "authorization" + | "cookie" + | "set-cookie" + | "www-authenticate" + | "authentication-info" + | "api-key" + | "x-api-key" + | "litellm-api-key" + | "forwarded" + | PEER_HEADER + ) || name.as_str().starts_with("content-") + || name.as_str().starts_with("x-forwarded-") + || name.as_str().starts_with("sec-websocket-") +} + +// Stripping drops the handshake's Connection and Upgrade, so each hop gets them back. +fn restore_websocket_handshake(headers: &mut HeaderMap) { + headers.insert(header::CONNECTION, HeaderValue::from_static("upgrade")); + headers.insert(header::UPGRADE, HeaderValue::from_static("websocket")); +} diff --git a/litellm-rust/crates/gateway-forward/src/server.rs b/litellm-rust/crates/gateway-forward/src/server.rs new file mode 100644 index 00000000000..0388642446a --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/server.rs @@ -0,0 +1,116 @@ +use std::{convert::Infallible, io, net::SocketAddr, pin::pin, time::Duration}; + +use axum::{body::Body, extract::Request}; +use hyper::{body::Incoming, server::conn::http1, service::service_fn}; +use hyper_util::rt::{TokioIo, TokioTimer}; +use tokio::net::TcpListener; + +use crate::{Forward, forward::MAX_HEADERS}; + +/// The client's TCP connection to the public listener. +#[derive(Clone, Copy, Debug)] +pub(crate) struct ClientSocket { + pub peer: SocketAddr, + pub local: SocketAddr, +} + +/// Serves each public connection with hyper directly, because `axum::serve` cannot configure +/// the connection timeouts uvicorn applied to the same clients. +pub(crate) async fn serve( + forward: Forward, + listener: TcpListener, + shutdown: impl Future, + drain: Duration, +) { + let connections = forward.connections().clone(); + let _close_on_drop = connections.close_on_drop(); + let keepalive = forward.keepalive(); + let builder = builder(keepalive); + let mut shutdown = pin!(shutdown); + loop { + let (stream, peer) = tokio::select! { + () = &mut shutdown => break, + accepted = listener.accept() => match accepted { + Ok(accepted) => accepted, + Err(error) => { + back_off(error).await; + continue; + } + }, + }; + let local = match stream.local_addr() { + Ok(local) => local, + Err(error) => { + tracing::debug!(%error, "client connection closed before it was served"); + continue; + } + }; + if let Err(error) = stream.set_nodelay(true) { + tracing::warn!(%error, "could not disable Nagle on a client connection"); + } + let socket = ClientSocket { peer, local }; + let forward = forward.clone(); + let service = service_fn(move |request: Request| { + let forward = forward.clone(); + async move { Ok::<_, Infallible>(forward.handle(socket, request.map(Body::new)).await) } + }); + let connection = builder + .serve_connection(TokioIo::new(stream), service) + .with_upgrades(); + let draining = connections.clone(); + connections.spawn(async move { + let mut connection = pin!(connection); + let ended = tokio::select! { + ended = connection.as_mut() => ended, + () = draining.draining() => { + connection.as_mut().graceful_shutdown(); + connection.await + } + }; + if let Err(error) = ended { + tracing::debug!(%error, "client connection ended with an error"); + } + }); + } + drop(listener); + connections.begin_shutdown(); + if tokio::time::timeout(drain, connections.wait()) + .await + .is_err() + { + tracing::warn!( + ?drain, + "closing connections still open after the shutdown drain" + ); + } + connections.close(); + connections.wait().await; +} + +// uvicorn closes a client connection that sits idle for `--keepalive_timeout` between requests. +// Without the same limit, a client that vanishes without closing (NAT expiry, a crashed host) +// would hold its socket for the life of the process until the gateway runs out of descriptors. +// Operators already tune that timeout against their load balancer, so the public side keeps +// Python's value instead of introducing its own. hyper only runs the header timer while a +// connection waits for its next request, so it never cuts a long stream, upload or WebSocket. +fn builder(keepalive: Duration) -> http1::Builder { + let mut builder = http1::Builder::new(); + builder + .max_headers(MAX_HEADERS) + .timer(TokioTimer::new()) + .header_read_timeout(keepalive); + builder +} + +async fn back_off(error: io::Error) { + if matches!( + error.kind(), + io::ErrorKind::ConnectionRefused + | io::ErrorKind::ConnectionAborted + | io::ErrorKind::ConnectionReset + ) { + return; + } + tracing::error!(%error, "could not accept a client connection"); + tokio::time::sleep(Duration::from_secs(1)).await; +} diff --git a/litellm-rust/crates/gateway-forward/src/settings.rs b/litellm-rust/crates/gateway-forward/src/settings.rs new file mode 100644 index 00000000000..de01a96c519 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/settings.rs @@ -0,0 +1,113 @@ +use std::{ffi::OsString, net::SocketAddr, str::FromStr, time::Duration}; + +use axum::http::Uri; +use serde::Deserialize; + +use crate::Error; + +const UPSTREAM: &str = "LITELLM_FORWARD_UPSTREAM"; + +/// The Python proxy's private origin. Only a loopback IPv4 address parses, so nothing can point +/// the forwarder at another host. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Upstream(Uri); + +impl Upstream { + pub(crate) fn uri(&self) -> &Uri { + &self.0 + } +} + +impl FromStr for Upstream { + type Err = Error; + + fn from_str(value: &str) -> Result { + let uri: Uri = value.parse().map_err(Error::UpstreamUri)?; + let address = uri + .authority() + .filter(|_| uri.scheme_str() == Some("http")) + .filter(|authority| !authority.as_str().contains('@')) + .filter(|_| uri.path() == "/" && uri.query().is_none()) + .and_then(|authority| authority.as_str().parse::().ok()); + match address { + Some(SocketAddr::V4(address)) if address.ip().is_loopback() => Ok(Self(uri)), + _ => Err(Error::Upstream), + } + } +} + +#[derive(Deserialize)] +struct Environment { + litellm_forward_upstream: String, + litellm_forward_host: String, + litellm_forward_port: u16, + litellm_forward_upstream_keepalive: f64, + litellm_forward_shutdown_drain: f64, +} + +/// Everything forward mode runs with. The Python supervisor resolves each value from the settings +/// Python actually runs with, so none of them is guessed or defaulted here. +/// Setting `LITELLM_FORWARD_UPSTREAM` requires every other `LITELLM_FORWARD_*` setting below. +#[derive(Debug, Clone, PartialEq)] +pub struct Settings { + pub upstream: Upstream, + pub host: String, + pub port: u16, + pub upstream_keepalive: Duration, + pub shutdown_drain: Duration, +} + +impl Settings { + /// `None` when `LITELLM_FORWARD_UPSTREAM` is unset, which means the gateway is not forwarding. + pub fn from_env() -> Result, Error> { + Self::from_vars(std::env::vars_os()) + } + + pub fn from_vars( + variables: impl IntoIterator, + ) -> Result, Error> { + let variables: Vec<(String, String)> = variables + .into_iter() + .filter_map(|(key, value)| Some((key.into_string().ok()?, value.into_string().ok()?))) + .collect(); + if !variables.iter().any(|(key, _)| key == UPSTREAM) { + return Ok(None); + } + let environment: Environment = envy::from_iter(variables).map_err(|error| match error { + envy::Error::MissingValue(field) => Error::MissingSetting(field.to_uppercase()), + envy::Error::Custom(message) => Error::InvalidSetting(message), + })?; + Ok(Some(Self { + upstream: environment.litellm_forward_upstream.parse()?, + host: environment.litellm_forward_host, + port: environment.litellm_forward_port, + upstream_keepalive: positive_seconds( + "LITELLM_FORWARD_UPSTREAM_KEEPALIVE", + environment.litellm_forward_upstream_keepalive, + )?, + shutdown_drain: seconds( + "LITELLM_FORWARD_SHUTDOWN_DRAIN", + environment.litellm_forward_shutdown_drain, + )?, + })) + } +} + +fn seconds(variable: &str, value: f64) -> Result { + Duration::try_from_secs_f64(value).map_err(|_| { + Error::InvalidSetting(format!( + "{variable} must be a non-negative number of seconds, got {value}" + )) + }) +} + +/// The keep-alive also bounds the wait for a client's first request, so zero would close every +/// client connection before it could send one. +fn positive_seconds(variable: &str, value: f64) -> Result { + match seconds(variable, value)? { + Duration::ZERO => Err(Error::InvalidSetting(format!( + "{variable} must be a positive number of seconds, got {value}" + ))), + duration => Ok(duration), + } +} diff --git a/litellm-rust/crates/gateway-forward/src/websocket.rs b/litellm-rust/crates/gateway-forward/src/websocket.rs new file mode 100644 index 00000000000..84d89c75607 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/src/websocket.rs @@ -0,0 +1,86 @@ +use axum::{ + body::Body, + extract::Request, + http::{Method, Version}, + response::Response, +}; +use hyper::{ + body::Incoming, + upgrade::{OnUpgrade, Upgraded}, +}; +use hyper_util::rt::TokioIo; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +use crate::{Rejection, connections::Connections, rewrite}; + +pub(crate) fn take_handshake( + connections: &Connections, + request: &mut Request, +) -> Result, Rejection> { + if request.method() != Method::GET + || request.version() != Version::HTTP_11 + || !rewrite::is_websocket_handshake(request.headers()) + { + return Ok(None); + } + if connections.is_draining() { + return Err(Rejection::ShuttingDown); + } + Ok(Some(hyper::upgrade::on(request))) +} + +pub(crate) async fn switch( + connections: &Connections, + downstream: OnUpgrade, + mut response: Response, +) -> Result { + if !rewrite::is_websocket_handshake(response.headers()) { + return Err(Rejection::UnrequestedUpgrade); + } + let upstream = connections + .until_closed(hyper::upgrade::on(&mut response)) + .await + .ok_or(Rejection::ShuttingDown)? + .map_err(Rejection::UpgradeFailed)?; + connections.spawn(async move { + match downstream.await { + Ok(downstream) => relay(downstream, upstream).await, + Err(_) => tracing::debug!("websocket upgrade did not complete"), + } + }); + Ok(rewrite::switched_response(response).map(|_| Body::empty())) +} + +// uvicorn never half-closes a WebSocket, so Python's EOF means the socket is gone and the relay +// ends. Only a client that is gone ends it from the other side: the client's EOF just half-closes +// Python's side, and a write Python refuses means it closed with its last frames still in flight +// to the client. +async fn relay(downstream: Upgraded, upstream: Upgraded) { + let (mut client_read, mut client_write) = tokio::io::split(TokioIo::new(downstream)); + let (mut python_read, mut python_write) = tokio::io::split(TokioIo::new(upstream)); + let to_python = async { + let mut buffer = vec![0; 8192]; + loop { + let read = match client_read.read(&mut buffer).await { + Ok(0) => break, + Ok(read) => read, + Err(_) => return, + }; + if python_write.write_all(&buffer[..read]).await.is_err() { + return std::future::pending().await; + } + } + python_write.shutdown().await.ok(); + std::future::pending().await + }; + let to_client = async { + tokio::io::copy(&mut python_read, &mut client_write) + .await + .ok(); + client_write.shutdown().await.ok(); + }; + tokio::select! { + () = to_python => tracing::debug!("websocket client disconnected"), + () = to_client => {} + } +} diff --git a/litellm-rust/crates/gateway-forward/tests/client_identity.rs b/litellm-rust/crates/gateway-forward/tests/client_identity.rs new file mode 100644 index 00000000000..c484c89dc1d --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/client_identity.rs @@ -0,0 +1,223 @@ +mod support; + +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +use axum::{body::Body, http::StatusCode, response::Response}; +use litellm_gateway_forward::{Forward, PEER_HEADER}; +use rstest::rstest; +use support::{DRAIN, Gateway, Raw, UVICORN_KEEPALIVE, raw_upstream, recording, within}; +use tokio::net::{TcpListener, TcpStream}; + +fn ok() -> Response { + Response::new(Body::from("ok")) +} + +#[rstest] +#[case::ordinary("")] +#[case::connection_nominated( + "connection: X-Forwarded-For, X-Forwarded-Proto, Forwarded, X-Litellm-Rust-Peer\r\n" +)] +#[tokio::test] +async fn python_sees_forwarding_headers_untouched_and_the_tcp_peer_rust_saw( + #[case] connection: &str, +) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "GET /sso/callback HTTP/1.1\r\n\ + host: public.example\r\n\ + x-forwarded-for: 192.0.2.1\r\n\ + x-forwarded-for: 192.0.2.2\r\n\ + x-forwarded-proto: https\r\n\ + forwarded: for=192.0.2.1;proto=https\r\n\ + x-litellm-rust-peer: 10.1.2.3:1\r\n{connection}\r\n", + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + + let request = received.recv().await.unwrap(); + let forwarded_for: Vec<_> = request.headers.get_all("x-forwarded-for").iter().collect(); + assert_eq!(forwarded_for, ["192.0.2.1", "192.0.2.2"]); + assert_eq!(request.headers["x-forwarded-proto"], "https"); + assert_eq!(request.headers["forwarded"], "for=192.0.2.1;proto=https"); + let peers: Vec<_> = request.headers.get_all(PEER_HEADER).iter().collect(); + assert_eq!(peers, [raw.local_address().to_string().as_str()]); +} + +/// Python builds redirects and OAuth metadata from Host, and without one from the address it +/// listens on, which behind Rust is its private loopback port. So a request without a usable Host +/// carries the public address the client connected to, except HTTP/1.1, which h11 rejects. +#[rstest] +#[case::http11_without_host("HTTP/1.1", "", false)] +#[case::http10_without_host("HTTP/1.0", "", true)] +#[case::http11_with_an_empty_host("HTTP/1.1", "host:\r\n", true)] +#[tokio::test] +async fn a_request_without_a_usable_host_never_names_the_private_upstream( + #[case] version: &str, + #[case] host: &str, + #[case] forwarded: bool, +) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + + raw.send(format!("GET /sso/key/generate {version}\r\n{host}\r\n")) + .await; + + let status = within(raw.response_head()).await.unwrap().status; + let hosts = received.try_recv().ok().map(|request| { + request + .headers + .get_all("host") + .iter() + .map(|value| value.to_str().unwrap().to_owned()) + .collect::>() + }); + let expected = if forwarded { + (StatusCode::OK, Some(vec![gateway.address.to_string()])) + } else { + (StatusCode::BAD_REQUEST, None) + }; + assert_eq!((status, hosts), expected); +} + +#[rstest] +#[case::uppercase_with_port("PUBLIC.Example:4000")] +#[case::ipv6_literal("[::1]:4000")] +#[case::ipv4_with_port("192.0.2.10:8443")] +#[tokio::test] +async fn python_sees_the_host_header_byte_for_byte(#[case] host: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + + raw.send(format!("GET /ui HTTP/1.1\r\nHost: {host}\r\n\r\n")) + .await; + + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + let hosts: Vec<_> = request.headers.get_all("host").iter().collect(); + assert_eq!(hosts, [host]); +} + +/// Python's IP allowlists and rate limits read the peer exactly as uvicorn would have seen it on +/// the public socket, so a dual-stack listener reports IPv4 clients in their mapped form. +#[rstest] +#[case::ipv6("[::1]:0", "[::1]", "[::1]")] +#[case::ipv4_on_a_dual_stack_listener("[::]:0", "127.0.0.1", "[::ffff:127.0.0.1]")] +#[tokio::test] +async fn ipv6_listeners_report_the_peer_bracketed_with_its_port( + #[case] bind: &str, + #[case] connect: &str, + #[case] peer_ip: &str, +) { + let (upstream, mut received) = recording(ok).await; + let forward = Forward::new( + &format!("http://{upstream}").parse().unwrap(), + UVICORN_KEEPALIVE, + ); + let public = TcpListener::bind(bind).await.unwrap(); + let port = public.local_addr().unwrap().port(); + tokio::spawn(forward.serve(public, std::future::pending(), DRAIN)); + let mut raw = Raw::new( + TcpStream::connect(format!("{connect}:{port}")) + .await + .unwrap(), + ); + + raw.send("GET / HTTP/1.1\r\nhost: public.example\r\n\r\n") + .await; + + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!( + received.recv().await.unwrap().headers[PEER_HEADER], + format!("{peer_ip}:{}", raw.local_address().port()).as_str() + ); +} + +/// Without the peer header Python would take Rust's loopback hop as the client, which allowlists +/// often trust, so no spelling, repetition or Connection token may remove or replace it. +#[rstest] +#[case::named_as_hop_by_hop( + "connection: x-litellm-rust-peer\r\nx-litellm-rust-peer: 127.0.0.1:1\r\n" +)] +#[case::mixed_case_copies( + "X-LiteLLM-Rust-Peer: 127.0.0.1:1\r\nx-litellm-rust-peer: 192.0.2.7:2\r\n" +)] +#[case::comma_joined("x-litellm-rust-peer: 127.0.0.1:1, 192.0.2.7:2\r\n")] +#[tokio::test] +async fn a_client_can_neither_strip_nor_spoof_the_peer(#[case] spoof: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + + raw.send(format!( + "GET /key/info HTTP/1.1\r\nhost: public.example\r\n{spoof}\r\n" + )) + .await; + + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + let peers: Vec<_> = request.headers.get_all(PEER_HEADER).iter().collect(); + assert_eq!(peers, [raw.local_address().to_string().as_str()]); +} + +/// Pooled upstream connections carry requests from many clients in turn, so the peer belongs to +/// each request and never to the connection it travels on. +#[rstest] +#[tokio::test] +async fn each_request_carries_its_own_peer_over_a_shared_upstream_connection() { + let connections = Arc::new(AtomicUsize::new(0)); + let accepted = connections.clone(); + let upstream = raw_upstream(move |mut python| { + accepted.fetch_add(1, Ordering::SeqCst); + async move { + while let Some(head) = python.request_head().await { + let peer = head.header(PEER_HEADER).unwrap().to_vec(); + python + .send( + [ + format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\n\r\n", peer.len()) + .as_bytes(), + &peer, + ] + .concat(), + ) + .await; + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut clients = [ + gateway.connect().await, + gateway.connect().await, + gateway.connect().await, + ]; + + for index in [0, 1, 2, 2, 1, 0] { + let raw = &mut clients[index]; + raw.send("GET /v1/models HTTP/1.1\r\nhost: public.example\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + let peer = raw.body(head.content_length().unwrap()).await; + assert_eq!(peer, raw.local_address().to_string().into_bytes()); + } + assert_eq!(connections.load(Ordering::SeqCst), 1); +} diff --git a/litellm-rust/crates/gateway-forward/tests/download.rs b/litellm-rust/crates/gateway-forward/tests/download.rs new file mode 100644 index 00000000000..9707d2f53b7 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/download.rs @@ -0,0 +1,192 @@ +mod support; + +use std::{ + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use axum::{body::Body, extract::Request, http::StatusCode, response::Response}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, raw_upstream, recording, within}; +use tokio::sync::{Notify, mpsc}; + +#[rstest] +#[tokio::test] +async fn streams_a_body_without_a_content_length_as_python_produces_it() { + let first: &[u8] = &[0xff, 0xfb, 0x90, 0x64, 0x00, 0x0d, 0x0a]; + let rest = vec![0xa5; 64 * 1024]; + let finish = Arc::new(Notify::new()); + let finished = finish.clone(); + let tail = rest.clone(); + let upstream = raw_upstream(move |mut python| { + let finished = finished.clone(); + let tail = tail.clone(); + async move { + python.request_head().await.unwrap(); + python + .send([&b"HTTP/1.1 200 OK\r\ncontent-type: audio/mpeg\r\ntransfer-encoding: chunked\r\n\r\n7\r\n"[..], first, b"\r\n"].concat()) + .await; + finished.notified().await; + python + .send([format!("{:x}\r\n", tail.len()).as_bytes(), &tail, b"\r\n0\r\n\r\n"].concat()) + .await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut response = within( + client().request( + Request::post(gateway.url("/v1/audio/speech")) + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .unwrap(); + + assert!(!response.headers().contains_key("content-length")); + let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), first); + finish.notify_one(); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, rest); +} + +#[rstest] +#[tokio::test] +async fn range_requests_and_partial_content_pass_unchanged() { + let (upstream, mut received) = recording(|| -> Response { + Response::builder() + .status(StatusCode::PARTIAL_CONTENT) + .header("content-range", "bytes 2-5/10") + .header("accept-ranges", "bytes") + .body(Body::from("2345")) + .unwrap() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /v1/videos/video_1/content HTTP/1.1\r\nhost: x\r\nrange: bytes=2-5\r\nif-range: \"v1\"\r\n\r\n") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::PARTIAL_CONTENT); + assert_eq!(head.header("content-range"), Some(&b"bytes 2-5/10"[..])); + assert_eq!(raw.body(head.content_length().unwrap()).await, b"2345"); + let request = received.recv().await.unwrap(); + assert_eq!(request.headers["range"], "bytes=2-5"); + assert_eq!(request.headers["if-range"], "\"v1\""); +} + +#[rstest] +#[tokio::test] +async fn a_large_download_is_paced_by_the_client_and_keeps_its_content_length() { + const LENGTH: usize = 128 * 1024 * 1024; + let written = Arc::new(AtomicUsize::new(0)); + let writing = written.clone(); + let upstream = raw_upstream(move |mut python| { + let writing = writing.clone(); + async move { + python.request_head().await.unwrap(); + python + .send(format!("HTTP/1.1 200 OK\r\ncontent-type: application/octet-stream\r\ncontent-length: {LENGTH}\r\n\r\n")) + .await; + let chunk = [0xa5; 64 * 1024]; + while writing.load(Ordering::SeqCst) < LENGTH { + python.send(chunk).await; + writing.fetch_add(chunk.len(), Ordering::SeqCst); + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /v1/files/file-1/content HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.content_length(), Some(LENGTH)); + assert_eq!(head.header("transfer-encoding"), None); + tokio::time::sleep(Duration::from_secs(1)).await; + assert!( + written.load(Ordering::SeqCst) < LENGTH / 4, + "the forwarder buffered a download the client was not reading" + ); + let mut remaining = LENGTH; + while remaining > 0 { + remaining -= within(raw.body(remaining.min(1 << 20))).await.len(); + } +} + +/// The UI revalidates its static assets, and a 304 must carry exactly the validators Python sent, +/// with no length invented for a body it never has. +#[rstest] +#[tokio::test] +async fn a_conditional_request_gets_python_304_with_its_validators_only() { + let (event, mut events) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python| { + let event = event.clone(); + async move { + while let Some(head) = python.request_head().await { + event + .send(head.header("if-none-match").map(<[u8]>::to_vec)) + .unwrap(); + python + .send("HTTP/1.1 304 Not Modified\r\netag: \"5f1-abc\"\r\ncache-control: no-cache\r\nvary: Accept-Encoding\r\ndate: Tue, 03 Nov 2026 10:00:00 GMT\r\n\r\n") + .await; + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + + for _ in 0..2 { + raw.send("GET /ui/_next/static/chunks/main.js HTTP/1.1\r\nhost: x\r\nif-none-match: \"5f1-abc\"\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::NOT_MODIFIED); + let mut names: Vec<_> = head.headers.iter().map(|(name, _)| name.as_str()).collect(); + names.sort_unstable(); + assert_eq!(names, ["cache-control", "date", "etag", "vary"]); + assert_eq!(head.header("etag"), Some(&b"\"5f1-abc\""[..])); + assert_eq!( + head.header("date"), + Some(&b"Tue, 03 Nov 2026 10:00:00 GMT"[..]) + ); + assert_eq!( + within(events.recv()).await.unwrap().as_deref(), + Some(&b"\"5f1-abc\""[..]) + ); + } +} + +/// A client knows a download was cut short only because fewer bytes than the Content-Length +/// arrived before the connection closed, so the length must survive and the connection must end. +#[rstest] +#[tokio::test] +async fn a_download_python_cuts_short_closes_the_client_connection_before_its_length() { + let upstream = raw_upstream(|mut python| async move { + python.request_head().await.unwrap(); + python + .send([&b"HTTP/1.1 200 OK\r\ncontent-type: application/octet-stream\r\ncontent-length: 1000\r\n\r\n"[..], &[0xa5; 500]].concat()) + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /v1/files/file-1/content HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::OK); + assert_eq!(head.content_length(), Some(1000)); + assert_eq!(within(raw.until_closed()).await, [0xa5; 500]); +} diff --git a/litellm-rust/crates/gateway-forward/tests/exchange.rs b/litellm-rust/crates/gateway-forward/tests/exchange.rs new file mode 100644 index 00000000000..8fed7b18453 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/exchange.rs @@ -0,0 +1,855 @@ +mod support; + +use std::{ + convert::Infallible, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, +}; + +use axum::{ + Router, + body::{Body, Bytes}, + extract::Request, + http::{HeaderValue, StatusCode}, + response::Response, +}; +use litellm_gateway_forward::PEER_HEADER; +use rstest::rstest; +use support::{Gateway, Head, client, raw_upstream, recording, serve, within}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + sync::mpsc, +}; + +fn ok() -> Response { + Response::new(Body::from("ok")) +} + +/// An upstream that reports each bodiless request head exactly as it arrived on the wire. +async fn heads() -> (std::net::SocketAddr, mpsc::UnboundedReceiver) { + let (sender, receiver) = mpsc::unbounded_channel(); + let address = raw_upstream(move |mut python| { + let sender = sender.clone(); + async move { + while let Some(head) = python.request_head().await { + sender.send(head).unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok") + .await; + } + } + }) + .await; + (address, receiver) +} + +#[rstest] +#[tokio::test] +async fn forwards_the_request_as_the_client_sent_it_minus_hop_headers() { + let (upstream, mut received) = recording(|| { + Response::builder() + .status(StatusCode::UNAUTHORIZED) + .header("connection", "x-upstream-hop") + .header("x-upstream-hop", "remove") + .header("x-response", "keep") + .body(Body::from("python error")) + .unwrap() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send( + "PATCH /v1/messages?x=a%2Fb&x=c HTTP/1.1\r\n\ + host: public.example\r\n\ + authorization: Bearer test\r\n\ + connection: x-remove, keep-alive\r\n\ + x-remove: remove\r\n\ + proxy-authorization: remove\r\n\ + te: trailers\r\n\ + content-length: 7\r\n\r\npayload", + ) + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::UNAUTHORIZED); + assert_eq!(head.header("x-response"), Some(&b"keep"[..])); + assert_eq!(head.header("x-upstream-hop"), None); + let body = raw.body(head.content_length().unwrap()).await; + assert_eq!(body, b"python error"); + + let request = received.recv().await.unwrap(); + assert_eq!(request.method, "PATCH"); + assert_eq!(request.uri, "/v1/messages?x=a%2Fb&x=c"); + assert_eq!(request.headers["host"], "public.example"); + assert_eq!(request.headers["authorization"], "Bearer test"); + assert!(!request.headers.contains_key("x-remove")); + assert!(!request.headers.contains_key("proxy-authorization")); + assert!(!request.headers.contains_key("te")); + assert!(!request.headers.contains_key("connection")); + assert_eq!(request.body, "payload"); +} + +#[rstest] +#[case::host("HoSt", "public.example")] +#[case::length("Content-Length", "7")] +#[case::authorization("AUTHORIZATION", "Bearer test")] +#[case::cookie("Cookie", "session=test")] +#[case::api_key("X-Api-Key", "test")] +#[case::azure_key("Api-Key", "test")] +#[case::litellm_key("Litellm-Api-Key", "test")] +#[case::forwarded("Forwarded", "for=192.0.2.1;proto=https")] +#[case::forwarded_for("X-Forwarded-For", "192.0.2.1")] +#[case::forwarded_proto("X-Forwarded-Proto", "https")] +#[case::forwarded_host("X-Forwarded-Host", "public.example")] +#[case::content_type("Content-Type", "application/octet-stream")] +#[case::content_encoding("Content-Encoding", "identity")] +#[tokio::test] +async fn connection_cannot_remove_end_to_end_headers(#[case] name: &str, #[case] value: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + let host = if name.eq_ignore_ascii_case("host") { + "" + } else { + "host: public.example\r\n" + }; + let length = if name.eq_ignore_ascii_case("content-length") { + "" + } else { + "content-length: 7\r\n" + }; + raw.send(format!( + "POST / HTTP/1.1\r\n{host}{length}{name}: {value}\r\nconnection: x-remove\r\nconnection: {name}, keep-alive\r\nx-remove: remove\r\n\r\npayload" + )).await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = within(received.recv()).await.unwrap(); + assert_eq!(request.headers[name], value); + assert_eq!(request.body, "payload"); + assert!(!request.headers.contains_key("connection")); + assert!(!request.headers.contains_key("x-remove")); +} + +#[rstest] +#[tokio::test] +async fn connection_cannot_remove_response_credentials_or_representation_metadata() { + let (upstream, _received) = recording(|| { + Response::builder() + .status(StatusCode::UNAUTHORIZED) + .header("connection", "Content-Length, Set-Cookie, WWW-Authenticate, Authentication-Info, Content-Type, Content-Encoding, x-remove") + .header("content-length", "2") + .header("set-cookie", "session=; Max-Age=0") + .header("www-authenticate", "Bearer") + .header("authentication-info", "nextnonce=test") + .header("content-type", "text/plain") + .header("content-encoding", "identity") + .header("x-remove", "remove") + .body(Body::from("ok")) + .unwrap() + }).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET / HTTP/1.1\r\nhost: x\r\n\r\n").await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::UNAUTHORIZED); + assert_eq!(head.content_length(), Some(2)); + assert_eq!(head.header("set-cookie"), Some(&b"session=; Max-Age=0"[..])); + assert_eq!(head.header("www-authenticate"), Some(&b"Bearer"[..])); + assert_eq!( + head.header("authentication-info"), + Some(&b"nextnonce=test"[..]) + ); + assert_eq!(head.header("content-type"), Some(&b"text/plain"[..])); + assert_eq!(head.header("content-encoding"), Some(&b"identity"[..])); + assert_eq!(head.header("x-remove"), None); + assert_eq!(raw.body(2).await, b"ok"); +} + +#[rstest] +#[case::get("GET")] +#[case::post("POST")] +#[case::put("PUT")] +#[case::delete("DELETE")] +#[case::options("OPTIONS")] +#[case::extension("PURGE")] +#[case::webdav("PROPFIND")] +#[case::lowercase("get")] +#[case::punctuated("M-SEARCH")] +#[tokio::test] +async fn forwards_every_method(#[case] method: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let response = client() + .request( + Request::builder() + .method(method) + .uri(gateway.url("/mcp")) + .body(Body::from("body")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let request = received.recv().await.unwrap(); + assert_eq!(request.method, method); + assert_eq!(request.body, "body"); +} + +#[rstest] +#[case::double_slash("//v1//models")] +#[case::encoded_slash("/v1/files/a%2Fb/content")] +#[case::encoded_hash_in_query("/v1/files?purpose=batch%23retry&limit=1")] +#[case::encoded_hash_in_path("/v1/files/report%231.pdf/content")] +#[case::encoded_hash_before_query("/gemini/v1beta/files/a%23b?key=k")] +#[case::encoded_query("/v1/models?q=%E2%9C%93&empty=&flag")] +#[case::utf8_bytes_escaped("/mcp/%F0%9F%A6%80")] +#[case::lowercase_encoded_slash("/v1/files/a%2fb/content")] +#[case::dot_segments("/openai/./v1/../v1/models")] +#[case::encoded_dot_segments("/openai/%2e%2e/v1/%2E/models")] +#[case::matrix_params("/langfuse/api;v=1/public;x=y/traces")] +#[case::plus_and_encoded_space("/gemini/v1beta/models?q=a+b&r=a%20b&s=%2B")] +#[case::empty_query("/v1/models?")] +#[case::question_marks_in_query("/v1/models??a=b?c")] +#[case::colons_in_segments( + "/v1/projects/p/locations/l/publishers/google/models/m:streamGenerateContent?alt=sse" +)] +#[case::sub_delims("/~u/@:!$&'()*+,=/x")] +#[case::raw_bytes_browsers_leave_unescaped("/a{b}|c^d\\e\"f?g={}|^`")] +#[case::invalid_percent_escapes("/a%zz%00b?%")] +#[tokio::test] +async fn forwards_the_request_target_byte_for_byte(#[case] target: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!("GET {target} HTTP/1.1\r\nhost: x\r\n\r\n")) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().uri, target); +} + +#[rstest] +#[tokio::test] +async fn forwards_a_query_string_as_long_as_uvicorn_accepts() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + let target = format!("/vertex_ai/v1/models?key=%2F{}&end", "a%20b".repeat(3000)); + raw.send(format!("GET {target} HTTP/1.1\r\nhost: x\r\n\r\n")) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().uri, target.as_str()); +} + +#[rstest] +#[case::in_query("/v1/files?purpose=batch#retry&limit=1", "/v1/files?purpose=batch")] +#[case::in_path("/v1/files/report#1.pdf/content", "/v1/files/report")] +#[case::before_query("/gemini/v1beta/files/a#b?key=k", "/gemini/v1beta/files/a")] +#[tokio::test] +async fn forwards_only_the_path_and_query_before_a_fragment( + #[case] target: &str, + #[case] expected: &str, +) { + let (upstream, mut received) = heads().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!("GET {target} HTTP/1.1\r\nhost: x\r\n\r\n")) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(within(received.recv()).await.unwrap().path, expected); +} + +#[rstest] +#[tokio::test] +async fn absolute_form_targets_reach_python_as_origin_form() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send( + "GET http://elsewhere.example/v1/models?x=1 HTTP/1.1\r\nhost: elsewhere.example\r\n\r\n", + ) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + assert_eq!(request.uri, "/v1/models?x=1"); +} + +#[rstest] +#[tokio::test] +async fn relays_a_response_with_more_headers_than_hyper_allows_by_default() { + let upstream = serve(axum::Router::new().fallback(|| async { + let headers = (0..150).map(|index| (format!("llm_provider-x-{index}"), "v")); + let mut response = ok(); + for (name, value) in headers { + response.headers_mut().insert( + axum::http::HeaderName::from_bytes(name.as_bytes()).unwrap(), + HeaderValue::from_static(value), + ); + } + response + })) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + + raw.send("GET /v1/chat/completions HTTP/1.1\r\nhost: public.example\r\n\r\n") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::OK); + assert_eq!(head.header("llm_provider-x-149"), Some(&b"v"[..])); +} + +/// uvicorn's h11 parser has no header count limit, only a 16 KiB cap on the whole head. +#[rstest] +#[tokio::test] +async fn forwards_a_request_with_as_many_headers_as_uvicorn_accepts() { + let (sender, mut received) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |python| { + let sender = sender.clone(); + async move { + let mut stream = python.into_stream(); + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + head.push(stream.read_u8().await.unwrap()); + } + sender.send(head).unwrap(); + stream + .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\n\r\n") + .await + .unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + let headers: String = (0..1000).map(|index| format!("x-{index}: v\r\n")).collect(); + raw.send(format!( + "GET /v1/chat/completions HTTP/1.1\r\nhost: x\r\n{headers}\r\n" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let head = String::from_utf8(within(received.recv()).await.unwrap()).unwrap(); + assert!(head.contains(&headers)); +} + +async fn header_fields_python_receives(fields: &str) -> Vec<(String, Vec)> { + let (upstream, mut received) = heads().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "GET /anthropic/v1/messages HTTP/1.1\r\nhost: x\r\n{fields}\r\n" + )) + .await; + within(raw.response_head()).await.unwrap(); + within(received.recv()) + .await + .unwrap() + .headers + .into_iter() + .filter(|(name, _)| name.starts_with("x-") && name != PEER_HEADER) + .collect() +} + +fn fields(expected: &[(&str, &str)]) -> Vec<(String, Vec)> { + expected + .iter() + .map(|(name, value)| ((*name).to_owned(), value.as_bytes().to_vec())) + .collect() +} + +/// Python reads `scope["headers"]` as the list the client sent, in that order. +#[rstest] +#[ignore = "divergence: HeaderMap::remove can move the headers after a dropped hop-by-hop one"] +#[tokio::test] +async fn dropping_a_hop_by_hop_header_keeps_the_others_in_order() { + let received = header_fields_python_receives( + "x-a: 1\r\nconnection: keep-alive\r\nx-b: 2\r\nx-c: 3\r\nx-d: 4\r\n", + ) + .await; + assert_eq!( + received, + fields(&[("x-a", "1"), ("x-b", "2"), ("x-c", "3"), ("x-d", "4")]) + ); +} + +/// uvicorn speaks only HTTP/1.1, so an HTTP/2 prior-knowledge client never reaches Python through +/// Rust either. hyper closes the connection where h11 answers 400, which no HTTP/2 client reads. +#[rstest] +#[tokio::test] +async fn an_http2_client_never_reaches_python() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n\x00\x00\x00\x04\x00\x00\x00\x00\x00") + .await; + + assert!(within(raw.until_closed()).await.is_empty()); + assert!(received.try_recv().is_err()); +} + +#[rstest] +#[ignore = "divergence: HeaderMap groups every value of a name at its first position"] +#[tokio::test] +async fn interleaved_repeated_headers_reach_python_interleaved() { + let received = header_fields_python_receives("x-a: 1\r\nx-b: 2\r\nx-a: 3\r\n").await; + assert_eq!( + received, + fields(&[("x-a", "1"), ("x-b", "2"), ("x-a", "3")]) + ); +} + +/// h11 trims optional whitespace around a value and keeps everything inside it, including empty +/// values and names using every token character. +#[rstest] +#[tokio::test] +async fn python_sees_header_values_as_h11_would_parse_them() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET / HTTP/1.1\r\nhost: x\r\nx-empty:\r\nx-blank: \r\nx-inner: a \t b \t\r\nX-Token!#$%&'*+.^_`|~: v\r\nx-empty: second\r\n\r\n") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + let values = |name: &str| -> Vec<&[u8]> { + request + .headers + .get_all(name) + .iter() + .map(HeaderValue::as_bytes) + .collect() + }; + assert_eq!(values("x-empty"), [&b""[..], b"second"]); + assert_eq!(values("x-blank"), [&b""[..]]); + assert_eq!(values("x-inner"), [&b"a \t b"[..]]); + assert_eq!(values("x-token!#$%&'*+.^_`|~"), [&b"v"[..]]); +} + +/// RFC 9112 section 5.2 lets a server either reject obs-fold or unfold it, and h11 unfolds. +#[rstest] +#[ignore = "divergence: hyper rejects obs-fold with a 400 where uvicorn unfolds it to one space"] +#[tokio::test] +async fn a_folded_header_reaches_python_unfolded() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET / HTTP/1.1\r\nhost: x\r\nx-a: b\r\n c\r\n\r\n") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().headers["x-a"], "b c"); +} + +#[rstest] +#[tokio::test] +async fn keeps_repeated_response_headers_and_opaque_header_bytes() { + let (upstream, mut received) = recording(|| { + Response::builder() + .header("set-cookie", "a=1; Path=/") + .header("set-cookie", "b=2; HttpOnly") + .header("x-opaque", HeaderValue::from_bytes(b"caf\xe9").unwrap()) + .body(Body::empty()) + .unwrap() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(b"GET / HTTP/1.1\r\nhost: x\r\nx-opaque: caf\xe9\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + let cookies: Vec<_> = head + .headers + .iter() + .filter(|(name, _)| name == "set-cookie") + .map(|(_, value)| value.as_slice()) + .collect(); + assert_eq!(cookies, [&b"a=1; Path=/"[..], b"b=2; HttpOnly"]); + assert_eq!(head.header("x-opaque"), Some(&b"caf\xe9"[..])); + assert_eq!( + received.recv().await.unwrap().headers["x-opaque"].as_bytes(), + b"caf\xe9" + ); +} + +/// An HTTP/1.0 client cannot decode chunked framing, so a streamed answer reaches it delimited by +/// the connection closing, as uvicorn sends it. +#[rstest] +#[tokio::test] +async fn an_http10_client_gets_a_streamed_answer_unframed_and_python_gets_its_body() { + let (upstream, mut received) = recording(|| { + Response::new(Body::from_stream(futures_util::stream::iter( + ["str", "eamed"].map(|piece| Ok::<_, Infallible>(Bytes::from_static(piece.as_bytes()))), + ))) + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /v1/completions HTTP/1.0\r\nhost: x\r\ncontent-length: 7\r\n\r\npayload") + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::OK); + assert_eq!(head.header("transfer-encoding"), None); + assert_eq!(within(raw.until_closed()).await, b"streamed"); + let request = received.recv().await.unwrap(); + assert_eq!(request.method, "POST"); + assert_eq!(request.body, "payload"); +} + +#[rstest] +#[tokio::test] +async fn answers_pipelined_requests_in_order() { + let upstream = serve( + Router::new().fallback(|request: Request| async move { request.uri().path().to_owned() }), + ) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /first HTTP/1.1\r\nhost: x\r\n\r\nHEAD /second HTTP/1.1\r\nhost: x\r\n\r\nGET /third HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + let first = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(first.content_length().unwrap()).await, b"/first"); + let second = within(raw.response_head()).await.unwrap(); + assert_eq!(second.content_length(), Some(7)); + let third = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(third.content_length().unwrap()).await, b"/third"); +} + +#[rstest] +#[case::no_content(StatusCode::NO_CONTENT)] +#[case::not_modified(StatusCode::NOT_MODIFIED)] +#[tokio::test] +async fn bodiless_statuses_keep_the_connection_reusable(#[case] status: StatusCode) { + let upstream = serve(Router::new().fallback(move |request: Request| async move { + match request.uri().path() { + "/next" => (StatusCode::OK, "next"), + _ => (status, ""), + } + })) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /empty HTTP/1.1\r\nhost: x\r\n\r\nGET /next HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + assert_eq!(within(raw.response_head()).await.unwrap().status, status); + let next = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(next.content_length().unwrap()).await, b"next"); +} + +#[rstest] +#[tokio::test] +async fn informational_responses_keep_the_connection_reusable() { + let upstream = raw_upstream(|mut python| async move { + while let Some(head) = python.request_head().await { + match head.path.as_str() { + "/hints" => { + python + .send("HTTP/1.1 103 Early Hints\r\nlink: ; rel=preload\r\n\r\nHTTP/1.1 200 OK\r\ncontent-length: 5\r\n\r\nfinal") + .await + } + _ => { + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 4\r\n\r\nnext") + .await + } + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /hints HTTP/1.1\r\nhost: x\r\n\r\nGET /next HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + let mut hinted = within(raw.response_head()).await.unwrap(); + while hinted.status.is_informational() { + hinted = within(raw.response_head()).await.unwrap(); + } + assert_eq!(hinted.status, StatusCode::OK); + assert_eq!(raw.body(hinted.content_length().unwrap()).await, b"final"); + let next = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(next.content_length().unwrap()).await, b"next"); +} + +#[rstest] +#[case::found( + StatusCode::FOUND, + "https://idp.example/authorize?state=a%2Fb&redirect_uri=https%3A%2F%2Fpublic.example%2Fsso%2Fcallback" +)] +#[case::see_other(StatusCode::SEE_OTHER, "/ui/?login=success")] +#[case::temporary(StatusCode::TEMPORARY_REDIRECT, "./mcp/oauth/authorize?client_id=x")] +#[tokio::test] +async fn redirects_reach_the_client_with_location_and_cookies_unchanged( + #[case] status: StatusCode, + #[case] location: &'static str, +) { + let (upstream, _received) = recording(move || { + Response::builder() + .status(status) + .header("location", location) + .header( + "set-cookie", + "token=a%2Fb; Path=/; HttpOnly; Secure; SameSite=Lax", + ) + .header("set-cookie", "state=; Max-Age=0") + .header( + "set-cookie", + "session=x; Expires=Wed, 04 Nov 2026 10:00:00 GMT; Path=/", + ) + .body(Body::empty()) + .unwrap() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /sso/key/generate HTTP/1.1\r\nhost: public.example\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, status); + assert_eq!(head.header("location"), Some(location.as_bytes())); + let cookies: Vec<_> = head + .headers + .iter() + .filter(|(name, _)| name == "set-cookie") + .map(|(_, value)| value.as_slice()) + .collect(); + assert_eq!( + cookies, + [ + &b"token=a%2Fb; Path=/; HttpOnly; Secure; SameSite=Lax"[..], + b"state=; Max-Age=0", + b"session=x; Expires=Wed, 04 Nov 2026 10:00:00 GMT; Path=/", + ] + ); +} + +#[rstest] +#[tokio::test] +async fn non_websocket_upgrades_reach_python_as_plain_requests() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET / HTTP/1.1\r\nhost: x\r\nconnection: upgrade, http2-settings\r\nupgrade: h2c\r\nhttp2-settings: AAMAAABkAAQAoAAAAAIAAAAA\r\n\r\n") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + assert!(!request.headers.contains_key("upgrade")); + assert!(!request.headers.contains_key("connection")); + assert!(!request.headers.contains_key("http2-settings")); +} + +#[rstest] +#[tokio::test] +async fn forwards_an_asterisk_form_options_request() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("OPTIONS * HTTP/1.1\r\nhost: x\r\n\r\n").await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + assert_eq!(request.method, "OPTIONS"); + assert_eq!(request.uri, "*"); +} + +#[rstest] +#[tokio::test] +async fn an_authority_form_target_is_a_400_python_never_sees() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("CONNECT api.example:443 HTTP/1.1\r\nhost: api.example:443\r\n\r\n") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::BAD_REQUEST + ); + assert!(received.try_recv().is_err()); +} + +#[rstest] +#[tokio::test] +async fn a_client_connection_close_ends_only_the_client_connection() { + let connections = Arc::new(AtomicUsize::new(0)); + let accepted = connections.clone(); + let upstream = raw_upstream(move |mut python| { + accepted.fetch_add(1, Ordering::SeqCst); + async move { + while python.request_head().await.is_some() { + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok") + .await; + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + for _ in 0..2 { + let mut raw = gateway.connect().await; + raw.send("GET / HTTP/1.1\r\nhost: x\r\nconnection: close\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.header("connection"), Some(&b"close"[..])); + assert_eq!(raw.body(2).await, b"ok"); + within(raw.closed()).await; + } + assert_eq!(connections.load(Ordering::SeqCst), 1); +} + +#[rstest] +#[tokio::test] +async fn a_python_connection_close_ends_only_the_upstream_connection() { + let connections = Arc::new(AtomicUsize::new(0)); + let accepted = connections.clone(); + let upstream = raw_upstream(move |mut python| { + accepted.fetch_add(1, Ordering::SeqCst); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 500 Internal Server Error\r\ncontent-length: 5\r\nconnection: close\r\n\r\nerror") + .await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + for _ in 0..2 { + raw.send("GET / HTTP/1.1\r\nhost: x\r\n\r\n").await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(head.header("connection"), None); + assert_eq!(raw.body(5).await, b"error"); + } + assert_eq!(connections.load(Ordering::SeqCst), 2); +} + +/// uvicorn answers HEAD on a streaming route with chunked framing and no body, so the client +/// must not be told the body is empty, and must not wait for one before its next request. +#[rstest] +#[tokio::test] +async fn head_answered_with_chunked_framing_claims_no_length_and_keeps_the_connection() { + let upstream = raw_upstream(|mut python| async move { + while let Some(head) = python.request_head().await { + match head.path.as_str() { + "/next" => { + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 4\r\n\r\nnext") + .await + } + _ => { + python + .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n") + .await + } + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("HEAD /mcp HTTP/1.1\r\nhost: x\r\n\r\nGET /next HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::OK); + assert_eq!(head.content_length(), None); + let next = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(next.content_length().unwrap()).await, b"next"); +} + +const GZIP: &[u8] = &[ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x4b, 0x49, 0x2c, 0x49, 0xb4, 0x52, + 0x30, 0xe4, 0xe2, 0x02, 0x00, 0xce, 0xcd, 0x78, 0x54, 0x09, 0x00, 0x00, 0x00, +]; + +#[rstest] +#[case::fixed_length(false)] +#[case::streamed(true)] +#[tokio::test] +async fn content_codings_cross_unchanged_both_ways(#[case] streamed: bool) { + let (upstream, mut received) = recording(move || { + let body = match streamed { + true => Body::from_stream(futures_util::stream::iter( + GZIP.chunks(5) + .map(|piece| Ok::<_, Infallible>(Bytes::from_static(piece))), + )), + false => Body::from(GZIP), + }; + Response::builder() + .header("content-encoding", "gzip") + .header("vary", "Accept-Encoding") + .body(body) + .unwrap() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send( + [ + format!( + "POST /v1/chat/completions HTTP/1.1\r\nhost: x\r\ncontent-encoding: gzip\r\naccept-encoding: gzip, br;q=0.5\r\ncontent-length: {}\r\n\r\n", + GZIP.len() + ) + .as_bytes(), + GZIP, + ] + .concat(), + ) + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.header("content-encoding"), Some(&b"gzip"[..])); + assert_eq!(head.header("vary"), Some(&b"Accept-Encoding"[..])); + let body = match head.content_length() { + Some(length) => raw.body(length).await, + None => within(raw.chunked_body()).await, + }; + assert_eq!(body, GZIP); + assert_eq!(head.content_length().is_none(), streamed); + let request = received.recv().await.unwrap(); + assert_eq!(request.headers["content-encoding"], "gzip"); + assert_eq!(request.headers["accept-encoding"], "gzip, br;q=0.5"); + assert_eq!(request.body, GZIP); +} + +#[rstest] +#[tokio::test] +async fn python_sees_no_accept_encoding_the_client_did_not_send() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /metrics/ HTTP/1.1\r\nhost: x\r\n\r\n").await; + within(raw.response_head()).await.unwrap(); + assert!( + !received + .recv() + .await + .unwrap() + .headers + .contains_key("accept-encoding") + ); +} diff --git a/litellm-rust/crates/gateway-forward/tests/framing.rs b/litellm-rust/crates/gateway-forward/tests/framing.rs new file mode 100644 index 00000000000..32a2c3f2b8d --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/framing.rs @@ -0,0 +1,259 @@ +mod support; + +use std::{collections::BTreeSet, net::SocketAddr}; + +use axum::http::StatusCode; +use rstest::rstest; +use support::{Gateway, Head, Raw, raw_upstream, within}; +use tokio::sync::mpsc; + +struct Captured { + head: Head, + body: Vec, +} + +/// A keep-alive upstream that decodes framing like h11 and reports every request it reads. +async fn capturing() -> (SocketAddr, mpsc::UnboundedReceiver) { + let (sender, receiver) = mpsc::unbounded_channel(); + let address = raw_upstream(move |mut python: Raw| { + let sender = sender.clone(); + async move { + while let Some(head) = python.request_head().await { + let body = match head.header("transfer-encoding") { + Some(_) => python.chunked_body().await, + None => python.body(content_length(&head)).await, + }; + sender.send(Captured { head, body }).unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok") + .await; + } + } + }) + .await; + (address, receiver) +} + +/// The length h11 reads from every Content-Length field, which may repeat one value. +fn content_length(head: &Head) -> usize { + let lengths: BTreeSet = head + .headers + .iter() + .filter(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .flat_map(|(_, value)| value.split(|byte| *byte == b',')) + .map(|length| { + std::str::from_utf8(length.trim_ascii()) + .unwrap() + .parse() + .unwrap() + }) + .collect(); + assert!( + lengths.len() < 2, + "python got conflicting lengths {lengths:?}" + ); + lengths.first().copied().unwrap_or(0) +} + +/// RFC 9112 section 6.3: Transfer-Encoding overrides Content-Length. Forwarding both would let +/// Python split the stream at a different place than Rust did. Section 6.1 then closes the +/// connection after the response, so a request smuggled behind it never runs. +#[rstest] +#[tokio::test] +async fn a_request_framed_two_ways_reaches_python_framed_by_transfer_encoding_and_closes() { + let (upstream, mut received) = capturing().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /v1/files HTTP/1.1\r\nhost: x\r\ncontent-length: 3\r\ntransfer-encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\nGET /next HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + raw.body(2).await; + within(raw.until_closed()).await; + + let upload = received.recv().await.unwrap(); + assert_eq!(upload.head.header("content-length"), None); + assert_eq!( + upload.head.header("transfer-encoding"), + Some(&b"chunked"[..]) + ); + assert_eq!(upload.body, b"hello"); + assert!( + received.try_recv().is_err(), + "python ran the smuggled request" + ); +} + +#[rstest] +#[case::conflicting_content_lengths("content-length: 4\r\ncontent-length: 40\r\n")] +#[case::content_length_list("content-length: 4, 40\r\n")] +#[case::signed_content_length("content-length: +4\r\n")] +#[case::chunked_not_last("transfer-encoding: chunked, identity\r\ncontent-length: 4\r\n")] +#[case::unknown_coding("transfer-encoding: xchunked\r\ncontent-length: 4\r\n")] +#[case::space_before_colon("transfer-encoding : chunked\r\ncontent-length: 4\r\n")] +#[case::folded_transfer_encoding("transfer-encoding:\r\n chunked\r\ncontent-length: 4\r\n")] +#[case::bare_cr("x-a: b\rx-b: c\r\ncontent-length: 4\r\n")] +#[case::nul_byte("x-a: b\x00c\r\ncontent-length: 4\r\n")] +#[tokio::test] +async fn ambiguous_framing_never_reaches_python(#[case] headers: &str) { + let (upstream, mut received) = capturing().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\n{headers}\r\nbodyGET /smuggled HTTP/1.1\r\nhost: x\r\n\r\n" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::BAD_REQUEST + ); + within(raw.closed()).await; + assert!(received.try_recv().is_err(), "python received a request"); +} + +#[rstest] +#[case::bare_lf("4\nbody\n0\n\n")] +#[case::invalid_size("zz\r\nbody\r\n0\r\n\r\n")] +#[case::size_overflow("10000000000000004\r\nbody\r\n0\r\n\r\n")] +#[tokio::test] +async fn a_malformed_request_body_is_a_400_not_a_502(#[case] chunks: &str) { + let upstream = raw_upstream(|mut python| async move { python.closed().await }).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n{chunks}" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::BAD_REQUEST + ); +} + +/// hyper decodes only the chunked layer, so any other coding would reach Python still coded but +/// framed as a plain body. uvicorn turns every h11 parse error into a 400, ignoring the 501 h11 +/// suggests for codings. +#[rstest] +#[case::listed_before_chunked("transfer-encoding: gzip, chunked\r\n")] +#[case::split_across_fields("transfer-encoding: gzip\r\ntransfer-encoding: chunked\r\n")] +#[tokio::test] +async fn a_transfer_coding_other_than_chunked_is_the_400_uvicorn_sends_python_never_sees( + #[case] headers: &str, +) { + let (upstream, mut received) = capturing().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\n{headers}\r\n4\r\nbody\r\n0\r\n\r\n" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::BAD_REQUEST + ); + assert!( + received.try_recv().is_err(), + "python received a still-coded body" + ); +} + +async fn assert_python_reads_the_body_and_the_next_request(headers: &str) { + let (upstream, mut received) = capturing().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\n{headers}\r\npayloadGET /next HTTP/1.1\r\nhost: x\r\n\r\n" + )) + .await; + for _ in 0..2 { + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + raw.body(2).await; + } + assert_eq!(received.recv().await.unwrap().body, b"payload"); + assert_eq!(received.recv().await.unwrap().head.path, "/next"); +} + +#[rstest] +#[case::repeated_fields("content-length: 7\r\ncontent-length: 7\r\n")] +#[case::leading_zeros("content-length: 007\r\n")] +#[tokio::test] +async fn a_content_length_h11_reads_one_way_reaches_python_framed_the_same(#[case] headers: &str) { + assert_python_reads_the_body_and_the_next_request(headers).await; +} + +/// h11 accepts a Content-Length list that repeats one value, as RFC 9110 section 8.6 allows. +#[rstest] +#[ignore = "divergence: hyper's server parses each Content-Length field as one number and 400s a list"] +#[tokio::test] +async fn a_content_length_list_of_one_value_reaches_python() { + assert_python_reads_the_body_and_the_next_request("content-length: 7, 7\r\n").await; +} + +#[rstest] +#[case::leading_zeros("0004\r\nbody\r\n0\r\n\r\n", "body")] +#[case::uppercase_hex("A\r\nbody\r\nbody\r\n0\r\n\r\n", "body\r\nbody")] +#[case::whitespace_after_size("4 \r\nbody\r\n0\r\n\r\n", "body")] +#[case::extension_and_trailer("4;a=\"b;c\"\r\nbody\r\n0\r\nx-sum: 1\r\n\r\n", "body")] +#[tokio::test] +async fn chunk_framing_h11_accepts_reaches_python_decoded( + #[case] chunks: &str, + #[case] body: &str, +) { + let (upstream, mut received) = capturing().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n{chunks}GET /next HTTP/1.1\r\nhost: x\r\n\r\n" + )) + .await; + for _ in 0..2 { + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + raw.body(2).await; + } + assert_eq!(received.recv().await.unwrap().body, body.as_bytes()); + assert_eq!(received.recv().await.unwrap().head.path, "/next"); +} + +/// h11 reads a head whose lines end in a bare LF, so Python gets the same request through Rust. +#[rstest] +#[tokio::test] +async fn a_head_with_bare_lf_line_endings_reaches_python() { + let (upstream, mut received) = capturing().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /v1/files HTTP/1.1\nhost: x\nx-a: b\ncontent-length: 4\n\nbody") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let upload = received.recv().await.unwrap(); + assert_eq!(upload.head.header("x-a"), Some(&b"b"[..])); + assert_eq!(upload.body, b"body"); +} + +/// h11 decodes a chunked body from an HTTP/1.0 client, and RFC 9112 section 6.1 only asks the +/// server to close the connection afterwards. +#[rstest] +#[ignore = "divergence: hyper rejects Transfer-Encoding on HTTP/1.0 with a 400 that uvicorn never sends"] +#[tokio::test] +async fn an_http10_chunked_upload_reaches_python() { + let (upstream, mut received) = capturing().await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /v1/files HTTP/1.0\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n") + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().body, b"body"); +} diff --git a/litellm-rust/crates/gateway-forward/tests/idle_stream.rs b/litellm-rust/crates/gateway-forward/tests/idle_stream.rs new file mode 100644 index 00000000000..a07c0466481 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/idle_stream.rs @@ -0,0 +1,102 @@ +mod support; + +use std::time::Duration; + +use axum::{ + body::Body, + extract::Request, + http::StatusCode, + response::{IntoResponse, Response}, +}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, raw_upstream, recording, within}; +use tokio::{sync::mpsc, time::Instant}; + +const SSE_HEAD: &str = "HTTP/1.1 200 OK\r\n\ + date: Tue, 03 Nov 2026 00:00:00 GMT\r\n\ + server: uvicorn\r\n\ + cache-control: no-cache, no-transform\r\n\ + connection: keep-alive\r\n\ + content-type: text/event-stream\r\n\ + mcp-session-id: session-1\r\n\ + x-accel-buffering: no\r\n\ + transfer-encoding: chunked\r\n\r\n"; + +const PING: &str = ": ping - 2026-11-03 00:20:00.000000+00:00\r\n\r\n"; + +const MESSAGE: &str = "event: message\r\ndata: {\"jsonrpc\":\"2.0\",\"method\":\"notifications/tools/list_changed\"}\r\n\r\n"; + +fn chunk(data: &str) -> String { + format!("{:x}\r\n{data}\r\n", data.len()) +} + +/// An MCP client holds a GET stream open for server-initiated messages, which may not come for +/// many minutes, so neither hop may close a stream for being idle. +#[rstest] +#[tokio::test(start_paused = true)] +async fn an_idle_mcp_stream_stays_open_in_both_directions() { + let idle = Duration::from_secs(20 * 60); + let (session, mut sessions) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python| { + let session = session.clone(); + async move { + let head = python.request_head().await.unwrap(); + session + .send(head.header("mcp-session-id").map(<[u8]>::to_vec)) + .unwrap(); + python.send(SSE_HEAD).await; + tokio::time::sleep(idle).await; + python.send(chunk(PING)).await; + python.send(chunk(MESSAGE)).await; + python.send("0\r\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let started = Instant::now(); + + let response = client() + .request( + Request::get(gateway.url("/github/mcp")) + .header("accept", "text/event-stream") + .header("mcp-session-id", "session-1") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.headers()["mcp-session-id"], "session-1"); + assert_eq!(response.headers()["x-accel-buffering"], "no"); + assert!(!response.headers().contains_key("connection")); + assert_eq!( + sessions.recv().await.unwrap().as_deref(), + Some(&b"session-1"[..]) + ); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(body, [PING, MESSAGE].concat()); + assert!(started.elapsed() >= idle); +} + +#[rstest] +#[tokio::test] +async fn ends_a_session_with_a_bodiless_delete() { + let (upstream, mut received) = recording(|| -> Response { + ([("mcp-session-id", "session-1")], StatusCode::NO_CONTENT).into_response() + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("DELETE /mcp HTTP/1.1\r\nhost: x\r\nmcp-session-id: session-1\r\n\r\n") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::NO_CONTENT); + assert_eq!(head.header("mcp-session-id"), Some(&b"session-1"[..])); + let request = received.recv().await.unwrap(); + assert_eq!(request.method, "DELETE"); + assert_eq!(request.headers["mcp-session-id"], "session-1"); + assert!(!request.headers.contains_key("transfer-encoding")); + assert!(request.body.is_empty()); +} diff --git a/litellm-rust/crates/gateway-forward/tests/lifecycle.rs b/litellm-rust/crates/gateway-forward/tests/lifecycle.rs new file mode 100644 index 00000000000..97907738ad6 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/lifecycle.rs @@ -0,0 +1,524 @@ +mod support; + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use axum::{Router, body::Body, extract::Request, http::StatusCode, response::Response}; +use http_body_util::BodyExt; +use litellm_gateway_forward::input_closed; +use rstest::rstest; +use support::{ + DRAIN, Gateway, Raw, UVICORN_KEEPALIVE, client, raw_upstream, recording, serve, serve_until, + unreachable_upstream, within, +}; +use tokio::{ + net::{TcpListener, TcpStream}, + sync::{Notify, mpsc}, +}; + +fn ok() -> Response { + Response::new(Body::from("ok")) +} + +/// Python stops a provider stream when it sees the client go away, so a client disconnect must +/// close Rust's connection to Python at every stage of the exchange. +#[rstest] +#[case::waiting_for_headers( + "POST /v1/chat/completions HTTP/1.1\r\nhost: x\r\ncontent-length: 0\r\n\r\n", + false +)] +#[case::mid_stream( + "POST /v1/chat/completions HTTP/1.1\r\nhost: x\r\ncontent-length: 0\r\n\r\n", + true +)] +#[case::mid_upload( + "POST /v1/files HTTP/1.1\r\nhost: x\r\ncontent-length: 1000000\r\n\r\npartial", + false +)] +#[case::waiting_for_websocket_handshake( + "GET /v1/realtime HTTP/1.1\r\nhost: x\r\nconnection: Upgrade\r\nupgrade: websocket\r\nsec-websocket-version: 13\r\nsec-websocket-key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n", + false +)] +#[tokio::test] +async fn client_disconnect_closes_the_upstream_connection( + #[case] request: &'static str, + #[case] headers_sent: bool, +) { + let (event, mut events) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let event = event.clone(); + async move { + python.request_head().await.unwrap(); + if headers_sent { + python + .send("HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n6\r\ndata: \r\n") + .await; + } + event.send("received").unwrap(); + python.closed().await; + event.send("closed").unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(request).await; + assert_eq!(within(events.recv()).await, Some("received")); + if headers_sent { + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + } + drop(raw); + assert_eq!(within(events.recv()).await, Some("closed")); +} + +/// Python treats EOF on its socket as the client leaving, so Rust must not half-close the +/// upstream connection once it has sent the request body. +#[rstest] +#[tokio::test] +async fn python_sees_no_eof_while_its_response_is_pending() { + let (verdict, mut verdicts) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let verdict = verdict.clone(); + async move { + let head = python.request_head().await.unwrap(); + python.body(head.content_length().unwrap()).await; + let eof = tokio::time::timeout(Duration::from_millis(200), python.closed()) + .await + .is_ok(); + verdict.send(eof).unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok") + .await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + + let response = within( + client().request( + Request::post(gateway.url("/mcp")) + .body(Body::from("{}")) + .unwrap(), + ), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(verdicts.recv().await, Some(false)); +} + +/// uvicorn closes a connection that sat idle past its keep-alive timeout. A request Rust sends on +/// that connection as uvicorn closes it is lost, so Rust must retire idle connections first. +#[rstest] +#[case::reused_while_fresh(Duration::from_millis(0))] +#[case::past_the_pool_timeout(Duration::from_millis(250))] +#[case::past_python_keepalive(Duration::from_millis(450))] +#[tokio::test] +async fn never_reuses_a_connection_python_is_closing_for_idleness(#[case] idle: Duration) { + let keepalive = Duration::from_millis(400); + let upstream = raw_upstream(move |mut python: Raw| async move { + let mut idle_since = None::; + while python.request_head().await.is_some() { + if idle_since.is_some_and(|since| since.elapsed() >= keepalive) { + return; + } + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok") + .await; + idle_since = Some(Instant::now()); + } + }) + .await; + let gateway = Gateway::with_keepalive(upstream, keepalive).await; + let client = client(); + let first = within(client.request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + first.into_body().collect().await.unwrap(); + tokio::time::sleep(idle).await; + let second = + within(client.request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + assert_eq!(second.status(), StatusCode::OK); +} + +#[rstest] +#[case::health("/health/liveliness")] +#[case::messages("/v1/messages")] +#[tokio::test] +async fn unreachable_python_is_a_502(#[case] path: &str) { + let gateway = Gateway::start(unreachable_upstream()).await; + let response = + within(client().request(Request::get(gateway.url(path)).body(Body::empty()).unwrap())) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); +} + +#[rstest] +#[case::bad_status_line(b"HTTP/1.1 OK\r\ncontent-length: 0\r\n\r\n".to_vec())] +#[case::header_without_colon(b"HTTP/1.1 200 OK\r\nbroken header\r\ncontent-length: 0\r\n\r\n".to_vec())] +#[case::not_http(b"SSH-2.0-OpenSSH_9.6\r\n".to_vec())] +#[case::conflicting_lengths( + b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\ncontent-length: 3\r\n\r\nok".to_vec() +)] +#[case::oversized_head( + [&b"HTTP/1.1 200 OK\r\nx-big: "[..], &vec![b'a'; 1024 * 1024], b"\r\n\r\n"].concat() +)] +#[case::closed_without_reply(Vec::new())] +#[tokio::test] +async fn an_upstream_that_breaks_http_is_a_502_and_the_client_connection_survives( + #[case] reply: Vec, +) { + let upstream = raw_upstream(move |mut python: Raw| { + let reply = reply.clone(); + async move { + python.request_head().await.unwrap(); + if reply.is_empty() { + return; + } + python.send(reply).await; + std::future::pending::<()>().await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + + for _ in 0..2 { + raw.send("GET / HTTP/1.1\r\nhost: x\r\n\r\n").await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::BAD_GATEWAY); + assert_eq!(head.content_length(), Some(0)); + } +} + +#[rstest] +#[tokio::test] +async fn python_restarting_is_a_502_until_it_listens_again() { + let address = unreachable_upstream(); + let gateway = Gateway::start(address).await; + let get = || client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap()); + assert_eq!( + within(get()).await.unwrap().status(), + StatusCode::BAD_GATEWAY + ); + + let listener = TcpListener::bind(address).await.unwrap(); + let router = Router::new().fallback(|| async { "up" }); + tokio::spawn(async move { axum::serve(listener, router).await }); + + assert_eq!(within(get()).await.unwrap().status(), StatusCode::OK); +} + +/// A request Python already read may have run, so it is answered with a 502 and never replayed. +#[rstest] +#[tokio::test] +async fn a_request_python_read_before_closing_is_a_502_and_never_replayed() { + let (event, mut seen) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let event = event.clone(); + async move { + while let Some(head) = python.request_head().await { + python.body(head.content_length().unwrap()).await; + event.send(head.path.clone()).unwrap(); + if head.path == "/v1/chat/completions" { + return; + } + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 2\r\n\r\nok") + .await; + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /warm HTTP/1.1\r\nhost: x\r\ncontent-length: 0\r\n\r\n") + .await; + let warm = within(raw.response_head()).await.unwrap(); + raw.body(warm.content_length().unwrap()).await; + + raw.send("POST /v1/chat/completions HTTP/1.1\r\nhost: x\r\ncontent-length: 2\r\n\r\n{}") + .await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::BAD_GATEWAY); + let paths: Vec<_> = std::iter::from_fn(|| seen.try_recv().ok()).collect(); + assert_eq!(paths, ["/warm", "/v1/chat/completions"]); +} + +#[rstest] +#[tokio::test] +async fn shutdown_lets_an_in_flight_stream_finish() { + let release = Arc::new(Notify::new()); + let released = release.clone(); + let upstream = raw_upstream(move |mut python: Raw| { + let released = released.clone(); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n5\r\nfirst\r\n") + .await; + released.notified().await; + python.send("4\r\nlast\r\n0\r\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = + within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + let server = gateway.shutdown(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!server.is_finished()); + release.notify_one(); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, "firstlast"); + within(server).await.unwrap(); +} + +#[rstest] +#[tokio::test] +async fn the_supervisor_closing_stdin_drains_like_a_stop_signal() { + let release = Arc::new(Notify::new()); + let released = release.clone(); + let upstream = raw_upstream(move |mut python: Raw| { + let released = released.clone(); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n5\r\nfirst\r\n") + .await; + released.notified().await; + python.send("4\r\nlast\r\n0\r\n\r\n").await; + } + }) + .await; + let (stdin, supervisor) = std::io::pipe().unwrap(); + let (address, server) = serve_until(upstream, UVICORN_KEEPALIVE, input_closed(stdin)).await; + let response = within( + client().request( + Request::get(format!("http://{address}/")) + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(TcpStream::connect(address).await.is_ok()); + + drop(supervisor); + + within(async { + while TcpStream::connect(address).await.is_ok() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; + assert!(!server.is_finished()); + release.notify_one(); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, "firstlast"); + within(server).await.unwrap(); +} + +#[rstest] +#[tokio::test] +async fn shutdown_stops_waiting_for_a_stream_still_open_after_the_drain() { + let upstream = raw_upstream(|mut python: Raw| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n5\r\nfirst\r\n") + .await; + std::future::pending::<()>().await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = + within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + let started = std::time::Instant::now(); + + within(gateway.shutdown()).await.unwrap(); + + assert!(started.elapsed() >= DRAIN); + drop(response); +} + +#[rstest] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_keep_alive_connections_each_get_their_own_responses() { + let upstream = serve(Router::new().fallback(|request: Request| async move { + let index: usize = request.uri().query().unwrap().parse().unwrap(); + Response::builder() + .header("etag", format!("\"{index}\"")) + .body(Body::from(index.to_string().repeat(1 + index * 1000))) + .unwrap() + })) + .await; + let gateway = Gateway::start(upstream).await; + let connections = 6; + + let tasks: Vec<_> = (0..connections) + .map(|first| { + let address = gateway.address; + tokio::spawn(async move { + let mut raw = Raw::new(TcpStream::connect(address).await.unwrap()); + for index in (first..50).step_by(connections) { + raw.send(format!( + "GET /ui/chunk.js?{index} HTTP/1.1\r\nhost: x\r\n\r\n" + )) + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.header("etag"), Some(format!("\"{index}\"").as_bytes())); + let body = raw.body(head.content_length().unwrap()).await; + assert_eq!( + body, + index.to_string().repeat(1 + index * 1000).into_bytes() + ); + } + }) + }) + .collect(); + + for task in tasks { + within(task).await.unwrap(); + } +} + +// Python's `--keepalive_timeout` governed idle client connections before Rust took the public +// port, and operators tune it against their load balancer's idle timeout, so it still does. +// Without it, a client that vanishes without closing its connection holds a socket forever and +// the gateway eventually runs out of file descriptors. +const IDLE: Duration = Duration::from_millis(400); + +#[rstest] +#[tokio::test] +async fn closes_a_client_connection_idle_past_python_keepalive() { + let (upstream, _received) = recording(ok).await; + let gateway = Gateway::with_keepalive(upstream, IDLE).await; + let mut raw = gateway.connect().await; + + for _ in 0..2 { + tokio::time::sleep(IDLE / 2).await; + raw.send("GET /health/liveliness HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::OK); + raw.body(head.content_length().unwrap()).await; + } + let idle_since = Instant::now(); + within(raw.closed()).await; + + assert!(idle_since.elapsed() >= IDLE * 3 / 4); +} + +/// uvicorn would hold a connection that never sends a request, which lets a client park sockets +/// for free, so the same limit applies before the first request too. +#[rstest] +#[tokio::test] +async fn closes_a_client_connection_that_never_sends_a_request() { + let (upstream, _received) = recording(ok).await; + let gateway = Gateway::with_keepalive(upstream, IDLE).await; + let mut raw = gateway.connect().await; + + within(raw.closed()).await; +} + +/// The limit only covers the wait for the next request. A response that goes quiet for longer +/// belongs to Python, which decides how long it may take. +#[rstest] +#[tokio::test] +async fn keeps_a_connection_open_while_a_response_goes_quiet() { + let upstream = raw_upstream(|mut python: Raw| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n") + .await; + tokio::time::sleep(IDLE * 3).await; + python.send("4\r\ndone\r\n0\r\n\r\n").await; + }) + .await; + let gateway = Gateway::with_keepalive(upstream, IDLE).await; + let mut raw = gateway.connect().await; + + raw.send("GET /v1/chat/completions HTTP/1.1\r\nhost: x\r\n\r\n") + .await; + + within(raw.response_head()).await.unwrap(); + assert_eq!(within(raw.chunked_body()).await, b"done"); +} + +/// Likewise for an upload that pauses longer than the limit before it finishes. +#[rstest] +#[tokio::test] +async fn keeps_a_connection_open_while_an_upload_goes_quiet() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::with_keepalive(upstream, IDLE).await; + let mut raw = gateway.connect().await; + + raw.send("POST /v1/files HTTP/1.1\r\nhost: x\r\ncontent-length: 8\r\n\r\nhalf") + .await; + tokio::time::sleep(IDLE * 3).await; + raw.send("done").await; + + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().body, "halfdone"); +} + +/// uvicorn answers a request it rejects (413, 401) as soon as it reads the head and then closes, +/// leaving the upload unread. The client must get that answer, not a 502 or a reset. +#[rstest] +#[case::sized("content-length: 16777216\r\n", vec![b'x'; 256 * 1024])] +#[case::chunked( + "transfer-encoding: chunked\r\n", + [&b"40000\r\n"[..], &[b'x'; 256 * 1024], b"\r\n"].concat() +)] +#[tokio::test] +async fn python_rejecting_an_upload_unread_and_closing_still_answers_the_client( + #[case] framing: &'static str, + #[case] partial_upload: Vec, +) { + let upstream = raw_upstream(|mut python: Raw| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 413 Payload Too Large\r\ncontent-type: application/json\r\ncontent-length: 26\r\nconnection: close\r\n\r\n{\"error\":\"file too large\"}") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\n{framing}\r\n" + )) + .await; + raw.send(partial_upload).await; + + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!( + raw.body(head.content_length().unwrap()).await, + b"{\"error\":\"file too large\"}" + ); +} diff --git a/litellm-rust/crates/gateway-forward/tests/roundtrip.rs b/litellm-rust/crates/gateway-forward/tests/roundtrip.rs new file mode 100644 index 00000000000..c220c7fe8e9 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/roundtrip.rs @@ -0,0 +1,360 @@ +mod support; + +use std::{ + convert::Infallible, + sync::{Arc, Mutex}, +}; + +use axum::{ + body::{Body, Bytes}, + http::{HeaderMap, HeaderName, HeaderValue, Method, Request, StatusCode}, + response::Response, +}; +use http_body_util::BodyExt; +use litellm_gateway_forward::PEER_HEADER; +use proptest::{prelude::*, test_runner::TestRunner}; +use support::{Gateway, client, recording, within}; + +#[derive(Clone, Debug)] +struct Message { + headers: Vec<(String, Vec>)>, + body: Vec, + chunked: bool, +} + +#[derive(Clone, Debug)] +struct Exchange { + method: Method, + target: String, + request: Message, + status: StatusCode, + response: Message, +} + +/// Any value h11 reads back unchanged: visible ASCII, obs-text and interior whitespace. +fn header_value() -> impl Strategy> { + prop::collection::vec( + prop_oneof![8 => 0x21u8..=0x7e, 2 => 0x80u8..=0xff, 1 => Just(b' '), 1 => Just(b'\t')], + 0..24, + ) + .prop_map(|value| value.trim_ascii().to_vec()) +} + +fn method() -> impl Strategy { + prop_oneof![ + Just(Method::GET), + Just(Method::POST), + Just(Method::PUT), + Just(Method::PATCH), + Just(Method::DELETE), + Just(Method::OPTIONS), + "[A-Za-z0-9!#$%&'*+.^_`|~-]{1,16}" + .prop_filter("HEAD and CONNECT frame differently", |method| { + !["HEAD", "CONNECT"].contains(&method.as_str()) + }) + .prop_map(|method| Method::from_bytes(method.as_bytes()).unwrap()), + ] +} + +fn message() -> impl Strategy { + ( + prop::collection::btree_map( + "x-[a-z0-9!#$%&'*+.^_`|~-]{1,12}", + prop::collection::vec(header_value(), 1..4), + 0..12, + ), + prop::collection::vec(any::(), 0..32 * 1024), + any::(), + ) + .prop_map(|(headers, body, chunked)| Message { + headers: headers.into_iter().collect(), + body, + chunked, + }) +} + +fn target() -> impl Strategy { + let segment = prop_oneof![ + "[A-Za-z0-9._~!$&'()*+,;=:@-]{0,8}", + "%[0-9A-Fa-f]{2}", + Just(".".to_owned()), + Just("..".to_owned()), + Just(String::new()), + ]; + ( + prop::collection::vec(segment, 0..6), + prop::option::of("[A-Za-z0-9=&._~%/?:@!$'()*+,;-]{0,40}"), + ) + .prop_map(|(segments, query)| { + let path = format!("/{}", segments.join("/")); + match query { + Some(query) => format!("{path}?{query}"), + None => path, + } + }) +} + +fn exchange() -> impl Strategy { + let status = (200u16..600) + .prop_filter("bodiless status", |code| ![204, 205, 304].contains(code)) + .prop_map(|code| StatusCode::from_u16(code).unwrap()); + (method(), target(), message(), status, message()).prop_map( + |(method, target, request, status, response)| Exchange { + method, + target, + request, + status, + response, + }, + ) +} + +fn body(message: &Message) -> Body { + if !message.chunked { + return Body::from(message.body.clone()); + } + let chunks: Vec> = message + .body + .chunks(1000) + .map(|chunk| Ok(Bytes::copy_from_slice(chunk))) + .collect(); + Body::from_stream(futures_util::stream::iter(chunks)) +} + +fn apply(headers: &mut HeaderMap, message: &Message) { + if message.chunked { + headers.insert("transfer-encoding", HeaderValue::from_static("chunked")); + } + for (name, values) in &message.headers { + for value in values { + headers.append( + HeaderName::from_bytes(name.as_bytes()).unwrap(), + HeaderValue::from_bytes(value).unwrap(), + ); + } + } +} + +fn assert_carried(headers: &HeaderMap, message: &Message, framing_and_hop: &[&str]) { + for (name, values) in &message.headers { + let carried: Vec<&[u8]> = headers + .get_all(name) + .iter() + .map(HeaderValue::as_bytes) + .collect(); + let sent: Vec<&[u8]> = values.iter().map(Vec::as_slice).collect(); + assert_eq!(carried, sent, "header {name}"); + } + for name in headers.keys() { + assert!( + framing_and_hop.contains(&name.as_str()) + || message + .headers + .iter() + .any(|(sent, _)| sent == name.as_str()), + "unexpected header {name}" + ); + } +} + +/// Any request a client can send and any response Python can give cross the forwarder unchanged, +/// apart from the framing and peer headers the forwarder owns. +#[test] +fn every_exchange_crosses_the_forwarder_unchanged() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let current = Arc::new(Mutex::new(None::)); + let responding = current.clone(); + let (gateway, received) = runtime.block_on(async { + let (upstream, received) = recording(move || { + let exchange = responding.lock().unwrap().clone().unwrap(); + let mut response = Response::new(body(&exchange.response)); + *response.status_mut() = exchange.status; + apply(response.headers_mut(), &exchange.response); + response + }) + .await; + ( + Gateway::start(upstream).await, + tokio::sync::Mutex::new(received), + ) + }); + let client = client(); + TestRunner::new(ProptestConfig::with_cases(256)) + .run(&exchange(), |exchange| { + *current.lock().unwrap() = Some(exchange.clone()); + runtime.block_on(async { + while received.lock().await.try_recv().is_ok() {} + let mut request = Request::builder() + .method(exchange.method.clone()) + .uri(gateway.url(&exchange.target)) + .body(body(&exchange.request)) + .unwrap(); + apply(request.headers_mut(), &exchange.request); + let response = support::within(client.request(request)).await.unwrap(); + + let python = received.lock().await.recv().await.unwrap(); + assert_eq!(python.method, exchange.method); + assert_eq!(python.uri, exchange.target.as_str()); + assert_eq!(python.body, exchange.request.body); + assert_carried( + &python.headers, + &exchange.request, + &["host", "content-length", "transfer-encoding", PEER_HEADER], + ); + assert_eq!(python.headers.get_all(PEER_HEADER).iter().count(), 1); + + assert_eq!(response.status(), exchange.status); + let headers = response.headers().clone(); + assert_carried( + &headers, + &exchange.response, + &["date", "content-length", "transfer-encoding"], + ); + let body = support::within(response.into_body().collect()) + .await + .unwrap(); + assert_eq!(body.to_bytes(), exchange.response.body); + }); + Ok(()) + }) + .unwrap(); +} + +#[derive(Clone, Debug)] +enum Framing { + Unframed, + Length, + Chunked(usize), +} + +#[derive(Clone, Debug)] +struct WireRequest { + method: Method, + target: String, + headers: Vec<(String, Vec)>, + body: Vec, + framing: Framing, +} + +fn wire_request() -> impl Strategy { + let framing = prop_oneof![ + Just(Framing::Unframed), + Just(Framing::Length), + (1usize..2048).prop_map(Framing::Chunked), + ]; + ( + method(), + target(), + prop::collection::vec( + ("[xX]-[A-Za-z0-9!#$%&'*+.^_`|~-]{1,12}", header_value()), + 0..24, + ), + prop::collection::vec(any::(), 0..8 * 1024), + framing, + ) + .prop_map(|(method, target, headers, body, framing)| WireRequest { + method, + target, + headers, + body: match framing { + Framing::Unframed => Vec::new(), + _ => body, + }, + framing, + }) +} + +fn encode(request: &WireRequest) -> Vec { + let framing = match request.framing { + Framing::Unframed => String::new(), + Framing::Length => format!("Content-Length: {}\r\n", request.body.len()), + Framing::Chunked(_) => "Transfer-Encoding: chunked\r\n".to_owned(), + }; + let fields = request + .headers + .iter() + .flat_map(|(name, value)| [name.as_bytes(), b": ", value, b"\r\n"].concat()); + let body = match request.framing { + Framing::Chunked(size) => request + .body + .chunks(size) + .flat_map(|chunk| { + [format!("{:x}\r\n", chunk.len()).as_bytes(), chunk, b"\r\n"].concat() + }) + .chain(*b"0\r\n\r\n") + .collect(), + _ => request.body.clone(), + }; + [ + format!( + "{} {} HTTP/1.1\r\nHost: x\r\n", + request.method, request.target + ) + .into_bytes(), + fields.collect(), + framing.into_bytes(), + b"\r\n".to_vec(), + body, + ] + .concat() +} + +fn by_name<'a>(fields: impl Iterator) -> Vec<(String, &'a [u8])> { + let mut fields: Vec<_> = fields + .map(|(name, value)| (name.to_ascii_lowercase(), value)) + .collect(); + fields.sort_by(|(left, _), (right, _)| left.cmp(right)); + fields +} + +/// Pipelined requests in any framing, written in arbitrary pieces, reach Python one by one with +/// the bytes h11 would have read from the client directly. +#[test] +fn pipelined_wire_requests_reach_python_as_sent() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let (gateway, received) = runtime.block_on(async { + let (upstream, received) = recording(|| Response::new(Body::from("ok"))).await; + ( + Gateway::start(upstream).await, + tokio::sync::Mutex::new(received), + ) + }); + let requests = (prop::collection::vec(wire_request(), 1..5), 1usize..4096); + TestRunner::new(ProptestConfig::with_cases(256)) + .run(&requests, |(requests, piece)| { + runtime.block_on(async { + let mut received = received.lock().await; + while received.try_recv().is_ok() {} + let mut raw = gateway.connect().await; + let wire: Vec = requests.iter().flat_map(encode).collect(); + for bytes in wire.chunks(piece) { + raw.send(bytes).await; + } + for request in &requests { + let head = within(raw.response_head()).await.unwrap(); + assert_eq!(head.status, StatusCode::OK, "{request:?}"); + assert_eq!(raw.body(2).await, b"ok"); + let python = within(received.recv()).await.unwrap(); + assert_eq!(python.method, request.method); + assert_eq!(python.uri, request.target.as_str()); + assert_eq!(python.body, request.body); + assert_eq!(python.headers["host"], "x"); + let forwarded = python.headers.iter().filter(|(name, _)| { + !["host", "content-length", "transfer-encoding", PEER_HEADER] + .contains(&name.as_str()) + }); + assert_eq!( + by_name(forwarded.map(|(name, value)| (name.as_str(), value.as_bytes()))), + by_name( + request + .headers + .iter() + .map(|(name, value)| (name.as_str(), value.as_slice())) + ) + ); + } + }); + Ok(()) + }) + .unwrap(); +} diff --git a/litellm-rust/crates/gateway-forward/tests/settings.rs b/litellm-rust/crates/gateway-forward/tests/settings.rs new file mode 100644 index 00000000000..ce0d95e723f --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/settings.rs @@ -0,0 +1,110 @@ +use std::{ffi::OsString, time::Duration}; + +use litellm_gateway_forward::{Error, Settings}; +use rstest::rstest; + +const COMPLETE: [(&str, &str); 5] = [ + ("LITELLM_FORWARD_UPSTREAM", "http://127.0.0.1:53211"), + ("LITELLM_FORWARD_HOST", "0.0.0.0"), + ("LITELLM_FORWARD_PORT", "4000"), + ("LITELLM_FORWARD_UPSTREAM_KEEPALIVE", "2"), + ("LITELLM_FORWARD_SHUTDOWN_DRAIN", "40"), +]; + +fn parse(variables: &[(&str, &str)]) -> Result, Error> { + Settings::from_vars( + variables + .iter() + .map(|(key, value)| (OsString::from(key), OsString::from(value))), + ) +} + +fn without(variable: &str) -> Vec<(&'static str, &'static str)> { + COMPLETE + .into_iter() + .filter(|(key, _)| *key != variable) + .collect() +} + +fn with<'a>(variable: &'a str, value: &'a str) -> Vec<(&'a str, &'a str)> { + without(variable) + .into_iter() + .chain([(variable, value)]) + .collect() +} + +#[rstest] +fn reads_every_variable() { + assert_eq!( + parse(&COMPLETE).unwrap(), + Some(Settings { + upstream: "http://127.0.0.1:53211".parse().unwrap(), + host: "0.0.0.0".into(), + port: 4000, + upstream_keepalive: Duration::from_secs(2), + shutdown_drain: Duration::from_secs(40), + }) + ); +} + +#[rstest] +fn without_an_upstream_the_gateway_is_not_forwarding() { + assert_eq!(parse(&without("LITELLM_FORWARD_UPSTREAM")).unwrap(), None); +} + +#[rstest] +#[case::host("LITELLM_FORWARD_HOST")] +#[case::port("LITELLM_FORWARD_PORT")] +#[case::upstream_keepalive("LITELLM_FORWARD_UPSTREAM_KEEPALIVE")] +#[case::shutdown_drain("LITELLM_FORWARD_SHUTDOWN_DRAIN")] +fn a_missing_variable_fails_naming_it(#[case] variable: &str) { + let error = parse(&without(variable)).unwrap_err(); + + assert!(error.to_string().contains(variable), "{error}"); +} + +#[rstest] +#[case::port_overflow("LITELLM_FORWARD_PORT", "65536")] +#[case::port_text("LITELLM_FORWARD_PORT", "http")] +#[case::keepalive_text("LITELLM_FORWARD_UPSTREAM_KEEPALIVE", "soon")] +#[case::keepalive_negative("LITELLM_FORWARD_UPSTREAM_KEEPALIVE", "-1")] +#[case::keepalive_zero("LITELLM_FORWARD_UPSTREAM_KEEPALIVE", "0")] +#[case::drain_text("LITELLM_FORWARD_SHUTDOWN_DRAIN", "soon")] +#[case::drain_negative("LITELLM_FORWARD_SHUTDOWN_DRAIN", "-1")] +#[case::drain_infinite("LITELLM_FORWARD_SHUTDOWN_DRAIN", "inf")] +fn an_invalid_variable_fails_naming_it(#[case] variable: &str, #[case] value: &str) { + let error = parse(&with(variable, value)).unwrap_err(); + + assert!(error.to_string().contains(variable), "{error}"); +} + +#[rstest] +fn fractional_drain_seconds_are_used_as_given() { + let settings = parse(&with("LITELLM_FORWARD_SHUTDOWN_DRAIN", "12.5")) + .unwrap() + .unwrap(); + + assert_eq!(settings.shutdown_drain, Duration::from_millis(12_500)); +} + +#[rstest] +#[case::keepalive("KEEPALIVE_TIMEOUT")] +#[case::graceful_shutdown("GRACEFUL_SHUTDOWN_TIMEOUT")] +#[case::host("HOST")] +#[case::port("PORT")] +fn python_settings_are_never_read(#[case] variable: &str) { + let mut variables = COMPLETE.to_vec(); + variables.push((variable, "9")); + + assert_eq!(parse(&variables).unwrap(), parse(&COMPLETE).unwrap()); +} + +#[rstest] +fn a_non_loopback_upstream_fails_startup() { + let error = parse(&with("LITELLM_FORWARD_UPSTREAM", "http://192.0.2.1:4000")).unwrap_err(); + + assert!( + error.to_string().contains("LITELLM_FORWARD_UPSTREAM"), + "{error}" + ); +} diff --git a/litellm-rust/crates/gateway-forward/tests/slow_head.rs b/litellm-rust/crates/gateway-forward/tests/slow_head.rs new file mode 100644 index 00000000000..764733aa524 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/slow_head.rs @@ -0,0 +1,41 @@ +mod support; + +use std::time::Duration; + +use axum::{body::Body, extract::Request, http::StatusCode}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, client, raw_upstream}; +use tokio::time::Instant; + +/// Reasoning models with `stream=false`, image and video generation, and model health checks send +/// no response headers until the provider finishes, which can take many minutes. +#[rstest] +#[tokio::test(start_paused = true)] +async fn waits_for_response_headers_as_long_as_python_takes() { + let thinking = Duration::from_secs(30 * 60); + let upstream = raw_upstream(move |mut python| async move { + python.request_head().await.unwrap(); + tokio::time::sleep(thinking).await; + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 5\r\n\r\nimage") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let started = Instant::now(); + + let response = client() + .request( + Request::post(gateway.url("/v1/images/generations")) + .body(Body::from("{}")) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert!(started.elapsed() >= thinking); + let body = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(body, "image"); +} diff --git a/litellm-rust/crates/gateway-forward/tests/stream.rs b/litellm-rust/crates/gateway-forward/tests/stream.rs new file mode 100644 index 00000000000..eaa12f20b77 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/stream.rs @@ -0,0 +1,313 @@ +mod support; + +use std::{ + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use axum::{body::Body, extract::Request, http::StatusCode}; +use http_body_util::BodyExt; +use hyper::body::Incoming; +use rstest::rstest; +use support::{Gateway, client, raw_upstream, within}; +use tokio::{ + io::AsyncWriteExt, + sync::{Notify, mpsc}, +}; + +fn chunk(bytes: &[u8]) -> Vec { + [format!("{:x}\r\n", bytes.len()).as_bytes(), bytes, b"\r\n"].concat() +} + +const EVENTS: [&str; 3] = ["data: 1\n\n", "data: 2\n\n", "data: [DONE]\n\n"]; + +#[rstest] +#[tokio::test] +async fn each_event_reaches_the_client_before_python_produces_the_next( + #[values(true, false)] chunked: bool, +) { + let produce = Arc::new(Notify::new()); + let next = produce.clone(); + let upstream = raw_upstream(move |mut python| { + let next = next.clone(); + async move { + python.request_head().await.unwrap(); + let framing = match chunked { + true => "transfer-encoding: chunked".to_owned(), + false => format!("content-length: {}", EVENTS.concat().len()), + }; + python + .send(format!( + "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\n{framing}\r\n\r\n" + )) + .await; + for event in EVENTS { + match chunked { + true => python.send(chunk(event.as_bytes())).await, + false => python.send(event).await, + } + next.notified().await; + } + if chunked { + python.send("0\r\n\r\n").await; + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut response = within( + client().request( + Request::post(gateway.url("/v1/chat/completions")) + .body(Body::from(r#"{"stream":true}"#)) + .unwrap(), + ), + ) + .await + .unwrap(); + + for event in EVENTS { + let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), event); + produce.notify_one(); + } + let rest = within(response.into_body().collect()).await.unwrap(); + assert!(rest.to_bytes().is_empty()); +} + +#[rstest] +#[tokio::test] +async fn relays_a_binary_framed_stream_unchanged() { + let message: Vec = [ + &[ + 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x20, 0x8a, 0x0d, 0x0a, 0xff, + ][..], + b"\x0b:event-type\x07\x00\x05chunk\x0d:content-type\x07\x00\x10application/json", + b"0\r\n\r\n{\"bytes\":\"eyJ0eXBlIjoiY29udGVudCJ9\"}", + &[0xc3, 0x28, 0x00, 0x0d], + ] + .concat(); + let sent = message.clone(); + let upstream = raw_upstream(move |mut python| { + let sent = sent.clone(); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: application/vnd.amazon.eventstream\r\ntransfer-encoding: chunked\r\n\r\n") + .await; + for piece in sent.chunks(7) { + python.send(chunk(piece)).await; + } + python.send("0\r\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = within( + client().request( + Request::post(gateway.url("/bedrock/model/m/converse-stream")) + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .unwrap(); + + assert_eq!( + response.headers()["content-type"], + "application/vnd.amazon.eventstream" + ); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, message); +} + +#[rstest] +#[tokio::test] +async fn relays_a_body_delimited_by_the_upstream_closing() { + let upstream = raw_upstream(|mut python| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\nconnection: close\r\n\r\nuntil eof") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap()); + let body = within(response).await.unwrap().into_body().collect(); + assert_eq!(within(body).await.unwrap().to_bytes(), "until eof"); +} + +async fn drain(mut body: Incoming) -> (Vec, Result<(), hyper::Error>) { + let mut delivered = Vec::new(); + loop { + match within(body.frame()).await { + None => return (delivered, Ok(())), + Some(Err(error)) => return (delivered, Err(error)), + Some(Ok(frame)) => delivered.extend_from_slice(&frame.into_data().unwrap()), + } + } +} + +#[rstest] +#[case::after_the_head("transfer-encoding: chunked", "", "")] +#[case::between_events("transfer-encoding: chunked", "6\r\ndata: \r\n", "data: ")] +#[case::mid_event("transfer-encoding: chunked", "a\r\ndata:", "data:")] +#[case::short_of_its_content_length("content-length: 100", "data: 1\n\n", "data: 1\n\n")] +#[tokio::test] +async fn an_upstream_dying_mid_stream_fails_the_client_body_instead_of_ending_it( + #[case] framing: &'static str, + #[case] sent: &'static str, + #[case] delivered: &'static str, +) { + let upstream = raw_upstream(move |mut python| async move { + python.request_head().await.unwrap(); + python + .send(format!( + "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\n{framing}\r\n\r\n{sent}" + )) + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let response = + within(client().request(Request::get(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let (received, ended) = drain(response.into_body()).await; + assert_eq!(received, delivered.as_bytes()); + assert!(ended.is_err()); +} + +#[rstest] +#[tokio::test] +async fn response_headers_reach_the_client_before_python_sends_any_body() { + let release = Arc::new(Notify::new()); + let released = release.clone(); + let upstream = raw_upstream(move |mut python| { + let released = released.clone(); + async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream; charset=utf-8\r\nx-accel-buffering: no\r\nx-litellm-call-id: call-1\r\ntransfer-encoding: chunked\r\n\r\n") + .await; + released.notified().await; + python.send(chunk(b"data: 1\n\n")).await; + python.send("0\r\n\r\n").await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + + let response = + within(client().request(Request::post(gateway.url("/")).body(Body::empty()).unwrap())) + .await + .unwrap(); + + assert_eq!(response.headers()["x-litellm-call-id"], "call-1"); + assert_eq!(response.headers()["x-accel-buffering"], "no"); + release.notify_one(); + let body = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(body, "data: 1\n\n"); +} + +/// A connection closed without any response looks retryable to client SDKs, which would re-run a +/// request Python already ran. Python dying right after its head races the relay, hence the loop. +#[rstest] +#[tokio::test] +async fn a_head_python_sent_right_before_dying_still_reaches_the_client() { + let upstream = raw_upstream(|mut python| async move { + python.request_head().await.unwrap(); + python + .send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n") + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + for _ in 0..20 { + let mut raw = gateway.connect().await; + raw.send("POST / HTTP/1.1\r\nhost: x\r\ncontent-length: 0\r\n\r\n") + .await; + let head = within(raw.response_head()).await; + assert_eq!(head.map(|head| head.status), Some(StatusCode::OK)); + } +} + +#[rstest] +#[tokio::test] +async fn a_client_that_stops_reading_stalls_python_instead_of_being_buffered_for() { + let written = Arc::new(AtomicUsize::new(0)); + let counted = written.clone(); + let (failed, mut write_failed) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python| { + let counted = counted.clone(); + let failed = failed.clone(); + async move { + python.request_head().await.unwrap(); + let mut python = python.into_stream(); + python + .write_all(b"HTTP/1.1 200 OK\r\ntransfer-encoding: chunked\r\n\r\n") + .await + .unwrap(); + let piece = chunk(&[0x5a; 64 * 1024]); + while python.write_all(&piece).await.is_ok() { + counted.fetch_add(piece.len(), Ordering::SeqCst); + } + failed.send(()).unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST / HTTP/1.1\r\nhost: x\r\ncontent-length: 0\r\n\r\n") + .await; + within(raw.response_head()).await.unwrap(); + + tokio::time::sleep(Duration::from_secs(1)).await; + let stalled = written.load(Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(250)).await; + + assert_eq!(written.load(Ordering::SeqCst), stalled); + assert!(stalled < 64 * 1024 * 1024, "{stalled} bytes in flight"); + drop(raw); + within(write_failed.recv()).await.unwrap(); +} + +#[rstest] +#[tokio::test] +async fn an_http10_client_receives_a_streamed_body_delimited_by_close() { + let upstream = raw_upstream(|mut python| async move { + python.request_head().await.unwrap(); + python + .send([ + &b"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\n\r\n"[..], + &chunk(b"data: 1\n"), + &chunk(b"data: 2\n"), + b"0\r\n\r\n", + ] + .concat()) + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("GET /v1/stream HTTP/1.0\r\nhost: x\r\n\r\n").await; + + let response = String::from_utf8(within(raw.until_closed()).await).unwrap(); + let (head, body) = response.split_once("\r\n\r\n").unwrap(); + assert!(head.starts_with("HTTP/1.0 200 OK"), "{head}"); + assert!( + !head.to_ascii_lowercase().contains("transfer-encoding"), + "{head}" + ); + assert_eq!(body, "data: 1\ndata: 2\n"); +} diff --git a/litellm-rust/crates/gateway-forward/tests/support/mod.rs b/litellm-rust/crates/gateway-forward/tests/support/mod.rs new file mode 100644 index 00000000000..7d92ca4a14a --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/support/mod.rs @@ -0,0 +1,320 @@ +#![allow( + dead_code, + reason = "each test binary uses a different subset of the harness" +)] + +use std::{future::Future, net::SocketAddr, time::Duration}; + +use axum::{ + Router, + body::{Body, Bytes}, + extract::Request, + http::{HeaderMap, Method, StatusCode, Uri}, +}; +use http_body_util::BodyExt; +use hyper_util::{ + client::legacy::{Client, connect::HttpConnector}, + rt::TokioExecutor, +}; +use litellm_gateway_forward::Forward; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + sync::{mpsc, oneshot}, + task::JoinHandle, +}; + +pub const WAIT: Duration = Duration::from_secs(5); +pub const UVICORN_KEEPALIVE: Duration = Duration::from_secs(5); +pub const DRAIN: Duration = Duration::from_secs(1); + +pub async fn within(future: impl Future) -> T { + tokio::time::timeout(WAIT, future).await.expect("timed out") +} + +/// A forwarder in front of `upstream` that drains once `shutdown` resolves, as `run` does. +pub async fn serve_until( + upstream: SocketAddr, + keepalive: Duration, + shutdown: impl Future + Send + 'static, +) -> (SocketAddr, JoinHandle<()>) { + let forward = Forward::new(&format!("http://{upstream}").parse().unwrap(), keepalive); + let public = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = public.local_addr().unwrap(); + ( + address, + tokio::spawn(forward.serve(public, shutdown, DRAIN)), + ) +} + +/// A running forwarder in front of `upstream`, listening on a real socket like production. +pub struct Gateway { + pub address: SocketAddr, + stop: oneshot::Sender<()>, + server: JoinHandle<()>, +} + +impl Gateway { + pub async fn start(upstream: SocketAddr) -> Self { + Self::with_keepalive(upstream, UVICORN_KEEPALIVE).await + } + + pub async fn with_keepalive(upstream: SocketAddr, keepalive: Duration) -> Self { + let (stop, stopped) = oneshot::channel::<()>(); + let (address, server) = serve_until(upstream, keepalive, async { + stopped.await.ok(); + }) + .await; + Self { + address, + stop, + server, + } + } + + pub fn url(&self, path_and_query: &str) -> Uri { + format!("http://{}{path_and_query}", self.address) + .parse() + .unwrap() + } + + pub async fn connect(&self) -> Raw { + Raw::new(TcpStream::connect(self.address).await.unwrap()) + } + + /// Starts graceful shutdown and returns the server task, which ends once in-flight work drains + /// or `DRAIN` passes. + pub fn shutdown(self) -> JoinHandle<()> { + self.stop.send(()).unwrap(); + self.server + } +} + +pub fn client() -> Client { + Client::builder(TokioExecutor::new()).build_http() +} + +/// What the upstream received, after the forwarder rewrote it. +pub struct Received { + pub method: Method, + pub uri: Uri, + pub headers: HeaderMap, + pub body: Bytes, +} + +/// An axum upstream that records every request it fully reads and answers with `respond`. +pub async fn recording( + respond: impl Fn() -> axum::response::Response + Clone + Send + Sync + 'static, +) -> (SocketAddr, mpsc::UnboundedReceiver) { + let (sender, receiver) = mpsc::unbounded_channel(); + let address = serve(Router::new().fallback(move |request: Request| { + let sender = sender.clone(); + let respond = respond.clone(); + async move { + let (parts, body) = request.into_parts(); + let body = body.collect().await.unwrap().to_bytes(); + sender + .send(Received { + method: parts.method, + uri: parts.uri, + headers: parts.headers, + body, + }) + .unwrap(); + respond() + } + })) + .await; + (address, receiver) +} + +pub async fn serve(router: Router) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + address +} + +/// An upstream that hands each accepted connection to `handle` as raw bytes, for behavior a +/// well-behaved HTTP server would never produce. +pub async fn raw_upstream(handle: F) -> SocketAddr +where + F: Fn(Raw) -> Fut + Send + Sync + 'static, + Fut: Future + Send + 'static, +{ + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + tokio::spawn(handle(Raw::new(stream))); + } + }); + address +} + +pub fn unreachable_upstream() -> SocketAddr { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.local_addr().unwrap() +} + +pub struct Head { + pub status: StatusCode, + pub headers: Vec<(String, Vec)>, + pub path: String, +} + +impl Head { + pub fn header(&self, name: &str) -> Option<&[u8]> { + self.headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_slice()) + } + + pub fn content_length(&self) -> Option { + self.header("content-length") + .map(|value| std::str::from_utf8(value).unwrap().parse().unwrap()) + } +} + +/// One TCP connection driven byte by byte, so tests control framing, timing and disconnects. +/// Bytes read past a message head stay buffered for the next read. +pub struct Raw { + stream: TcpStream, + buffer: Vec, +} + +impl Raw { + pub fn new(stream: TcpStream) -> Self { + Self { + stream, + buffer: Vec::new(), + } + } + + pub fn into_stream(self) -> TcpStream { + assert!(self.buffer.is_empty(), "buffered bytes would be lost"); + self.stream + } + + pub fn local_address(&self) -> SocketAddr { + self.stream.local_addr().unwrap() + } + + pub async fn send(&mut self, bytes: impl AsRef<[u8]>) { + self.stream.write_all(bytes.as_ref()).await.unwrap(); + } + + /// Reads one request head, or `None` when the peer closed the connection first. + pub async fn request_head(&mut self) -> Option { + self.head(|bytes| { + let mut headers = [httparse::EMPTY_HEADER; 256]; + let mut request = httparse::Request::new(&mut headers); + let httparse::Status::Complete(length) = request.parse(bytes).unwrap() else { + return None; + }; + Some(( + length, + Head { + status: StatusCode::OK, + headers: owned(request.headers), + path: request.path.unwrap().to_owned(), + }, + )) + }) + .await + } + + pub async fn response_head(&mut self) -> Option { + self.head(|bytes| { + let mut headers = [httparse::EMPTY_HEADER; 256]; + let mut response = httparse::Response::new(&mut headers); + let httparse::Status::Complete(length) = response.parse(bytes).unwrap() else { + return None; + }; + Some(( + length, + Head { + status: StatusCode::from_u16(response.code.unwrap()).unwrap(), + headers: owned(response.headers), + path: String::new(), + }, + )) + }) + .await + } + + async fn head(&mut self, parse: impl Fn(&[u8]) -> Option<(usize, Head)>) -> Option { + loop { + if let Some((length, head)) = parse(&self.buffer) { + self.buffer.drain(..length); + return Some(head); + } + if self.fill().await == 0 { + return None; + } + } + } + + /// Reads exactly `length` body bytes, using what is already buffered first. + pub async fn body(&mut self, length: usize) -> Vec { + while self.buffer.len() < length { + assert_ne!(self.fill().await, 0, "closed mid-body"); + } + self.buffer.drain(..length).collect() + } + + /// Reads a whole chunked body and returns its decoded bytes. + pub async fn chunked_body(&mut self) -> Vec { + let mut decoded = Vec::new(); + loop { + let size_line = self.line().await; + let size = + usize::from_str_radix(size_line.split(';').next().unwrap().trim(), 16).unwrap(); + if size == 0 { + while !self.line().await.is_empty() {} + return decoded; + } + decoded.extend(self.body(size).await); + assert_eq!(self.body(2).await, b"\r\n"); + } + } + + async fn line(&mut self) -> String { + loop { + if let Some(end) = self.buffer.windows(2).position(|window| window == b"\r\n") { + let line = String::from_utf8(self.buffer[..end].to_vec()).unwrap(); + self.buffer.drain(..end + 2); + return line; + } + assert_ne!(self.fill().await, 0, "closed mid-line"); + } + } + + async fn fill(&mut self) -> usize { + let mut chunk = [0; 8192]; + let read = self.stream.read(&mut chunk).await.unwrap_or(0); + self.buffer.extend_from_slice(&chunk[..read]); + read + } + + /// Everything the peer sends until it closes, including bytes already buffered. + pub async fn until_closed(&mut self) -> Vec { + while self.fill().await != 0 {} + std::mem::take(&mut self.buffer) + } + + /// Resolves once the peer closes the connection, discarding anything it still sends. + pub async fn closed(&mut self) { + let mut chunk = [0; 8192]; + while self.stream.read(&mut chunk).await.unwrap_or(0) != 0 {} + } +} + +fn owned(headers: &[httparse::Header<'_>]) -> Vec<(String, Vec)> { + headers + .iter() + .map(|header| (header.name.to_owned(), header.value.to_vec())) + .collect() +} diff --git a/litellm-rust/crates/gateway-forward/tests/upload.rs b/litellm-rust/crates/gateway-forward/tests/upload.rs new file mode 100644 index 00000000000..3dc865c9c86 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/upload.rs @@ -0,0 +1,339 @@ +mod support; + +use std::{ + convert::Infallible, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use axum::{ + Router, + body::{Body, Bytes}, + extract::Request, + http::StatusCode, + response::Response, +}; +use http_body_util::BodyExt; +use rstest::rstest; +use support::{Gateway, Raw, client, raw_upstream, recording, serve, within}; +use tokio::{ + io::AsyncWriteExt, + sync::{Notify, mpsc}, +}; + +fn ok() -> Response { + Response::new(Body::from("ok")) +} + +#[rstest] +#[tokio::test] +async fn streams_upload_and_response_incrementally_without_a_body_limit() { + let upstream = serve( + Router::new().fallback(|request: Request| async { Response::new(request.into_body()) }), + ) + .await; + let gateway = Gateway::start(upstream).await; + let (sender, receiver) = mpsc::channel::>(1); + let stream = futures_util::stream::unfold(receiver, |mut receiver| async { + receiver.recv().await.map(|chunk| (chunk, receiver)) + }); + sender.send(Ok(Bytes::from_static(b"first"))).await.unwrap(); + let mut response = within( + client().request( + Request::post(gateway.url("/v1/files")) + .body(Body::from_stream(stream)) + .unwrap(), + ), + ) + .await + .unwrap(); + let frame = within(response.body_mut().frame()).await.unwrap().unwrap(); + assert_eq!(frame.into_data().unwrap(), "first"); + let large = Bytes::from(vec![b'x'; 8 * 1024 * 1024]); + sender.send(Ok(large.clone())).await.unwrap(); + drop(sender); + let rest = within(response.into_body().collect()) + .await + .unwrap() + .to_bytes(); + assert_eq!(rest, large); +} + +/// hyper sends the 100 as soon as the request is forwarded, where uvicorn waits for the app to +/// read the body, so an upload Python rejects on headers alone is still sent in full. +#[rstest] +#[tokio::test] +async fn answers_expect_continue_so_large_uploads_start() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send("POST /v1/audio/transcriptions HTTP/1.1\r\nhost: x\r\nexpect: 100-continue\r\ncontent-length: 4\r\n\r\n") + .await; + let interim = within(raw.response_head()).await.unwrap(); + assert_eq!(interim.status, StatusCode::CONTINUE); + raw.send("data").await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().body, "data"); +} + +/// curl sends `expect: 100-continue` for any body over 1 MiB and waits for the 100 before sending +/// the multipart body, which must reach Python intact even where it looks like framing. +#[rstest] +#[tokio::test] +async fn expect_continue_delivers_a_large_multipart_upload_byte_for_byte() { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let boundary = "------------------------d74496d66958873e"; + let audio: Vec = (0..3 * 1024 * 1024) + .map(|index: u32| (index.wrapping_mul(2_654_435_761) >> 24) as u8) + .chain(*b"\r\n0\r\n\r\nPOST /smuggled HTTP/1.1\r\n\r\n--") + .collect(); + let body = [ + format!("--{boundary}\r\ncontent-disposition: form-data; name=\"model\"\r\n\r\nwhisper-1\r\n--{boundary}\r\ncontent-disposition: form-data; name=\"file\"; filename=\"a b.wav\"\r\ncontent-type: audio/wav\r\n\r\n").into_bytes(), + audio, + format!("\r\n--{boundary}--\r\n").into_bytes(), + ] + .concat(); + let content_type = format!("multipart/form-data; boundary={boundary}"); + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/audio/transcriptions HTTP/1.1\r\nhost: x\r\ncontent-type: {content_type}\r\nexpect: 100-continue\r\ncontent-length: {}\r\n\r\n", + body.len() + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::CONTINUE + ); + raw.send(&body).await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + let request = received.recv().await.unwrap(); + assert_eq!(request.headers["content-type"], content_type.as_str()); + assert_eq!(request.body, body); +} + +#[rstest] +#[case::get("GET")] +#[case::delete("DELETE")] +#[case::options("OPTIONS")] +#[case::post("POST")] +#[tokio::test] +async fn forwards_a_chunked_body_on_any_method(#[case] method: &str) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "{method} /v1/search HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + assert_eq!(received.recv().await.unwrap().body, "body"); +} + +#[rstest] +#[case::no_framing("", "", &[], "")] +#[case::empty_content_length("content-length: 0\r\n", "", &[("content-length", "0")], "")] +#[case::empty_chunked( + "transfer-encoding: chunked\r\n", + "0\r\n\r\n", + &[("transfer-encoding", "chunked")], + "" +)] +#[case::sized( + "content-length: 7\r\n", + "payload", + &[("content-length", "7")], + "payload" +)] +#[case::chunk_extensions_and_trailers( + "transfer-encoding: chunked\r\n", + "4;name=\"v\"\r\nbody\r\n0\r\nx-checksum: 1\r\n\r\n", + &[("transfer-encoding", "chunked")], + "body" +)] +#[tokio::test] +async fn a_body_reaches_python_with_the_framing_the_client_chose( + #[values("POST", "GET")] method: &str, + #[case] framing: &str, + #[case] wire_body: &str, + #[case] python_framing: &[(&str, &str)], + #[case] python_body: &str, +) { + let (upstream, mut received) = recording(ok).await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "{method} /v1/files HTTP/1.1\r\nhost: x\r\n{framing}\r\n{wire_body}" + )) + .await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); + + let request = received.recv().await.unwrap(); + let forwarded: Vec<(&str, &str)> = ["content-length", "transfer-encoding"] + .into_iter() + .flat_map(|name| { + request + .headers + .get_all(name) + .iter() + .map(move |value| (name, value.to_str().unwrap())) + }) + .collect(); + assert_eq!(request.method, method); + assert_eq!(forwarded, python_framing); + assert_eq!(request.body, python_body); +} + +#[rstest] +#[tokio::test] +async fn a_large_upload_is_paced_by_python_reading_it() { + const LENGTH: usize = 128 * 1024 * 1024; + let start_reading = Arc::new(Notify::new()); + let reading = start_reading.clone(); + let upstream = raw_upstream(move |mut python| { + let reading = reading.clone(); + async move { + let head = python.request_head().await.unwrap(); + reading.notified().await; + let mut remaining = head.content_length().unwrap(); + while remaining > 0 { + remaining -= python.body(remaining.min(1 << 20)).await.len(); + } + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 0\r\n\r\n") + .await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let (read, mut write) = gateway.connect().await.into_stream().into_split(); + let sent = Arc::new(AtomicUsize::new(0)); + let sending = sent.clone(); + let upload = tokio::spawn(async move { + write + .write_all( + format!("POST /v1/files HTTP/1.1\r\nhost: x\r\ncontent-length: {LENGTH}\r\n\r\n") + .as_bytes(), + ) + .await + .unwrap(); + let chunk = [0x5a; 64 * 1024]; + while sending.load(Ordering::SeqCst) < LENGTH { + write.write_all(&chunk).await.unwrap(); + sending.fetch_add(chunk.len(), Ordering::SeqCst); + } + write + }); + + tokio::time::sleep(Duration::from_secs(1)).await; + assert!( + sent.load(Ordering::SeqCst) < LENGTH / 4, + "the forwarder buffered an upload Python was not reading" + ); + start_reading.notify_one(); + let write = tokio::time::timeout(Duration::from_secs(60), upload) + .await + .unwrap() + .unwrap(); + let mut raw = Raw::new(read.reunite(write).unwrap()); + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::OK + ); +} + +#[rstest] +#[case::between_chunks("5\r\nhello\r\n")] +#[case::mid_chunk("100\r\nhello")] +#[tokio::test] +async fn a_client_abandoning_a_chunked_upload_never_ends_python_body_cleanly( + #[case] sent: &'static str, +) { + let (forwarded, mut received) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python| { + let forwarded = forwarded.clone(); + async move { + python.request_head().await.unwrap(); + forwarded.send(python.until_closed().await).unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\ntransfer-encoding: chunked\r\n\r\n{sent}" + )) + .await; + tokio::time::sleep(Duration::from_millis(100)).await; + drop(raw); + + let body = within(received.recv()).await.unwrap(); + assert!(body.starts_with(b"5\r\nhello\r\n")); + assert!( + !body.ends_with(b"0\r\n\r\n"), + "python saw a truncated upload end cleanly" + ); +} + +#[rstest] +#[tokio::test] +async fn an_early_answer_during_an_upload_keeps_both_connections_in_sync() { + let (event, mut events) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python| { + let event = event.clone(); + async move { + while let Some(head) = python.request_head().await { + let length = head.content_length(); + event.send(head.path.into_bytes()).unwrap(); + let Some(length) = length else { + python + .send("HTTP/1.1 200 OK\r\ncontent-length: 4\r\n\r\nnext") + .await; + continue; + }; + python + .send("HTTP/1.1 413 Payload Too Large\r\ncontent-length: 2\r\n\r\n{}") + .await; + event.send(python.body(length).await).unwrap(); + } + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let upload = b"GET /smuggled HTTP/1.1\r\nhost: x\r\n\r\n".repeat(1024); + let (first_half, second_half) = upload.split_at(upload.len() / 2); + let mut raw = gateway.connect().await; + raw.send(format!( + "POST /v1/files HTTP/1.1\r\nhost: x\r\ncontent-length: {}\r\n\r\n", + upload.len() + )) + .await; + raw.send(first_half).await; + + let rejected = within(raw.response_head()).await.unwrap(); + assert_eq!(rejected.status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(raw.body(2).await, b"{}"); + raw.send(second_half).await; + raw.send("GET /next HTTP/1.1\r\nhost: x\r\n\r\n").await; + let next = within(raw.response_head()).await.unwrap(); + assert_eq!(raw.body(next.content_length().unwrap()).await, b"next"); + + assert_eq!(within(events.recv()).await.unwrap(), b"/v1/files"); + assert_eq!(within(events.recv()).await.unwrap(), upload); + assert_eq!(within(events.recv()).await.unwrap(), b"/next"); +} diff --git a/litellm-rust/crates/gateway-forward/tests/upstream.rs b/litellm-rust/crates/gateway-forward/tests/upstream.rs new file mode 100644 index 00000000000..25e4768a54e --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/upstream.rs @@ -0,0 +1,21 @@ +use litellm_gateway_forward::Upstream; +use rstest::rstest; + +#[rstest] +#[case::https("https://127.0.0.1:4000")] +#[case::public("http://192.0.2.1:4000")] +#[case::hostname("http://localhost:4000")] +#[case::ipv6("http://[::1]:4000")] +#[case::path("http://127.0.0.1:4000/api")] +#[case::query("http://127.0.0.1:4000/?x=y")] +#[case::credentials("http://user@127.0.0.1:4000")] +fn rejects_anything_but_a_loopback_http_origin(#[case] upstream: &str) { + assert!(upstream.parse::().is_err()); +} + +#[rstest] +#[case::standard("http://127.0.0.1:4000")] +#[case::loopback_range("http://127.0.0.2:4000/")] +fn accepts_a_loopback_http_origin(#[case] upstream: &str) { + assert!(upstream.parse::().is_ok()); +} diff --git a/litellm-rust/crates/gateway-forward/tests/websocket.rs b/litellm-rust/crates/gateway-forward/tests/websocket.rs new file mode 100644 index 00000000000..ff64ddf38d3 --- /dev/null +++ b/litellm-rust/crates/gateway-forward/tests/websocket.rs @@ -0,0 +1,719 @@ +mod support; + +use std::{ + net::SocketAddr, + sync::Arc, + time::{Duration, Instant}, +}; + +use axum::{ + Router, + http::{HeaderValue, StatusCode}, + response::IntoResponse, +}; +use futures_util::{SinkExt, StreamExt, future::join_all}; +use litellm_gateway_forward::PEER_HEADER; +use rstest::rstest; +use support::{DRAIN, Gateway, Head, Raw, raw_upstream, serve, within}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpStream, + sync::{Barrier, mpsc}, +}; +use tokio_tungstenite::{ + WebSocketStream, accept_hdr_async, client_async, + tungstenite::{ + Error, Message, + client::IntoClientRequest, + handshake::{ + derive_accept_key, + server::{ErrorResponse, Request as Handshake, Response as Accepted}, + }, + protocol::{CloseFrame, frame::coding::CloseCode}, + }, +}; + +const KEY: &str = "dGhlIHNhbXBsZSBub25jZQ=="; + +async fn connect( + address: SocketAddr, + connection: &'static str, +) -> Result, Box> { + let mut request = format!("ws://{address}/v1/realtime?model=m") + .into_client_request() + .unwrap(); + let headers = request.headers_mut(); + headers.insert("authorization", HeaderValue::from_static("Bearer test")); + headers.insert( + "sec-websocket-protocol", + HeaderValue::from_static("realtime"), + ); + headers.insert("connection", HeaderValue::from_static(connection)); + let stream = TcpStream::connect(address).await.unwrap(); + let (socket, response) = client_async(request, stream).await.map_err(Box::new)?; + assert_eq!(response.headers()["sec-websocket-protocol"], "realtime"); + Ok(socket) +} + +/// Greets, echoes every message, and closes with 4000 when told to "bye". +#[expect( + clippy::result_large_err, + reason = "tungstenite's handshake callback returns ErrorResponse" +)] +async fn echo() -> SocketAddr { + raw_upstream(|python| async move { + let callback = |request: &Handshake, mut response: Accepted| { + assert_eq!(request.uri(), "/v1/realtime?model=m"); + assert_eq!(request.headers()["authorization"], "Bearer test"); + assert!(request.headers().contains_key(PEER_HEADER)); + response.headers_mut().insert( + "sec-websocket-protocol", + HeaderValue::from_static("realtime"), + ); + Ok::<_, ErrorResponse>(response) + }; + let mut socket = accept_hdr_async(python.into_stream(), callback) + .await + .unwrap(); + socket.send(Message::text("greeting")).await.unwrap(); + while let Some(Ok(message)) = socket.next().await { + if message == Message::text("bye") { + socket + .close(Some(CloseFrame { + code: CloseCode::Library(4000), + reason: "done".into(), + })) + .await + .unwrap(); + continue; + } + if message.is_close() { + break; + } + socket.send(message).await.unwrap(); + } + }) + .await +} + +fn handshake(target: &str, headers: &str) -> String { + format!( + "GET {target} HTTP/1.1\r\nhost: public.example\r\nconnection: Upgrade\r\nupgrade: websocket\r\nsec-websocket-version: 13\r\nsec-websocket-key: {KEY}\r\n{headers}\r\n" + ) +} + +/// The 101 uvicorn writes for `handshake`, with `headers` added. +fn switching(handshake: &Head, headers: &str) -> String { + let accept = derive_accept_key(handshake.header("sec-websocket-key").unwrap()); + format!( + "HTTP/1.1 101 Switching Protocols\r\nupgrade: websocket\r\nconnection: Upgrade\r\nsec-websocket-accept: {accept}\r\n{headers}\r\n" + ) +} + +fn values(head: &Head, name: &str) -> Vec { + head.headers + .iter() + .filter(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| String::from_utf8(value.clone()).unwrap()) + .collect() +} + +/// Accepts with `headers` in the 101, hands the handshake head to the test, then echoes bytes. +async fn byte_echo(headers: &'static str) -> (SocketAddr, mpsc::UnboundedReceiver) { + let (sender, handshakes) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let sender = sender.clone(); + async move { + let handshake = python.request_head().await.unwrap(); + python.send(switching(&handshake, headers)).await; + sender.send(handshake).unwrap(); + echo_bytes(python).await; + } + }) + .await; + (upstream, handshakes) +} + +async fn echo_bytes(python: Raw) { + let mut stream = python.into_stream(); + let (mut read, mut write) = stream.split(); + tokio::io::copy(&mut read, &mut write).await.ok(); +} + +async fn assert_echoes(socket: &mut TcpStream, message: &[u8]) { + socket.write_all(message).await.unwrap(); + let mut echoed = vec![0; message.len()]; + within(socket.read_exact(&mut echoed)).await.unwrap(); + assert_eq!(echoed, message); +} + +async fn open(gateway: &Gateway, handshake: String) -> (TcpStream, Head) { + let mut raw = gateway.connect().await; + raw.send(handshake).await; + let switched = within(raw.response_head()).await.unwrap(); + assert_eq!(switched.status, StatusCode::SWITCHING_PROTOCOLS); + (raw.into_stream(), switched) +} + +#[rstest] +#[tokio::test] +async fn relays_messages_both_ways( + #[values( + "Upgrade", + "keep-alive, Upgrade", + "upgrade", + "Upgrade, Sec-WebSocket-Key, Sec-WebSocket-Version" + )] + connection: &'static str, + #[values( + Message::text("hello"), + Message::binary(vec![0, 1, 2]), + Message::binary(vec![7; 4 * 1024 * 1024]) + )] + message: Message, +) { + let gateway = Gateway::start(echo().await).await; + let mut socket = within(connect(gateway.address, connection)).await.unwrap(); + assert_eq!( + socket.next().await.unwrap().unwrap(), + Message::text("greeting") + ); + socket.send(message.clone()).await.unwrap(); + assert_eq!(within(socket.next()).await.unwrap().unwrap(), message); + socket.close(None).await.unwrap(); + while socket.next().await.is_some() {} + drop(socket); + within(gateway.shutdown()).await.unwrap(); +} + +#[rstest] +#[tokio::test] +async fn relays_a_close_python_initiates() { + let gateway = Gateway::start(echo().await).await; + let mut socket = within(connect(gateway.address, "Upgrade")).await.unwrap(); + socket.next().await.unwrap().unwrap(); + socket.send(Message::text("bye")).await.unwrap(); + let Message::Close(Some(frame)) = within(socket.next()).await.unwrap().unwrap() else { + panic!("expected python's close frame"); + }; + assert_eq!(frame.code, CloseCode::Library(4000)); + assert_eq!(frame.reason, "done"); +} + +#[rstest] +#[tokio::test] +async fn passes_an_upstream_rejection_through() { + let upstream = serve( + Router::new() + .fallback(|| async { (StatusCode::FORBIDDEN, "origin not allowed").into_response() }), + ) + .await; + let gateway = Gateway::start(upstream).await; + let error = within(connect(gateway.address, "Upgrade")) + .await + .unwrap_err(); + let Error::Http(response) = *error else { + panic!("expected the upstream rejection, got {error}"); + }; + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(response.body().as_deref(), Some(&b"origin not allowed"[..])); +} + +/// The idle timeout only covers a connection waiting for its next HTTP request, and a realtime +/// session may sit silent between turns. +#[rstest] +#[tokio::test] +async fn keeps_a_quiet_socket_open_past_the_keepalive() { + let gateway = Gateway::with_keepalive(echo().await, Duration::from_millis(400)).await; + let mut socket = within(connect(gateway.address, "Upgrade")).await.unwrap(); + socket.next().await.unwrap().unwrap(); + tokio::time::sleep(Duration::from_millis(1200)).await; + socket.send(Message::text("still here")).await.unwrap(); + assert_eq!( + within(socket.next()).await.unwrap().unwrap(), + Message::text("still here") + ); +} + +/// Browsers carry the API key in a subprotocol, Python accepts with the first one it was offered, +/// and uvicorn negotiates permessage-deflate, so every handshake header must cross unchanged. +#[rstest] +#[tokio::test] +async fn handshake_headers_cross_unchanged_both_ways() { + let (upstream, mut handshakes) = byte_echo( + "sec-websocket-protocol: realtime\r\nsec-websocket-extensions: permessage-deflate; server_max_window_bits=12; client_max_window_bits=12\r\nset-cookie: a=1\r\nset-cookie: b=2\r\nx-litellm-call-id: abc\r\n", + ) + .await; + let gateway = Gateway::start(upstream).await; + let target = "/v1/realtime?model=gpt-realtime&api-key=sk%2Fa%2Bb%3D&dup=1&dup=2"; + let (_socket, switched) = open( + &gateway, + handshake( + target, + "sec-websocket-protocol: realtime, openai-insecure-api-key.sk-websocket-test, openai-beta.realtime-v1\r\nsec-websocket-protocol: second-line\r\nsec-websocket-extensions: permessage-deflate; client_max_window_bits\r\ncookie: token=abc\r\n", + ), + ) + .await; + let handshake = within(handshakes.recv()).await.unwrap(); + + assert_eq!(handshake.path, target); + assert_eq!( + values(&handshake, "sec-websocket-protocol"), + [ + "realtime, openai-insecure-api-key.sk-websocket-test, openai-beta.realtime-v1", + "second-line" + ] + ); + assert_eq!( + values(&handshake, "sec-websocket-extensions"), + ["permessage-deflate; client_max_window_bits"] + ); + assert_eq!(values(&handshake, "cookie"), ["token=abc"]); + assert_eq!(values(&switched, "sec-websocket-protocol"), ["realtime"]); + assert_eq!( + values(&switched, "sec-websocket-extensions"), + ["permessage-deflate; server_max_window_bits=12; client_max_window_bits=12"] + ); + assert_eq!(values(&switched, "set-cookie"), ["a=1", "b=2"]); + assert_eq!(values(&switched, "x-litellm-call-id"), ["abc"]); +} + +#[rstest] +#[tokio::test] +async fn invents_no_subprotocol_or_extension() { + let (upstream, mut handshakes) = byte_echo("").await; + let gateway = Gateway::start(upstream).await; + let (_socket, switched) = open(&gateway, handshake("/v1/realtime", "")).await; + let handshake = within(handshakes.recv()).await.unwrap(); + for name in ["sec-websocket-protocol", "sec-websocket-extensions"] { + assert!(handshake.header(name).is_none()); + assert!(switched.header(name).is_none()); + } +} + +#[rstest] +#[case::capitalized_upgrade("Upgrade: WebSocket\r\nConnection: keep-alive, Upgrade\r\n")] +#[case::uppercase("UPGRADE: WEBSOCKET\r\nCONNECTION: UPGRADE\r\n")] +#[case::split_connection("Upgrade: websocket\r\nConnection: keep-alive\r\nConnection: Upgrade\r\n")] +#[tokio::test] +async fn accepts_any_spelling_of_the_upgrade_headers(#[case] upgrade: &str) { + let (upstream, _handshakes) = byte_echo("").await; + let gateway = Gateway::start(upstream).await; + let (mut socket, _) = open( + &gateway, + format!( + "GET /v1/realtime HTTP/1.1\r\nhost: x\r\n{upgrade}sec-websocket-version: 13\r\nsec-websocket-key: {KEY}\r\n\r\n" + ), + ) + .await; + socket.write_all(b"ping").await.unwrap(); + let mut echoed = [0; 4]; + within(socket.read_exact(&mut echoed)).await.unwrap(); + assert_eq!(&echoed, b"ping"); +} + +/// Compressed frames set RSV1 and every client frame is masked, so the socket is relayed as bytes, +/// never re-framed. +#[rstest] +#[tokio::test] +async fn relays_socket_bytes_exactly() { + let (upstream, _handshakes) = byte_echo("").await; + let gateway = Gateway::start(upstream).await; + let (socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + let sent: Vec = (0..3_000_017u32) + .map(|index| index.wrapping_mul(2_654_435_761).to_be_bytes()[1]) + .collect(); + let (mut read, mut write) = socket.into_split(); + let writing = sent.clone(); + let writer = tokio::spawn(async move { + for chunk in writing.chunks(7919) { + write.write_all(chunk).await.unwrap(); + } + write + }); + let mut echoed = vec![0; sent.len()]; + within(read.read_exact(&mut echoed)).await.unwrap(); + assert!(echoed == sent); + drop(writer.await.unwrap()); +} + +/// uvicorn writes the 101, an error event and a close frame back to back and then closes, so they +/// share a segment with the FIN, and a client may send its first frame with the handshake. +#[rstest] +#[tokio::test] +async fn keeps_bytes_that_arrive_with_the_handshake() { + let (early, mut received) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let early = early.clone(); + async move { + let handshake = python.request_head().await.unwrap(); + let mut segment = switching(&handshake, "").into_bytes(); + segment.extend(b"\x81\x05error\x88\x08\x03\xf0denied"); + python.send(segment).await; + early.send(within(python.body(6)).await).unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(handshake("/v1/realtime", "") + "early!").await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::SWITCHING_PROTOCOLS + ); + assert_eq!(within(received.recv()).await.unwrap(), b"early!"); + assert_eq!( + within(raw.until_closed()).await, + b"\x81\x05error\x88\x08\x03\xf0denied" + ); +} + +/// Python frees a realtime session's budget and parallel slot when its socket closes, so a client +/// that disappears without a close frame must still close Python's socket. +#[rstest] +#[case::fin(false)] +#[case::reset(true)] +#[tokio::test] +async fn a_vanished_client_closes_the_python_socket(#[case] reset: bool) { + let (event, mut events) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let event = event.clone(); + async move { + let handshake = python.request_head().await.unwrap(); + python.send(switching(&handshake, "")).await; + event.send("open").unwrap(); + python.closed().await; + event.send("closed").unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let (socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + assert_eq!(within(events.recv()).await, Some("open")); + if reset { + #[expect(deprecated, reason = "a zero linger is how a test sends a TCP reset")] + socket.set_linger(Some(Duration::ZERO)).unwrap(); + } + drop(socket); + assert_eq!(within(events.recv()).await, Some("closed")); +} + +#[rstest] +#[case::fin(false)] +#[case::reset(true)] +#[tokio::test] +async fn a_vanished_python_ends_the_client_socket(#[case] reset: bool) { + let upstream = raw_upstream(move |mut python: Raw| async move { + let handshake = python.request_head().await.unwrap(); + python.send(switching(&handshake, "")).await; + let socket = python.into_stream(); + if reset { + #[expect(deprecated, reason = "a zero linger is how a test sends a TCP reset")] + socket.set_linger(Some(Duration::ZERO)).unwrap(); + } + drop(socket); + }) + .await; + let gateway = Gateway::start(upstream).await; + let (mut socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + let mut rest = Vec::new(); + assert!(matches!( + within(socket.read_to_end(&mut rest)).await, + Ok(0) | Err(_) + )); +} + +#[rstest] +#[tokio::test] +async fn shutdown_closes_open_sockets_after_drain() { + let gateway = Gateway::start(echo().await).await; + let mut socket = within(connect(gateway.address, "Upgrade")).await.unwrap(); + socket.next().await.unwrap().unwrap(); + let started = Instant::now(); + within(gateway.shutdown()).await.unwrap(); + assert!(started.elapsed() >= DRAIN); + assert!(!matches!(socket.next().await, Some(Ok(Message::Text(_))))); +} + +#[rstest] +#[tokio::test] +async fn shutdown_relays_the_close_python_sends_while_draining() { + let gateway = Gateway::start(echo().await).await; + let mut socket = within(connect(gateway.address, "Upgrade")).await.unwrap(); + socket.next().await.unwrap().unwrap(); + let server = gateway.shutdown(); + tokio::time::sleep(Duration::from_millis(100)).await; + socket.send(Message::text("bye")).await.unwrap(); + let Message::Close(Some(frame)) = within(socket.next()).await.unwrap().unwrap() else { + panic!("expected python's close frame"); + }; + assert_eq!(frame.code, CloseCode::Library(4000)); + while socket.next().await.is_some() {} + drop(socket); + let started = Instant::now(); + within(server).await.unwrap(); + assert!(started.elapsed() < DRAIN / 2); +} + +#[rstest] +#[tokio::test] +async fn refuses_new_sockets_while_draining() { + let gateway = Gateway::start(echo().await).await; + let mut open = within(connect(gateway.address, "Upgrade")).await.unwrap(); + open.next().await.unwrap().unwrap(); + let address = gateway.address; + let server = gateway.shutdown(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + TcpStream::connect(address).await.is_err() || connect(address, "Upgrade").await.is_err() + ); + open.send(Message::text("still open")).await.unwrap(); + assert_eq!( + within(open.next()).await.unwrap().unwrap(), + Message::text("still open") + ); + drop(open); + within(server).await.unwrap(); +} + +/// uvicorn never half-closes: once Python closes the socket nothing more can cross it, so the +/// relay must end instead of waiting for the client to close its side too. +#[rstest] +#[tokio::test] +async fn relay_ends_when_python_closes_while_the_client_holds_its_socket() { + let upstream = raw_upstream(move |mut python: Raw| async move { + let handshake = python.request_head().await.unwrap(); + python.send(switching(&handshake, "")).await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let (mut socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + assert_eq!(within(socket.read(&mut [0; 1])).await.unwrap(), 0); + let started = Instant::now(); + within(gateway.shutdown()).await.unwrap(); + assert!(started.elapsed() < DRAIN / 2); + drop(socket); +} + +/// uvicorn answers a handshake Python closes before accepting with a 403, and one Python denies +/// through `websocket.http.response` with any status, headers and body, then drops the socket. The +/// client gets that response whole and keeps a connection that still reaches Python. +#[rstest] +#[case::closed_before_accept("403 Forbidden", None, "")] +#[case::denial_response( + "401 Unauthorized", + Some("Bearer realm=\"litellm\""), + "{\"error\":\"invalid api key\"}" +)] +#[tokio::test] +async fn a_denied_handshake_reaches_the_client_whole( + #[case] status: &'static str, + #[case] authenticate: Option<&'static str>, + #[case] body: &'static str, +) { + let upstream = raw_upstream(move |mut python: Raw| async move { + python.request_head().await.unwrap(); + let authenticate = authenticate + .map(|value| format!("www-authenticate: {value}\r\n")) + .unwrap_or_default(); + python + .send(format!( + "HTTP/1.1 {status}\r\ndate: Tue, 03 Nov 2026 08:00:00 GMT\r\nconnection: close\r\ncontent-length: {}\r\ncontent-type: text/plain; charset=utf-8\r\n{authenticate}\r\n{body}", + body.len() + )) + .await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + for _ in 0..2 { + raw.send(handshake("/v1/realtime", "")).await; + let denied = within(raw.response_head()).await.unwrap(); + assert_eq!(denied.status.as_str(), &status[..3]); + assert_eq!( + values(&denied, "www-authenticate"), + Vec::from_iter(authenticate.map(String::from)) + ); + assert!(denied.header("sec-websocket-accept").is_none()); + let length = denied.content_length().unwrap(); + assert_eq!(within(raw.body(length)).await, body.as_bytes()); + } +} + +/// Only Python's EOF ends a relay, so a client that half-closes after its close frame still gets +/// everything Python sends before closing its own side. +#[rstest] +#[tokio::test] +async fn a_client_half_close_still_receives_what_python_sends_after_it() { + let upstream = raw_upstream(|mut python: Raw| async move { + let handshake = python.request_head().await.unwrap(); + python.send(switching(&handshake, "")).await; + let received = python.until_closed().await; + python.send(received).await; + }) + .await; + let gateway = Gateway::start(upstream).await; + let (mut socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + let close = b"\x88\x82\x01\x02\x03\x04\x02\xea"; + socket.write_all(close).await.unwrap(); + socket.shutdown().await.unwrap(); + let mut replied = Vec::new(); + within(socket.read_to_end(&mut replied)).await.unwrap(); + assert_eq!(replied, close); +} + +/// Python authenticates every handshake before accepting it, so sessions opened together must all +/// reach Python before any of them is accepted, and each must stay on its own socket. +#[rstest] +#[tokio::test] +async fn opens_many_sockets_at_once_without_crossing_them() { + const SOCKETS: usize = 64; + let everyone = Arc::new(Barrier::new(SOCKETS)); + let upstream = raw_upstream(move |mut python: Raw| { + let everyone = everyone.clone(); + async move { + let handshake = python.request_head().await.unwrap(); + everyone.wait().await; + python.send(switching(&handshake, "")).await; + echo_bytes(python).await; + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let gateway = &gateway; + within(join_all((0..SOCKETS).map(|index| async move { + let (mut socket, _) = open(gateway, handshake("/v1/realtime", "")).await; + assert_echoes(&mut socket, format!("socket {index}").as_bytes()).await; + }))) + .await; +} + +/// Python resolves the key, budget and model before it accepts, and nothing bounds how long that +/// takes, so the idle timeout must not cut a handshake waiting on its 101. +#[rstest] +#[tokio::test] +async fn waits_for_a_101_slower_than_the_keepalive() { + let upstream = raw_upstream(|mut python: Raw| async move { + let handshake = python.request_head().await.unwrap(); + tokio::time::sleep(Duration::from_millis(1200)).await; + python.send(switching(&handshake, "")).await; + echo_bytes(python).await; + }) + .await; + let gateway = Gateway::with_keepalive(upstream, Duration::from_millis(400)).await; + let (mut socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + assert_echoes(&mut socket, b"late").await; +} + +/// A 101 the client did not ask for, or to a protocol other than the one it asked for, would hand +/// the client a socket it cannot speak, so it is a 502 and Python's socket is closed. +#[rstest] +#[case::plain_request("GET /v1/models HTTP/1.1\r\nhost: x\r\n\r\n".to_owned(), "websocket")] +#[case::other_protocol(handshake("/v1/realtime", ""), "h2c")] +#[tokio::test] +async fn an_unrequested_switch_is_a_502_that_closes_python( + #[case] request: String, + #[case] protocol: &'static str, +) { + let (closed, mut closes) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let closed = closed.clone(); + async move { + python.request_head().await.unwrap(); + python + .send(format!( + "HTTP/1.1 101 Switching Protocols\r\nconnection: upgrade\r\nupgrade: {protocol}\r\n\r\n" + )) + .await; + python.closed().await; + closed.send(()).unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let mut raw = gateway.connect().await; + raw.send(request).await; + assert_eq!( + within(raw.response_head()).await.unwrap().status, + StatusCode::BAD_GATEWAY + ); + within(closes.recv()).await.unwrap(); +} + +/// A realtime session streams audio for as long as the client reads it, so a client that stops +/// reading must stall Python's writes instead of piling them up in the gateway. +#[rstest] +#[tokio::test] +async fn a_client_that_stops_reading_stalls_python() { + let (sent, mut totals) = mpsc::unbounded_channel(); + let upstream = raw_upstream(move |mut python: Raw| { + let sent = sent.clone(); + async move { + let handshake = python.request_head().await.unwrap(); + python.send(switching(&handshake, "")).await; + python.body(2).await; + let mut stream = python.into_stream(); + let chunk = vec![0x82; 64 * 1024]; + let mut total = 0; + while let Ok(Ok(())) = + tokio::time::timeout(Duration::from_millis(300), stream.write_all(&chunk)).await + { + total += chunk.len(); + } + sent.send(total).unwrap(); + } + }) + .await; + let gateway = Gateway::start(upstream).await; + let (mut socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + socket.write_all(b"go").await.unwrap(); + let total = within(totals.recv()).await.unwrap(); + assert!( + total < 64 << 20, + "python wrote {total} bytes into a stalled socket" + ); +} + +/// Python closes a session that is still streaming audio by sending an error event and a close +/// frame and dropping its socket with the client's next frames unread, which resets it. Those last +/// bytes reached the client when it talked to uvicorn directly, so they must reach it through the +/// relay too, even though writing the client's frames to Python now fails. +#[rstest] +#[tokio::test] +async fn python_closing_mid_stream_still_delivers_its_last_frames() { + const LAST: &[u8] = b"\x81\x05error\x88\x02\x03\xf0"; + let upstream = raw_upstream(|mut python: Raw| async move { + let handshake = python.request_head().await.unwrap(); + python.send(switching(&handshake, "")).await; + python.body(1).await; + tokio::time::sleep(Duration::from_millis(20)).await; + python.send(LAST).await; + }) + .await; + let gateway = Gateway::start(upstream).await; + for _ in 0..20 { + let (mut socket, _) = open(&gateway, handshake("/v1/realtime", "")).await; + let (mut read, mut write) = socket.split(); + let streaming = async { + while write.write_all(b"\x82\x80\0\0\0\0").await.is_ok() { + tokio::time::sleep(Duration::from_micros(200)).await; + } + std::future::pending().await + }; + let mut received = Vec::new(); + let receiving = async { + let mut chunk = [0; 4096]; + while let Ok(read @ 1..) = read.read(&mut chunk).await { + received.extend_from_slice(&chunk[..read]); + } + }; + within(async { + tokio::select! { + () = streaming => {} + () = receiving => {} + } + }) + .await; + assert_eq!(received, LAST); + } +} diff --git a/litellm-rust/crates/gateway-inference/tests/caching.rs b/litellm-rust/crates/gateway-inference/tests/caching.rs index 143bc1ba6db..030c8a66ee3 100644 --- a/litellm-rust/crates/gateway-inference/tests/caching.rs +++ b/litellm-rust/crates/gateway-inference/tests/caching.rs @@ -36,7 +36,12 @@ async fn all_inference_endpoints_share_native_cache( } else { "message_stop" }; - let events = format!("event: {terminal}\ndata: {{\"type\":\"{terminal}\"}}\n\n"); + let output = if is_responses { + "" + } else { + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"hello\"}}\n\n" + }; + let events = format!("{output}event: {terminal}\ndata: {{\"type\":\"{terminal}\"}}\n\n"); let template = if stream { ResponseTemplate::new(200).set_body_raw(events.clone(), "text/event-stream") } else { diff --git a/litellm-rust/crates/gateway/AGENTS.md b/litellm-rust/crates/gateway/AGENTS.md index a33bd19cc80..16714cd9a1f 100644 --- a/litellm-rust/crates/gateway/AGENTS.md +++ b/litellm-rust/crates/gateway/AGENTS.md @@ -5,8 +5,11 @@ - Test mounting and server lifecycle through public HTTP behavior. Test endpoint semantics in the owning crate ## Redis -- `build_redis` builds the Redis clients once at boot (`src/redis.rs`). Components get a clone as a field, never a per-request extractor or a pool. redis-rs multiplexes one connection per node, so there is no connection pool and pool settings like `max_connections` are accepted and ignored +- `build_redis` builds the Redis clients once at boot (`src/redis/mod.rs`). Components get a clone as a field, never a per-request extractor or a pool. redis-rs multiplexes one connection per node, so there is no connection pool and pool settings like `max_connections` are accepted and ignored - Which Redis backs which role mirrors the proxy: coordination is `general_settings.coordination_redis`, else the `cache_params` Redis when `cache: true` and `type: redis`, else `REDIS_*` env. The transaction buffer reuses coordination, else env when `use_redis_transaction_buffer` is on. The response cache gets its own client so large values never queue in front of coordination commands -- `litellm-config` follows `litellm/_redis.py`'s merge of config blocks and `REDIS_*` env. A redis-py setting we can't honor (sentinel, IAM, custom TLS files, ...) fails boot with its name, never silently dropped +- `src/redis/layers.rs` is the Rust copy of `_merge_redis_kwargs` in `litellm/_redis.py`, which only this host needs because the Python host merges in Python. It puts the configured kwargs over their `REDIS_*` values and fills the `get_secret` fallbacks from the `SETTINGS` rows in `redis-resolve`, under the same conditions Python reads them, then hands the result to `litellm_redis_resolve::interpret`. Values come from a `SecretSnapshot` filled once at boot, read like `get_secret` (secret manager, then environment, with `true` and `false` read as bools), and the only straight environment read is Azure's `REDIS_USERNAME` +- A surface is a `ConfiguredRedis` constructor: `coordination` for `general_settings.coordination_redis`, which rejects a malformed `port` like the proxy's `CoordinationRedisParams` pydantic model, `cache_params` for `litellm_settings.cache_params`, which drops the cache-only keys and reads the `redis_startup_nodes` alias, and `redis_cache_kwargs` for a `RedisCache.redis_kwargs` dict in the contract tests. A new surface is a new constructor, never a flag +- Nulls depend on the surface. `coordination_redis` drops every null like `model_dump(exclude_none=True)`, `cache_params` keeps a null as an explicit `None` that hides the `REDIS_*` value unless `Cache` or `RedisCache` names that parameter, and `RedisCache.redis_kwargs` keeps every null. A missing `os.environ/` reference is a null, as `load_config` resolves it +- A setting the Rust client cannot honor warns and is dropped, and only a security check it cannot enforce fails boot, per `redis-resolve/AGENTS.md` - Boot never waits for a configured Redis (clients connect lazily). Only the env-only coordination fallback is pinged, for 2s, and treated as absent when unreachable, as in Python - `/health/readiness` never touches Redis, matching the proxy diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 6eeaa43e013..a14df9b9c53 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -12,6 +12,7 @@ envy = "0.4.2" http-body-util = "0.1" litellm-inference.workspace = true litellm-gateway-inference.workspace = true +litellm-gateway-forward.workspace = true litellm-gateway-mcp.workspace = true tokio-util = "0.7" thiserror.workspace = true @@ -19,9 +20,14 @@ futures-util.workspace = true litellm-gateway-auth.workspace = true litellm-gateway-ui.workspace = true litellm-config.workspace = true +litellm-core-utils.workspace = true litellm-error.workspace = true litellm-circuit-breaker.workspace = true litellm-redis-client.workspace = true +litellm-redis-connect.workspace = true +litellm-redis-resolve.workspace = true +litellm-redis-types.workspace = true +rustls.workspace = true litellm-http.workspace = true litellm-llms.workspace = true litellm-secrets.workspace = true @@ -31,10 +37,12 @@ serde.workspace = true tracing.workspace = true tokio = { workspace = true, features = ["signal"] } uuid.workspace = true +veil.workspace = true tower-sessions-moka-store = "0.15.0" [dev-dependencies] litellm-redis-testing.workspace = true +litellm-http = { workspace = true, features = ["test-support"] } base64.workspace = true rstest.workspace = true tempfile.workspace = true diff --git a/litellm-rust/crates/gateway/src/error.rs b/litellm-rust/crates/gateway/src/error.rs index e2f646c5448..c267e79353d 100644 --- a/litellm-rust/crates/gateway/src/error.rs +++ b/litellm-rust/crates/gateway/src/error.rs @@ -12,13 +12,12 @@ pub enum Error { Mcp(#[from] litellm_gateway_mcp::ConnectError), #[error(transparent)] Config(#[from] litellm_config::Error), - #[error("invalid Redis setting {setting}: {reason}")] - RedisInvalid { - setting: &'static str, - reason: &'static str, - }, #[error(transparent)] - RedisConnect(#[from] litellm_redis_client::ConnectError), + Redis(#[from] litellm_redis_resolve::Error), + #[error("Redis setting {setting} is not plain data")] + RedisNotPlainData { setting: String }, + #[error(transparent)] + RedisConnect(#[from] litellm_redis_connect::Error), #[error("MCP requires working gateway authentication")] Auth(#[from] litellm_gateway_auth::Error), } diff --git a/litellm-rust/crates/gateway/src/lib.rs b/litellm-rust/crates/gateway/src/lib.rs index 12a9bf9110a..0a0191b0499 100644 --- a/litellm-rust/crates/gateway/src/lib.rs +++ b/litellm-rust/crates/gateway/src/lib.rs @@ -68,15 +68,29 @@ pub fn build_inference(config: &Config) -> Result, Error> { Ok(Arc::new(Gateway::new(resources, http, secrets, router)?)) } -pub async fn build_redis(config: &Config) -> Result { - let environment = secrets::environment_values(&config.environment_variables)?; - let lookup = |name: &str| { +pub fn install_crypto_provider() { + if rustls::crypto::CryptoProvider::get_default().is_none() { + let _ = rustls::crypto::ring::default_provider().install_default(); + } +} + +pub async fn build_redis( + config: &Config, + pool: &HttpClientPool, + http: &litellm_http::HttpClientConfig, +) -> Result { + let environment = Arc::new(secrets::environment_values(&config.environment_variables)?); + let lookup: litellm_redis_connect::SharedEnvironment = Arc::new(move |name: &str| { environment .get(name) .map(|value| value.expose().to_owned()) .or_else(|| std::env::var(name).ok()) + }); + let resources = redis::Resources { + http: pool.client(http, ClientVariant::Provider)?, + lookup: lookup.clone(), }; - redis::connect(config, &lookup).await + redis::connect(config, &*lookup, &resources).await } pub async fn build_mcp( diff --git a/litellm-rust/crates/gateway/src/main.rs b/litellm-rust/crates/gateway/src/main.rs index 47c1fe83f50..176bc68d247 100644 --- a/litellm-rust/crates/gateway/src/main.rs +++ b/litellm-rust/crates/gateway/src/main.rs @@ -10,7 +10,7 @@ use serde_json::json; mod settings; -use settings::{Settings, UiSettings}; +use settings::{LogSettings, Settings, UiSettings}; struct StderrSink { level: Level, @@ -41,12 +41,27 @@ impl Sink for StderrSink { #[tokio::main] async fn main() -> Result<(), Box> { + litellm_gateway::install_crypto_provider(); + if let Some(forward) = litellm_gateway_forward::Settings::from_env()? { + let log: LogSettings = settings::from_iter(std::env::vars_os())?; + Logger::new(StderrSink { + level: log.log_level(), + }) + .install_global()?; + let stdin_closed = litellm_gateway_forward::input_closed(std::io::stdin()); + litellm_gateway_forward::run(forward, async { + tokio::select! { () = shutdown_signal() => (), () = stdin_closed => () } + }) + .await?; + return Ok(()); + } let settings: Settings = settings::from_iter(std::env::vars_os())?; let level = settings.log_level(); Logger::new(StderrSink { level }).install_global()?; let config = Config::load(settings.litellm_config)?; let inference = litellm_gateway::build_inference(&config)?; - let _redis = litellm_gateway::build_redis(&config).await?; + let _redis = + litellm_gateway::build_redis(&config, &inference.resources.pool, &inference.http).await?; let ui = match std::env::var_os("LITELLM_UI_PATH") { Some(directory) => { let ui_settings: UiSettings = settings::from_iter(std::env::vars_os())?; @@ -126,7 +141,20 @@ async fn shutdown_signal() { } } } - #[cfg(not(unix))] + // Windows support exists for local CLI runs only, the proxy is never deployed on Windows. The supervisor + // stops the gateway with CTRL_BREAK_EVENT, which has to drain the same way SIGTERM does. + #[cfg(windows)] + { + match tokio::signal::windows::ctrl_break() { + Ok(mut ctrl_break) => { + tokio::select! { _ = tokio::signal::ctrl_c() => (), _ = ctrl_break.recv() => () } + } + Err(_) => { + let _ = tokio::signal::ctrl_c().await; + } + } + } + #[cfg(not(any(unix, windows)))] { let _ = tokio::signal::ctrl_c().await; } diff --git a/litellm-rust/crates/gateway/src/redis/configured.rs b/litellm-rust/crates/gateway/src/redis/configured.rs new file mode 100644 index 00000000000..93297d0e57f --- /dev/null +++ b/litellm-rust/crates/gateway/src/redis/configured.rs @@ -0,0 +1,144 @@ +use std::collections::BTreeMap; + +use litellm_config::{RedisSettings, Value}; +use litellm_core_utils::settings::Lookup; +use litellm_redis_resolve::{Expected, Kwargs}; +use litellm_redis_types::{ProviderHandle, Resolution}; +use veil::Redact; + +use super::{layers, secrets::SecretSnapshot}; +use crate::Error; + +const CACHE_ONLY: &[&str] = &[ + "type", + "mode", + "ttl", + "default_in_memory_ttl", + "default_in_redis_ttl", + "supported_call_types", + "similarity_threshold", + "max_messages", + "_backend", +]; + +/// The redis-py kwargs one surface hands `_get_redis_client_logic`, before `REDIS_*` and secrets. +#[derive(Clone, Redact)] +pub(crate) struct ConfiguredRedis { + #[redact(with = "[REDACTED]")] + pub(super) fields: BTreeMap, + pub(super) nulls: Nulls, + pub(super) credential_provider: Option, +} + +/// Whether a null setting reaches redis-py as an explicit `None`, which outranks its `REDIS_*` value. +#[derive(Clone, Copy, Debug)] +pub(super) enum Nulls { + /// `coordination_redis` reaches redis-py through `model_dump(exclude_none=True)`. + Dropped, + /// `Cache` drops a null only for the parameters it names and passes any other key on. + KeptUnlessNamedByCache, + /// `RedisCache.redis_kwargs` already holds only the nulls redis-py receives. + #[cfg(test)] + Kept, +} + +impl ConfiguredRedis { + /// `general_settings.coordination_redis`, which the proxy validates as a pydantic model first. + /// An `os.environ/` port is only known after merging, where it is read like any other port. + pub(crate) fn coordination( + settings: &RedisSettings, + ) -> Result { + if let Some(port) = settings.fields().get("port").and_then(text) + && !port.starts_with("os.environ/") + && port.parse::().is_err() + { + return Err(litellm_redis_resolve::Error::Invalid { + setting: "port".to_owned(), + expected: Expected::Integer, + }); + } + Ok(Self { + fields: settings.fields().clone(), + nulls: Nulls::Dropped, + credential_provider: None, + }) + } + + /// `litellm_settings.cache_params` with `type: redis`, as `litellm.Cache` hands it to `RedisCache`. + pub(crate) fn cache_params(settings: &RedisSettings) -> Self { + let fields = settings.fields(); + let alias = fields + .get("redis_startup_nodes") + .filter(|_| !fields.contains_key("startup_nodes")) + .map(|nodes| ("startup_nodes".to_owned(), nodes.clone())); + let fields = fields + .iter() + .filter(|(key, _)| { + key.as_str() != "redis_startup_nodes" && !CACHE_ONLY.contains(&key.as_str()) + }) + .map(|(key, value)| (key.clone(), value.clone())) + .chain(alias) + .collect(); + Self { + fields, + nulls: Nulls::KeptUnlessNamedByCache, + credential_provider: None, + } + } + + /// `RedisCache.redis_kwargs`, the dict every Python surface ends in. + #[cfg(test)] + pub(crate) fn redis_cache_kwargs(fields: BTreeMap) -> Self { + Self { + fields, + nulls: Nulls::Kept, + credential_provider: None, + } + } + + /// The handle the host holds for the `credential_provider` object in these settings. + #[cfg(test)] + pub(crate) fn with_credential_provider(self, handle: ProviderHandle) -> Self { + Self { + credential_provider: Some(handle), + ..self + } + } + + /// No configured settings, so only `REDIS_*` and the other environment fallbacks apply. + pub(crate) fn environment_only() -> Self { + Self { + fields: BTreeMap::new(), + nulls: Nulls::Dropped, + credential_provider: None, + } + } + + /// `_merge_redis_kwargs` in `litellm/_redis.py`: the kwargs over their `REDIS_*` values with every + /// `get_secret` fallback filled. + pub(crate) fn merged(&self, secrets: &SecretSnapshot) -> Result { + layers::merged(self, secrets) + } + + /// The environment is only for what Python reads straight from `os.environ`, `REDIS_USERNAME` for Azure. + pub(crate) fn resolve( + &self, + secrets: &SecretSnapshot, + environment: &dyn Lookup, + ) -> Result { + Ok(litellm_redis_resolve::interpret( + &self.merged(secrets)?, + self.credential_provider, + environment.get("REDIS_USERNAME").as_deref(), + )?) + } +} + +pub(super) fn text(value: &Value) -> Option { + match value { + Value::String(text) => Some(text.clone()), + Value::Number(number) => Some(number.to_string()), + Value::Bool(flag) => Some(flag.to_string()), + _ => None, + } +} diff --git a/litellm-rust/crates/gateway/src/redis/layers.rs b/litellm-rust/crates/gateway/src/redis/layers.rs new file mode 100644 index 00000000000..1cf68b2cd4f --- /dev/null +++ b/litellm-rust/crates/gateway/src/redis/layers.rs @@ -0,0 +1,226 @@ +use litellm_config::Value; +use litellm_core_utils::settings::Lookup; +use litellm_redis_resolve::{Entry, Kwargs, SETTINGS, is_present, setting}; + +use super::configured::{ConfiguredRedis, Nulls}; +use crate::Error; + +const NAMED_BY_CACHE: &[&str] = &[ + "host", + "port", + "password", + "socket_timeout", + "startup_nodes", + "namespace", + "gcp_service_account", + "gcp_ssl_ca_certs", +]; + +/// How `_merge_redis_kwargs` reads a fallback: `get_secret` turns `true` and `false` into a bool, and +/// `get_secret_str` treats that bool as unset. +#[derive(Clone, Copy)] +enum Read { + Secret, + SecretStr, +} + +const TARGETS: &[(&str, Read)] = &[ + ("startup_nodes", Read::Secret), + ("sentinel_nodes", Read::Secret), + ("sentinel_password", Read::SecretStr), + ("service_name", Read::Secret), +]; + +const IDENTITIES: &[(&str, Read)] = &[ + ("gcp_service_account", Read::SecretStr), + ("gcp_ssl_ca_certs", Read::SecretStr), + ("azure_redis_ad_token", Read::Secret), +]; + +const AZURE: &[(&str, Read)] = &[ + ("azure_client_id", Read::SecretStr), + ("azure_tenant_id", Read::SecretStr), + ("azure_client_secret", Read::SecretStr), +]; + +const AWS: &[(&str, Read)] = &[("aws_iam_region", Read::SecretStr)]; + +/// The configured settings over their `REDIS_*` values, `{**environment_kwargs, **env_overrides}` +/// in `_merge_redis_kwargs`, with the `get_secret` fallbacks it reads filled in. A null left in the result is +/// an explicit `None` that hid its environment value. +pub(super) fn merged(configured: &ConfiguredRedis, lookup: &dyn Lookup) -> Result { + let block: Kwargs = configured + .fields + .iter() + .map(|(key, value)| (key, environment_reference(value, lookup))) + .filter(|(key, value)| !value.is_null() || keeps_null(configured.nulls, key)) + .map(|(key, value)| { + let value = serde_json::to_value(&value).map_err(|_| Error::RedisNotPlainData { + setting: key.clone(), + })?; + let entry = Entry { + value, + source: key.clone(), + }; + Ok((key.clone(), entry)) + }) + .collect::>()?; + let names_a_target = ["host", "startup_nodes", "sentinel_nodes"] + .iter() + .any(|key| is_set(&block, key)); + let drop_environment_url = names_a_target && !is_set(&block, "url"); + let layered: Kwargs = environment(lookup) + .filter(|(key, _)| !(key == "url" && drop_environment_url)) + .chain(block) + .collect(); + let fallbacks = fallbacks(&layered, configured.credential_provider.is_some(), lookup); + Ok(layered.into_iter().chain(fallbacks).collect()) +} + +// The fallbacks are read under the same conditions as _merge_redis_kwargs, so a secret Python never asks +// for is never filled in +fn fallbacks(layered: &Kwargs, has_provider: bool, lookup: &dyn Lookup) -> Kwargs { + let fill = |kwargs: &[(&str, Read)]| -> Kwargs { + kwargs + .iter() + .filter_map(|(kwarg, read)| fallback(layered, kwarg, *read, lookup)) + .collect() + }; + let targets = fill(TARGETS); + if has_provider || is_set(layered, "credential_provider") { + return targets; + } + let identities = fill(IDENTITIES); + let value = |kwarg: &str| { + layered + .get(kwarg) + .filter(|entry| is_present(&entry.value)) + .or_else(|| identities.get(kwarg)) + .map(|entry| &entry.value) + }; + let gcp = value("gcp_service_account").is_some(); + let azure = value("azure_redis_ad_token") + .is_some_and(|token| text(token).is_some_and(|token| token.eq_ignore_ascii_case("true"))); + let aws = layered + .get("aws_iam_auth") + .is_some_and(|entry| coerces_to_true(&entry.value)); + let azure_settings = match azure && !gcp { + true => fill(AZURE), + false => Kwargs::new(), + }; + let aws_settings = match aws && !gcp && !azure { + true => fill(AWS), + false => Kwargs::new(), + }; + targets + .into_iter() + .chain(identities) + .chain(azure_settings) + .chain(aws_settings) + .collect() +} + +fn fallback( + layered: &Kwargs, + kwarg: &str, + read: Read, + lookup: &dyn Lookup, +) -> Option<(String, Entry)> { + if layered + .get(kwarg) + .is_some_and(|entry| is_present(&entry.value)) + { + return None; + } + let names = setting(kwarg).map_or(&[][..], |setting| setting.fallback); + names.iter().find_map(|name| { + let value = lookup + .get(name) + .filter(|value| !value.is_empty()) + .map(secret)?; + match (read, &value) { + (Read::SecretStr, serde_json::Value::Bool(_)) => None, + _ => { + let entry = Entry { + value, + source: (*name).to_owned(), + }; + Some((kwarg.to_owned(), entry)) + } + } + }) +} + +// get_secret reads an environment value of true or false, in any case and padding, as a bool +fn secret_flag(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "true" => Some(true), + "false" => Some(false), + _ => None, + } +} + +fn secret(value: String) -> serde_json::Value { + secret_flag(&value).map_or(serde_json::Value::String(value), serde_json::Value::Bool) +} + +// _coerces_to_true: text is read like _str_to_bool, anything else by its truthiness +fn coerces_to_true(value: &serde_json::Value) -> bool { + match value { + serde_json::Value::String(text) => { + matches!(text.to_ascii_lowercase().as_str(), "true" | "1" | "yes") + } + serde_json::Value::Null => false, + serde_json::Value::Bool(flag) => *flag, + serde_json::Value::Number(number) => number.as_f64().is_some_and(|number| number != 0.0), + serde_json::Value::Array(items) => !items.is_empty(), + serde_json::Value::Object(entries) => !entries.is_empty(), + } +} + +fn text(value: &serde_json::Value) -> Option { + match value { + serde_json::Value::String(text) => Some(text.clone()), + serde_json::Value::Number(number) => Some(number.to_string()), + serde_json::Value::Bool(flag) => Some(flag.to_string()), + _ => None, + } +} + +fn keeps_null(nulls: Nulls, key: &str) -> bool { + match nulls { + Nulls::Dropped => false, + Nulls::KeptUnlessNamedByCache => !NAMED_BY_CACHE.contains(&key), + #[cfg(test)] + Nulls::Kept => true, + } +} + +fn environment(lookup: &dyn Lookup) -> impl Iterator + '_ { + SETTINGS.iter().filter_map(|setting| { + let source = setting.environment_name()?; + let value = lookup.get(&source)?; + let entry = Entry { + value: secret(value), + source, + }; + Some((setting.kwarg.to_owned(), entry)) + }) +} + +// load_config resolves os.environ/ to None when unset, so a missing reference is a null like any other +fn environment_reference(value: &Value, lookup: &dyn Lookup) -> Value { + match value { + Value::String(text) => match text.strip_prefix("os.environ/") { + Some(name) => lookup.get(name).map_or(Value::Null, |value| { + secret_flag(&value).map_or(Value::String(value), Value::Bool) + }), + None => value.clone(), + }, + _ => value.clone(), + } +} + +fn is_set(kwargs: &Kwargs, key: &str) -> bool { + kwargs.get(key).is_some_and(|entry| !entry.value.is_null()) +} diff --git a/litellm-rust/crates/gateway/src/redis.rs b/litellm-rust/crates/gateway/src/redis/mod.rs similarity index 60% rename from litellm-rust/crates/gateway/src/redis.rs rename to litellm-rust/crates/gateway/src/redis/mod.rs index c40b289029b..91428a825f7 100644 --- a/litellm-rust/crates/gateway/src/redis.rs +++ b/litellm-rust/crates/gateway/src/redis/mod.rs @@ -1,19 +1,31 @@ -use std::time::Duration; +use std::{sync::Arc, time::Duration}; -use litellm_circuit_breaker::{Breaker, Config as BreakerConfig, State}; -use litellm_config::{ - CacheSettings, Config, Lookup, RedisEndpoint, RedisSettings, ResolvedRedis, Value, - environment_has_target, -}; -use litellm_redis_client::{ - ClusterEndpoint, ClusterNode, DEFAULT_TIMEOUT, Endpoint, HostEndpoint, Namespace, RedisClient, - RedisConfig, -}; +use litellm_circuit_breaker::Config as BreakerConfig; +use litellm_config::{CacheSettings, Config, RedisSettings, Value}; +use litellm_core_utils::settings::Lookup; +use litellm_redis_client::{RedisClient, RedisEvents}; +use litellm_redis_connect::{HostServices, RedisHosts, SharedEnvironment, TracingEvents}; +use litellm_redis_resolve::Expected; +use litellm_redis_types::Resolution; use crate::Error; +mod configured; +mod layers; +mod secrets; +#[cfg(test)] +mod surface_tests; + +use configured::ConfiguredRedis; +use secrets::SecretSnapshot; + +pub(crate) struct Resources { + pub(crate) http: litellm_http::Client, + pub(crate) lookup: SharedEnvironment, +} + const ENVIRONMENT_PING_TIMEOUT: Duration = Duration::from_secs(2); -type Params = ResolvedRedis; +type Params = Resolution; pub struct RedisClients { pub coordination: Option, @@ -34,12 +46,15 @@ enum Target { Environment(Params), } -pub(crate) fn plan(config: &Config, lookup: Lookup<'_>) -> Result { +pub(crate) fn plan(config: &Config, lookup: &dyn Lookup) -> Result { let response_cache = redis_cache_params(config) - .map(|settings| settings.resolve_cache(lookup)) + .map(|settings| resolve(&ConfiguredRedis::cache_params(settings), lookup)) .transpose()?; let coordination = match &config.general_settings.coordination_redis { - Some(settings) => Some(Target::Configured(settings.resolve(lookup)?)), + Some(settings) => Some(Target::Configured(resolve( + &ConfiguredRedis::coordination(settings)?, + lookup, + )?)), None => match (&response_cache, config.litellm_settings.cache == Some(true)) { (Some(cache), _) => Some(Target::Configured(cache.clone())), (None, true) => environment(lookup)?.map(Target::Configured), @@ -57,12 +72,30 @@ pub(crate) fn plan(config: &Config, lookup: Lookup<'_>) -> Result { }) } -pub async fn connect(config: &Config, lookup: Lookup<'_>) -> Result { +pub(crate) async fn connect( + config: &Config, + lookup: &dyn Lookup, + resources: &Resources, +) -> Result { let plan = plan(config, lookup)?; let breaker = breaker_config(lookup)?; + let events: Arc = Arc::new(TracingEvents::new(lookup)); + let hosts = RedisHosts::default(); + let client = |params: Params| -> Result { + Ok(hosts.client( + params, + HostServices { + events: Some(events.clone()), + http: resources.http.clone(), + environment: resources.lookup.clone(), + credential_provider: None, + }, + breaker, + )?) + }; let coordination = match plan.coordination { - Some(Target::Configured(params)) => Some(client(params, breaker)?), - Some(Target::Environment(params)) => match client(params, breaker) { + Some(Target::Configured(params)) => Some(client(params)?), + Some(Target::Environment(params)) => match client(params) { Ok(client) => reachable(client).await, Err(_) => { tracing::warn!( @@ -75,13 +108,10 @@ pub async fn connect(config: &Config, lookup: Lookup<'_>) -> Result Some(coordination.clone()), - (None, Some(params)) => Some(client(params, breaker)?), + (None, Some(params)) => Some(client(params)?), (None, None) => None, }; - let response_cache = plan - .response_cache - .map(|params| client(params, breaker)) - .transpose()?; + let response_cache = plan.response_cache.map(client).transpose()?; Ok(RedisClients { coordination, transaction_buffer, @@ -98,17 +128,38 @@ fn redis_cache_params(config: &Config) -> Option<&RedisSettings> { } } -fn environment(lookup: Lookup<'_>) -> Result, Error> { +fn resolve(configured: &ConfiguredRedis, lookup: &dyn Lookup) -> Result { + let secrets = SecretSnapshot::read(configured.secret_names(), lookup); + let resolution = configured.resolve(&secrets, lookup)?; + for ignored in &resolution.ignored { + tracing::warn!( + "Redis setting {} is not supported by the Rust client and is ignored: {}", + ignored.setting, + ignored.consequence + ); + } + Ok(resolution) +} + +fn environment(lookup: &dyn Lookup) -> Result, Error> { match environment_has_target(lookup) { - true => RedisSettings::default() - .resolve(lookup) - .map(Some) - .map_err(Into::into), + true => Ok(Some(resolve(&ConfiguredRedis::environment_only(), lookup)?)), false => Ok(None), } } -fn optional_environment(lookup: Lookup<'_>) -> Option { +fn environment_has_target(lookup: &dyn Lookup) -> bool { + [ + "REDIS_HOST", + "REDIS_URL", + "REDIS_CLUSTER_NODES", + "REDIS_SENTINEL_NODES", + ] + .iter() + .any(|name| lookup.get(name).is_some()) +} + +fn optional_environment(lookup: &dyn Lookup) -> Option { match environment(lookup) { Ok(params) => params, Err(error) => { @@ -149,122 +200,101 @@ async fn reachable(client: RedisClient) -> Option { } } -fn client(params: Params, breaker: Option) -> Result { - tracing::info!(endpoint = ?params.endpoint, namespace = ?params.namespace, "redis client configured"); - let endpoint = match params.endpoint { - RedisEndpoint::Url(url) => Endpoint::Url(url), - RedisEndpoint::Host(host) => Endpoint::Host(HostEndpoint { - host: host.host, - port: host.port, - username: host.username, - password: host.password, - tls: host.tls, - db: host.db, - }), - RedisEndpoint::Cluster(cluster) => Endpoint::Cluster(ClusterEndpoint { - nodes: cluster - .nodes - .into_iter() - .map(|node| ClusterNode { - host: node.host, - port: node.port, - }) - .collect(), - username: cluster.username, - password: cluster.password, - tls: cluster.tls, - }), - }; - Ok(RedisClient::new(RedisConfig { - endpoint, - namespace: params.namespace.map(Namespace::new).unwrap_or_default(), - timeout: params.timeout.unwrap_or(DEFAULT_TIMEOUT), - breaker: breaker.map(|config| Breaker::new(config, Box::new(log_transition))), - })?) +fn parse_seconds(text: &str) -> Option { + text.parse::() + .ok() + .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) } -fn log_transition(transition: &litellm_circuit_breaker::Transition) { - match transition.to { - State::Open { .. } => tracing::warn!( - generation = transition.generation, - "redis circuit breaker opened, fast-failing redis calls" - ), - State::HalfOpen => tracing::info!( - generation = transition.generation, - "redis circuit breaker half open, probing redis" - ), - State::Closed { .. } => tracing::info!( - generation = transition.generation, - "redis circuit breaker closed, redis recovered" - ), - } -} - -fn breaker_config(lookup: Lookup<'_>) -> Result, Error> { - let enabled = lookup("REDIS_CIRCUIT_BREAKER_ENABLED") +fn breaker_config(lookup: &dyn Lookup) -> Result, Error> { + let enabled = lookup + .get("REDIS_CIRCUIT_BREAKER_ENABLED") .is_none_or(|value| value.eq_ignore_ascii_case("true")); if !enabled { return Ok(None); } let defaults = BreakerConfig::default(); - let seconds = |name: &'static str, default: Duration| -> Result { - lookup(name) - .map(|value| { - value - .parse::() - .ok() - .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) - .ok_or(Error::RedisInvalid { - setting: name, - reason: "must be a non-negative number of seconds", - }) - }) - .unwrap_or(Ok(default)) - }; - let failure_threshold = lookup("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD") + Ok(Some(BreakerConfig { + failure_threshold: environment_value( + lookup, + "REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", + Expected::Integer, + |text| text.parse().ok(), + )? + .unwrap_or(defaults.failure_threshold), + recovery_timeout: environment_value( + lookup, + "REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", + Expected::Seconds, + parse_seconds, + )? + .unwrap_or(defaults.recovery_timeout), + timeout_min_duration: environment_value( + lookup, + "REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", + Expected::Seconds, + parse_seconds, + )? + .unwrap_or(defaults.timeout_min_duration), + })) +} + +fn environment_value( + lookup: &dyn Lookup, + name: &str, + expected: Expected, + parse: impl Fn(&str) -> Option, +) -> Result, litellm_redis_resolve::Error> { + lookup + .get(name) .map(|value| { - value.parse().map_err(|_| Error::RedisInvalid { - setting: "REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", - reason: "must be an integer", + parse(&value).ok_or_else(|| litellm_redis_resolve::Error::Invalid { + setting: name.to_owned(), + expected, }) }) - .unwrap_or(Ok(defaults.failure_threshold))?; - Ok(Some(BreakerConfig { - failure_threshold, - recovery_timeout: seconds( - "REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", - defaults.recovery_timeout, - )?, - timeout_min_duration: seconds( - "REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", - defaults.timeout_min_duration, - )?, - })) + .transpose() } #[cfg(test)] mod tests { + use litellm_core_utils::settings::Lookup; use std::{collections::HashMap, time::Duration}; - use litellm_config::{Config, RedisEndpoint, RedisHost}; + use litellm_config::Config; + use litellm_redis_client::{Standalone, StaticAuth, commands}; use litellm_redis_testing::RedisServer; + use litellm_redis_types::{Auth, HealthCheck}; use rstest::rstest; - use super::{Params, Plan, Target, connect, plan}; + use std::sync::Arc; + + use super::{Params, Plan, Resources, Target, plan}; use crate::Error; + async fn connect(config: &Config, lookup: &dyn Lookup) -> Result { + let resources = Resources { + http: litellm_http::Client::for_test(), + lookup: Arc::new(|_: &str| None), + }; + super::connect(config, lookup, &resources).await + } + fn host(name: &str, port: u16) -> Params { Params { - endpoint: RedisEndpoint::Host(RedisHost { + target: litellm_redis_client::Target::Standalone(Standalone { host: name.to_owned(), port, - username: None, - password: None, - tls: false, + tls: None, db: 0, }), + auth: Auth::Static(StaticAuth::default()), namespace: None, timeout: None, + connect_timeout: None, + health_check: HealthCheck::Unset, + keepalive: true, + ignored: Vec::new(), } } @@ -273,7 +303,7 @@ mod tests { .iter() .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) .collect(); - plan(&Config::from_yaml(yaml).unwrap(), &|name| { + plan(&Config::from_yaml(yaml).unwrap(), &|name: &str| { environment.get(name).cloned() }) } @@ -350,36 +380,44 @@ mod tests { port_of(&server) )) .unwrap(); - let clients = connect(&config, &|_| None).await.unwrap(); + let clients = connect(&config, &|_: &str| None).await.unwrap(); let coordination = clients.coordination.unwrap(); let buffer = clients.transaction_buffer.unwrap(); let cache = clients.response_cache.unwrap(); - let unnamespaced = connect(&coordination_config(&server), &|_| None) + let unnamespaced = connect(&coordination_config(&server), &|_: &str| None) .await .unwrap() .coordination .unwrap(); coordination - .set("rate-limit", "1", Duration::from_secs(60)) + .send(commands::set("rate-limit", "1", Duration::from_secs(60))) .await .unwrap(); buffer - .set("spend", "2", Duration::from_secs(60)) + .send(commands::set("spend", "2", Duration::from_secs(60))) .await .unwrap(); assert_eq!( - cache.get::("rate-limit").await.unwrap().as_deref(), + cache + .send(commands::get::("rate-limit")) + .await + .unwrap() + .as_deref(), Some("1") ); assert_eq!( - cache.get::("spend").await.unwrap().as_deref(), + cache + .send(commands::get::("spend")) + .await + .unwrap() + .as_deref(), Some("2") ); assert_eq!( unnamespaced - .get::("responses:rate-limit") + .send(commands::get::("responses:rate-limit")) .await .unwrap() .as_deref(), @@ -387,7 +425,7 @@ mod tests { ); assert_eq!( unnamespaced - .get::("responses:spend") + .send(commands::get::("responses:spend")) .await .unwrap() .as_deref(), @@ -395,16 +433,21 @@ mod tests { ); assert!( unnamespaced - .get::("rate-limit") + .send(commands::get::("rate-limit")) + .await + .unwrap() + .is_none() + ); + assert!( + unnamespaced + .send(commands::get::("spend")) .await .unwrap() .is_none() ); - assert!(unnamespaced.get::("spend").await.unwrap().is_none()); } #[rstest] - #[case::malformed_port(&[("REDIS_HOST", "env"), ("REDIS_PORT", "not-a-port")])] #[case::malformed_cluster_nodes(&[("REDIS_CLUSTER_NODES", "[{\"host\": ")])] fn redis_migration_regression_malformed_optional_environment_is_absent( #[case] environment: &[(&str, &str)], @@ -413,54 +456,45 @@ mod tests { } #[rstest] - fn redis_migration_regression_explicit_malformed_setting_still_fails() { + #[case::optional_environment("model_list: []", &[("REDIS_HOST", "env"), ("REDIS_PORT", "not-a-port")])] + #[case::transaction_buffer_environment( + "general_settings:\n use_redis_transaction_buffer: true", + &[("REDIS_HOST", "env"), ("REDIS_PORT", "not-a-port")] + )] + #[case::response_cache( + "litellm_settings:\n cache: true\n cache_params:\n type: redis\n host: env\n port: not-a-port", + &[] + )] + fn a_malformed_port_outside_the_coordination_block_falls_back_to_6379_like_redis_py( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + ) { + let plan = resolved(yaml, environment).unwrap(); + + let target = match (plan.coordination, plan.transaction_buffer_fallback) { + (Some(Target::Configured(params) | Target::Environment(params)), _) + | (None, Some(params)) => params, + (None, None) => panic!("expected a Redis target"), + }; + assert_eq!(target.target, host("env", 6379).target); + } + + #[rstest] + #[case::port("port: not-a-port", "port")] + #[case::ssl("ssl: maybe", "ssl")] + fn a_malformed_coordination_block_fails_like_the_pydantic_model( + #[case] setting: &str, + #[case] expected: &str, + ) { let error = resolved( - "general_settings:\n coordination_redis:\n host: coord\n port: not-a-port", + &format!("general_settings:\n coordination_redis:\n host: coord\n {setting}"), &[], ) .unwrap_err(); - assert!(matches!( - error, - Error::Config(litellm_config::Error::RedisInvalid { - setting: "port", - .. - }) - )); - } - - #[rstest] - fn redis_migration_regression_enabled_transaction_buffer_rejects_invalid_environment() { - let error = resolved( - "general_settings:\n use_redis_transaction_buffer: true", - &[("REDIS_HOST", "env"), ("REDIS_PORT", "not-a-port")], - ) - .unwrap_err(); - - assert!(matches!( - error, - Error::Config(litellm_config::Error::RedisInvalid { - setting: "port", - .. - }) - )); - } - - #[rstest] - fn redis_migration_regression_invalid_response_cache_still_fails() { - let error = resolved( - "litellm_settings:\n cache: true\n cache_params:\n type: redis\n host: cache\n port: not-a-port", - &[], - ) - .unwrap_err(); - - assert!(matches!( - error, - Error::Config(litellm_config::Error::RedisInvalid { - setting: "port", - .. - }) - )); + assert!( + matches!(error, Error::Redis(litellm_redis_resolve::Error::Invalid { setting, .. }) if setting == expected) + ); } #[rstest] @@ -468,7 +502,7 @@ mod tests { async fn redis_migration_regression_invalid_optional_environment_url_is_absent() { let config = Config::from_yaml("model_list: []").unwrap(); - let clients = connect(&config, &|name| { + let clients = connect(&config, &|name: &str| { (name == "REDIS_URL").then(|| "not a redis URL".to_owned()) }) .await @@ -506,7 +540,7 @@ mod tests { let server = RedisServer::start().await.unwrap(); server.stop().await.unwrap(); - let clients = connect(&coordination_config(&server), &|_| None) + let clients = connect(&coordination_config(&server), &|_: &str| None) .await .unwrap(); let coordination = clients.coordination.unwrap(); @@ -521,12 +555,12 @@ mod tests { #[tokio::test] async fn the_coordination_client_survives_a_redis_restart_and_backs_the_transaction_buffer() { let server = RedisServer::start().await.unwrap(); - let clients = connect(&coordination_config(&server), &|_| None) + let clients = connect(&coordination_config(&server), &|_: &str| None) .await .unwrap(); let coordination = clients.coordination.unwrap(); coordination - .set("written", "1", Duration::from_secs(60)) + .send(commands::set("written", "1", Duration::from_secs(60))) .await .unwrap(); @@ -537,6 +571,30 @@ mod tests { assert!(buffer.ping().await.is_ok()); } + #[rstest] + #[tokio::test] + async fn an_environment_only_tls_redis_is_reached_with_its_ca() { + crate::install_crypto_provider(); + let server = litellm_redis_testing::RedisTlsServer::start(false) + .await + .unwrap(); + let ca = server.files().ca.display().to_string(); + let port = server.port().to_string(); + let config = Config::from_yaml("model_list: []").unwrap(); + + let clients = connect(&config, &|name: &str| match name { + "REDIS_HOST" => Some(server.host().to_owned()), + "REDIS_PORT" => Some(port.clone()), + "REDIS_SSL" => Some("true".to_owned()), + "REDIS_SSL_CA_CERTS" => Some(ca.clone()), + _ => None, + }) + .await + .unwrap(); + + assert!(clients.coordination.is_some()); + } + #[rstest] #[tokio::test] async fn an_unreachable_environment_only_redis_leaves_coordination_in_memory() { @@ -545,7 +603,7 @@ mod tests { server.stop().await.unwrap(); let config = Config::from_yaml("model_list: []").unwrap(); - let clients = connect(&config, &|name| match name { + let clients = connect(&config, &|name: &str| match name { "REDIS_HOST" => Some("127.0.0.1".to_owned()), "REDIS_PORT" => Some(port.clone()), _ => None, @@ -564,7 +622,7 @@ mod tests { let port = port_of(&server); let config = Config::from_yaml("model_list: []").unwrap(); - let clients = connect(&config, &|name| match name { + let clients = connect(&config, &|name: &str| match name { "REDIS_HOST" => Some("127.0.0.1".to_owned()), "REDIS_PORT" => Some(port.clone()), _ => None, diff --git a/litellm-rust/crates/gateway/src/redis/secrets.rs b/litellm-rust/crates/gateway/src/redis/secrets.rs new file mode 100644 index 00000000000..a969497f4c3 --- /dev/null +++ b/litellm-rust/crates/gateway/src/redis/secrets.rs @@ -0,0 +1,61 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use litellm_core_utils::settings::Lookup; +use litellm_secret_value::SecretValue; + +use litellm_config::Value; +use litellm_redis_resolve::SETTINGS; + +use super::configured::ConfiguredRedis; + +#[derive(Clone, Debug, Default)] +pub(crate) struct SecretSnapshot(BTreeMap>); + +impl SecretSnapshot { + pub(crate) fn read(names: BTreeSet, lookup: &dyn Lookup) -> Self { + Self( + names + .into_iter() + .map(|name| { + let value = lookup.get(&name).map(SecretValue::new); + (name, value) + }) + .collect(), + ) + } +} + +impl Lookup for SecretSnapshot { + fn get(&self, name: &str) -> Option { + match self.0.get(name) { + Some(value) => value.as_ref().map(|value| value.expose().to_owned()), + None => { + debug_assert!( + false, + "{name} is missing from ConfiguredRedis::secret_names" + ); + tracing::error!( + name, + "a Redis secret was read without being in the snapshot" + ); + None + } + } + } +} + +impl ConfiguredRedis { + pub(crate) fn secret_names(&self) -> BTreeSet { + let references = self.fields.values().filter_map(|value| match value { + Value::String(text) => text.strip_prefix("os.environ/").map(str::to_owned), + _ => None, + }); + let declared = SETTINGS.iter().flat_map(|setting| { + setting + .environment_name() + .into_iter() + .chain(setting.fallback.iter().map(|name| (*name).to_owned())) + }); + declared.chain(references).collect() + } +} diff --git a/litellm-rust/crates/gateway/src/redis/surface_tests.rs b/litellm-rust/crates/gateway/src/redis/surface_tests.rs new file mode 100644 index 00000000000..9ee881e2e04 --- /dev/null +++ b/litellm-rust/crates/gateway/src/redis/surface_tests.rs @@ -0,0 +1,964 @@ +use std::collections::{BTreeMap, HashMap}; + +use litellm_config::{CacheSettings, Config, Value}; +use litellm_redis_resolve::Expected; +use litellm_redis_types::{ + Auth, AwsIamAuth, AzureAdAuth, Node, ProviderHandle, Resolution, StaticAuth, Target, TlsOptions, +}; +use litellm_secret_value::SecretValue; +use rstest::rstest; +use serde_json::json; + +use super::{configured::ConfiguredRedis, secrets::SecretSnapshot}; +use crate::Error; + +fn resolution( + configured: &ConfiguredRedis, + environment: &HashMap, +) -> Result { + let lookup = |name: &str| environment.get(name).cloned(); + configured.resolve( + &SecretSnapshot::read(configured.secret_names(), &lookup), + &lookup, + ) +} + +const PROVIDER: ProviderHandle = ProviderHandle::new(7); + +fn redis_cache_kwargs(kwargs: serde_json::Value) -> ConfiguredRedis { + let has_provider = kwargs + .get("credential_provider") + .is_some_and(|provider| !provider.is_null()); + let configured = ConfiguredRedis::redis_cache_kwargs( + serde_json::from_value::>(kwargs).unwrap(), + ); + match has_provider { + true => configured.with_credential_provider(PROVIDER), + false => configured, + } +} + +fn coordination_resolution(yaml: &str, environment: &[(&str, &str)]) -> Result { + let environment: HashMap = environment + .iter() + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect(); + let config = Config::from_yaml(yaml).unwrap(); + let configured = config + .general_settings + .coordination_redis + .unwrap_or_default(); + let configured = ConfiguredRedis::coordination(&configured)?; + resolution(&configured, &environment) +} + +fn resolve(yaml: &str, environment: &[(&str, &str)]) -> Result { + coordination_resolution(yaml, environment) +} + +fn resolve_cache(yaml: &str) -> Result { + resolve_cache_in(yaml, &[]) +} + +fn resolve_cache_in(yaml: &str, environment: &[(&str, &str)]) -> Result { + let lookup = |name: &str| { + environment + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| (*value).to_owned()) + }; + let config = Config::from_yaml(yaml).unwrap(); + let configured = match config.litellm_settings.cache_params.unwrap() { + CacheSettings::Redis(settings) => ConfiguredRedis::cache_params(&settings), + CacheSettings::Other(_) => panic!("expected a Redis cache"), + }; + configured.resolve( + &SecretSnapshot::read(configured.secret_names(), &lookup), + &lookup, + ) +} + +fn cluster_ports(resolved: &Resolution) -> Vec { + match &resolved.target { + Target::Cluster(cluster) => cluster.nodes.iter().map(|node| node.port).collect(), + target => panic!("expected a cluster, got {target:?}"), + } +} + +#[rstest] +#[case::host( + "general_settings:\n coordination_redis:\n host: redis\n port: 6380\n db: 2\n ssl: true\n namespace: app\n socket_timeout: 1.5", + &[], + "host:redis:6380:2:true:app:1500" +)] +#[case::environment_references_resolve_the_host_and_port( + "general_settings:\n coordination_redis:\n host: os.environ/COORD_HOST\n port: os.environ/COORD_PORT", + &[("COORD_HOST", "referenced"), ("COORD_PORT", "6400")], + "host:referenced:6400:0:false::none" +)] +#[case::a_null_host_falls_back_to_the_environment( + "general_settings:\n coordination_redis:\n host: null", + &[("REDIS_HOST", "environment")], + "host:environment:6379:0:false::none" +)] +fn resolves_redis_through_the_parsed_config( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] expected: &str, +) { + let resolved = resolve(yaml, environment).unwrap(); + let description = match resolved.target { + Target::Unix(socket) => format!("unix:{}", socket.path.display()), + Target::Url(url) => format!("url:{}", url.url.expose()), + Target::Sentinel(sentinel) => format!("sentinel:{}", sentinel.service_name), + Target::Standalone(host) => format!( + "host:{}:{}:{}:{}:{}:{}", + host.host, + host.port, + host.db, + host.tls.is_some(), + resolved.namespace.unwrap_or_default(), + resolved + .timeout + .map(|timeout| timeout.as_millis().to_string()) + .unwrap_or_else(|| "none".to_owned()) + ), + Target::Cluster(cluster) => format!( + "cluster:{}", + cluster + .nodes + .iter() + .map(|node| format!("{}:{}", node.host, node.port)) + .collect::>() + .join(",") + ), + }; + + assert_eq!(description, expected); +} + +#[rstest] +#[case::tls_ciphers( + "general_settings:\n coordination_redis:\n host: redis\n ssl_ciphers: HIGH", + &[], + "ssl_ciphers" +)] +#[case::ssl_ciphers_from_environment( + "model_list: []", + &[("REDIS_HOST", "redis"), ("REDIS_SSL_CIPHERS", "HIGH")], + "REDIS_SSL_CIPHERS" +)] +#[case::connection_pool( + "general_settings:\n coordination_redis:\n host: redis\n connection_pool: shared", + &[], + "connection_pool" +)] +fn unsupported_settings_resolve_and_are_named_by_their_source( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] expected: &str, +) { + let resolution = coordination_resolution(yaml, environment).unwrap(); + + assert_eq!( + resolution.target, + Target::Standalone(litellm_redis_types::Standalone { + host: "redis".into(), + port: 6379, + db: 0, + tls: None, + }) + ); + let [ignored] = resolution.ignored.as_slice() else { + panic!("expected one ignored setting, got {:?}", resolution.ignored); + }; + assert_eq!(ignored.setting, expected); + assert!(!ignored.consequence.is_empty()); +} + +#[rstest] +#[case::kwarg( + "general_settings:\n coordination_redis:\n host: redis\n ssl: true\n ssl_validate_ocsp: true", + &[], + "ssl_validate_ocsp" +)] +#[case::stapled_from_environment( + "model_list: []", + &[("REDIS_HOST", "redis"), ("REDIS_SSL", "true"), ("REDIS_SSL_VALIDATE_OCSP_STAPLED", "yes")], + "REDIS_SSL_VALIDATE_OCSP_STAPLED" +)] +#[case::rediss_url_query( + "general_settings:\n coordination_redis:\n url: rediss://cache:6380/0?ssl_validate_ocsp=true", + &[], + "url ?ssl_validate_ocsp" +)] +fn an_ocsp_check_fails_instead_of_connecting_without_it( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] setting: &str, +) { + let error = coordination_resolution(yaml, environment).unwrap_err(); + + assert!( + matches!(&error, Error::Redis(litellm_redis_resolve::Error::Unenforceable { setting: named, .. }) if named == setting), + "{error:?}" + ); +} + +#[rstest] +#[case::kwarg(" host: redis\n ssl: true\n ssl_validate_ocsp: false")] +#[case::rediss_url_query(" url: rediss://cache:6380/0?ssl_validate_ocsp=false")] +fn an_ocsp_flag_set_to_false_asks_for_nothing(#[case] block: &str) { + let yaml = format!("general_settings:\n coordination_redis:\n{block}"); + + let resolution = coordination_resolution(&yaml, &[]).unwrap(); + + assert!(resolution.ignored.is_empty(), "{:?}", resolution.ignored); +} + +fn static_auth(username: Option<&str>, password: Option<&str>) -> Auth { + Auth::Static(StaticAuth { + username: username.map(litellm_secret_value::SecretValue::new), + password: password.map(litellm_secret_value::SecretValue::new), + }) +} + +#[rstest] +fn an_explicit_host_keeps_environment_credentials() { + let resolved = resolve( + "general_settings:\n coordination_redis:\n host: configured", + &[ + ("REDIS_PASSWORD", "environment-secret"), + ("REDIS_SSL", "True"), + ], + ) + .unwrap(); + + assert_eq!(resolved.auth, static_auth(None, Some("environment-secret"))); + assert!(matches!(resolved.target, Target::Standalone(host) if host.tls.is_some())); +} + +fn azure(username: Option<&str>) -> Auth { + Auth::AzureAd(AzureAdAuth { + client_id: Some("client".to_owned()), + tenant_id: None, + client_secret: None, + username: username.map(str::to_owned), + }) +} + +fn aws(serverless: bool) -> Auth { + Auth::AwsIam(AwsIamAuth { + user_name: "iam-user".to_owned(), + cache_name: "cache-1".to_owned(), + region: "us-east-1".to_owned(), + serverless, + }) +} + +#[rstest] +#[case::static_credentials( + "general_settings:\n coordination_redis:\n host: redis\n username: user\n password: pass", + &[], + static_auth(Some("user"), Some("pass")) +)] +#[case::gcp_beats_azure_and_aws( + "general_settings:\n coordination_redis:\n host: redis\n password: pass\n azure_redis_ad_token: \"True\"\n aws_iam_auth: true", + &[("REDIS_GCP_SERVICE_ACCOUNT", "projects/-/serviceAccounts/sa@p.iam.gserviceaccount.com")], + Auth::GcpIam { service_account: "projects/-/serviceAccounts/sa@p.iam.gserviceaccount.com".to_owned() } +)] +#[case::azure_beats_aws_and_takes_the_environment_username( + "general_settings:\n coordination_redis:\n host: redis\n username: configured-user\n azure_redis_ad_token: true\n azure_client_id: client\n aws_iam_auth: true", + &[("REDIS_USERNAME", "environment-user")], + azure(Some("environment-user")) +)] +#[case::azure_without_an_environment_username_sends_only_the_token( + "general_settings:\n coordination_redis:\n host: redis\n username: configured-user\n azure_redis_ad_token: true\n azure_client_id: client", + &[], + azure(None) +)] +#[case::azure_needs_the_literal_true( + "general_settings:\n coordination_redis:\n host: redis\n azure_redis_ad_token: \"1\"", + &[], + static_auth(None, None) +)] +#[case::aws_with_a_region_from_the_environment( + "general_settings:\n coordination_redis:\n host: redis\n ssl: true\n password: dropped\n aws_iam_auth: \"yes\"\n aws_iam_user_name: iam-user\n aws_iam_cache_name: cache-1", + &[("AWS_DEFAULT_REGION", "us-east-1")], + aws(false) +)] +#[case::aws_serverless_over_a_rediss_url( + "general_settings:\n coordination_redis:\n url: rediss://cache-1:6379\n aws_iam_auth: true\n aws_iam_user_name: iam-user\n aws_iam_cache_name: cache-1\n aws_iam_region: us-east-1\n aws_iam_serverless: true", + &[], + aws(true) +)] +fn auth_follows_python_precedence( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] expected: Auth, +) { + assert_eq!(resolve(yaml, environment).unwrap().auth, expected); +} + +#[rstest] +#[case::coordination( + "general_settings:\n coordination_redis:\n host: redis\n credential_provider: host-object", + &[] +)] +#[case::without_a_target( + "general_settings:\n coordination_redis:\n credential_provider: host-object", + &[("REDIS_HOST", "redis")] +)] +fn a_credential_provider_without_a_host_handle_fails( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], +) { + let error = resolve(yaml, environment).unwrap_err(); + + assert!( + matches!( + error, + Error::Redis(litellm_redis_resolve::Error::ProviderWithoutHandle) + ), + "{error}" + ); +} + +#[rstest] +#[case::without_tls( + "general_settings:\n coordination_redis:\n host: redis\n aws_iam_auth: true\n aws_iam_user_name: u\n aws_iam_cache_name: c\n aws_iam_region: r", + "requires TLS" +)] +#[case::a_plain_url_ignores_ssl( + "general_settings:\n coordination_redis:\n url: redis://cache:6379\n ssl: true\n aws_iam_auth: true\n aws_iam_user_name: u\n aws_iam_cache_name: c\n aws_iam_region: r", + "requires TLS" +)] +#[case::missing_settings( + "general_settings:\n coordination_redis:\n host: redis\n ssl: true\n aws_iam_auth: true\n aws_iam_cache_name: c", + "requires: aws_iam_user_name, aws_iam_region" +)] +fn incomplete_aws_iam_auth_fails(#[case] yaml: &str, #[case] message: &str) { + let error = resolve(yaml, &[]).unwrap_err(); + + assert!(error.to_string().contains(message), "{error}"); +} + +#[rstest] +#[case::defaults_verify_the_certificate_but_not_the_hostname( + " ssl: true", + &[], + Some(TlsOptions { ca_certs: None, ca_data: None, ca_path: None, certfile: None, keyfile: None, key_password: None, verify_certificate: true, check_hostname: false }) +)] +#[case::every_option( + " ssl: true\n ssl_ca_certs: /ca.pem\n ssl_certfile: /client.pem\n ssl_keyfile: /client.key\n ssl_password: key-secret\n ssl_cert_reqs: none\n ssl_check_hostname: true", + &[], + Some(TlsOptions { ca_certs: Some("/ca.pem".into()), ca_data: None, ca_path: None, certfile: Some("/client.pem".into()), keyfile: Some("/client.key".into()), key_password: Some(SecretValue::new("key-secret")), verify_certificate: false, check_hostname: true }) +)] +#[case::options_from_the_environment( + " ssl: true", + &[("REDIS_SSL_CA_CERTS", "/env-ca.pem"), ("REDIS_SSL_CERT_REQS", "optional"), ("REDIS_SSL_PASSWORD", "env-key-secret")], + Some(TlsOptions { ca_certs: Some("/env-ca.pem".into()), ca_data: None, ca_path: None, certfile: None, keyfile: None, key_password: Some(SecretValue::new("env-key-secret")), verify_certificate: true, check_hostname: false }) +)] +#[case::gcp_ca_certs_replace_the_configured_ones( + " ssl: true\n ssl_ca_certs: /ca.pem\n gcp_service_account: account\n gcp_ssl_ca_certs: /gcp-ca.pem", + &[], + Some(TlsOptions { ca_certs: Some("/gcp-ca.pem".into()), ca_data: None, ca_path: None, certfile: None, keyfile: None, key_password: None, verify_certificate: true, check_hostname: false }) +)] +#[case::options_without_ssl_stay_plaintext(" ssl_ca_certs: /ca.pem", &[], None)] +fn tls_options_resolve_like_redis_py( + #[case] block: &str, + #[case] environment: &[(&str, &str)], + #[case] expected: Option, +) { + let yaml = format!("general_settings:\n coordination_redis:\n host: redis\n{block}"); + + let resolved = resolve(&yaml, environment).unwrap(); + + assert!(matches!(resolved.target, Target::Standalone(host) if host.tls == expected)); +} + +#[rstest] +fn an_unknown_cert_requirement_fails() { + let error = resolve( + "general_settings:\n coordination_redis:\n host: redis\n ssl: true\n ssl_cert_reqs: strict", + &[], + ) + .unwrap_err(); + + assert!(matches!( + error, + Error::Redis(litellm_redis_resolve::Error::Invalid { setting, expected: Expected::CertRequirement }) if setting == "ssl_cert_reqs" + )); +} + +#[rstest] +#[case::from_config( + "general_settings:\n coordination_redis:\n sentinel_nodes: [[sentinel-a, 26379], [sentinel-b, \"26380\"]]\n service_name: primary\n sentinel_password: sentinel-secret\n db: 2", + &[] +)] +#[case::from_the_environment( + "model_list: []", + &[ + ("REDIS_SENTINEL_NODES", r#"[["sentinel-a", 26379], ["sentinel-b", "26380"]]"#), + ("REDIS_SERVICE_NAME", "primary"), + ("REDIS_SENTINEL_PASSWORD", "sentinel-secret"), + ("REDIS_DB", "2"), + ] +)] +fn sentinel_nodes_resolve_to_a_sentinel_endpoint( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], +) { + let Target::Sentinel(sentinel) = resolve(yaml, environment).unwrap().target else { + panic!("expected a sentinel endpoint"); + }; + + assert_eq!( + sentinel.monitors, + vec![ + Node { + host: "sentinel-a".to_owned(), + port: 26379 + }, + Node { + host: "sentinel-b".to_owned(), + port: 26380 + }, + ] + ); + assert_eq!(sentinel.service_name, "primary"); + assert_eq!( + sentinel.monitor_auth, + Some(StaticAuth { + username: None, + password: Some(litellm_secret_value::SecretValue::new("sentinel-secret")), + }) + ); + assert_eq!(sentinel.db, 2); +} + +#[rstest] +#[case::a_url_beats_sentinels( + "general_settings:\n coordination_redis:\n url: redis://single:6379\n sentinel_nodes: [[s, 26379]]\n service_name: primary", + "url:redis://single:6379" +)] +fn sentinel_precedence(#[case] yaml: &str, #[case] expected: &str) { + let resolved = resolve(yaml, &[]).unwrap(); + let description = match resolved.target { + Target::Url(url) => format!("url:{}", url.url.expose()), + Target::Sentinel(sentinel) => format!("sentinel:{}", sentinel.service_name), + other => panic!("unexpected {other:?}"), + }; + + assert_eq!(description, expected); +} + +#[rstest] +fn sentinels_without_a_service_name_fail() { + let error = resolve( + "general_settings:\n coordination_redis:\n sentinel_nodes: [[s, 26379]]", + &[], + ) + .unwrap_err(); + + assert!( + matches!( + error, + Error::Redis(litellm_redis_resolve::Error::NoSentinelService) + ), + "{error:?}" + ); +} + +#[rstest] +#[case::configured_host( + "general_settings:\n coordination_redis:\n host: ignored\n sentinel_nodes: [[s, 26379]]\n service_name: primary", + &[], + "host" +)] +#[case::environment_port( + "general_settings:\n coordination_redis:\n sentinel_nodes: [[s, 26379]]\n service_name: primary", + &[("REDIS_PORT", "6380")], + "REDIS_PORT" +)] +fn a_host_or_port_cannot_reach_sentinel( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] source: &str, +) { + let error = resolve(yaml, environment).unwrap_err(); + + assert!( + matches!(&error, Error::Redis(litellm_redis_resolve::Error::SentinelWithHost { setting }) if setting == source), + "{error:?}" + ); +} + +#[rstest] +#[case::not_pairs( + "general_settings:\n coordination_redis:\n sentinel_nodes: [{host: s, port: 26379}]\n service_name: primary", + &[], + "sentinel_nodes" +)] +#[case::from_the_environment( + "model_list: []", + &[("REDIS_SENTINEL_NODES", "not-json"), ("REDIS_SERVICE_NAME", "primary")], + "REDIS_SENTINEL_NODES" +)] +fn invalid_sentinel_nodes_name_their_source( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] source: &str, +) { + let error = resolve(yaml, environment).unwrap_err(); + + assert!( + matches!(&error, Error::Redis(litellm_redis_resolve::Error::Invalid { setting, expected: Expected::SentinelNodes }) if setting == source), + "{error:?}" + ); +} + +#[rstest] +#[case::from_config("general_settings:\n coordination_redis:\n startup_nodes: [{host: a}]", &[], "startup_nodes")] +#[case::from_the_environment("model_list: []", &[("REDIS_CLUSTER_NODES", "not-json")], "REDIS_CLUSTER_NODES")] +#[case::out_of_range_port_from_the_environment("model_list: []", &[("REDIS_HOST", "redis"), ("REDIS_PORT", "70000")], "REDIS_PORT")] +fn invalid_values_name_their_source( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] source: &str, +) { + let error = resolve(yaml, environment).unwrap_err(); + + assert!( + matches!(&error, Error::Redis(litellm_redis_resolve::Error::Invalid { setting, .. }) if setting == source), + "{error:?}" + ); +} + +#[rstest] +#[case::falls_back_to_the_socket_timeout(" socket_timeout: 1.5", Some(1500), None)] +#[case::explicit_connect_timeout( + " socket_timeout: 1.5\n socket_connect_timeout: 0.25", + Some(1500), + Some(250) +)] +fn connect_timeout_is_resolved_separately( + #[case] block: &str, + #[case] timeout_ms: Option, + #[case] connect_ms: Option, +) { + let resolved = resolve( + &format!("general_settings:\n coordination_redis:\n host: redis\n{block}"), + &[], + ) + .unwrap(); + + assert_eq!( + resolved.timeout.map(|timeout| timeout.as_millis()), + timeout_ms + ); + assert_eq!( + resolved.connect_timeout.map(|timeout| timeout.as_millis()), + connect_ms + ); +} + +#[rstest] +#[case::only_a_port("general_settings:\n coordination_redis:\n port: 6379", &[])] +#[case::a_structured_host_with_an_environment_target( + "general_settings:\n coordination_redis:\n host: []", + &[("REDIS_HOST", "environment"), ("REDIS_URL", "redis://environment:6379")] +)] +fn a_block_without_a_usable_target_fails(#[case] yaml: &str, #[case] environment: &[(&str, &str)]) { + let error = resolve(yaml, environment).unwrap_err(); + + assert!( + matches!(error, Error::Redis(litellm_redis_resolve::Error::NoTarget)), + "{error:?}" + ); +} + +#[rstest] +fn the_cache_alias_names_cluster_nodes() { + let resolved = resolve_cache( + "litellm_settings:\n cache_params:\n type: redis\n redis_startup_nodes:\n - host: node-a\n port: 7000", + ) + .unwrap(); + + assert_eq!(cluster_ports(&resolved), vec![7000]); +} + +#[rstest] +fn canonical_startup_nodes_beat_the_cache_alias() { + let resolved = resolve_cache( + "litellm_settings:\n cache_params:\n type: redis\n startup_nodes:\n - host: canonical\n port: 7000\n redis_startup_nodes:\n - host: alias\n port: 7100\n - host: alias\n port: 7101", + ) + .unwrap(); + + assert_eq!(cluster_ports(&resolved), vec![7000]); +} + +#[rstest] +fn a_null_canonical_startup_nodes_setting_shadows_the_cache_alias() { + let resolved = resolve_cache( + "litellm_settings:\n cache_params:\n type: redis\n host: cache\n startup_nodes: null\n redis_startup_nodes:\n - host: alias\n port: 7100", + ) + .unwrap(); + + assert!(matches!(resolved.target, Target::Standalone(_))); +} + +#[rstest] +fn a_missing_canonical_startup_nodes_reference_shadows_the_cache_alias() { + let resolved = resolve_cache( + "litellm_settings:\n cache_params:\n type: redis\n host: cache\n startup_nodes: os.environ/MISSING_NODES\n redis_startup_nodes:\n - host: alias\n port: 7100", + ) + .unwrap(); + + assert!(matches!(resolved.target, Target::Standalone(_))); +} + +#[rstest] +fn unsupported_environment_settings_never_carry_the_value() { + let resolution = coordination_resolution( + "model_list: []", + &[ + ("REDIS_HOST", "redis"), + ("REDIS_SSL_CIPHERS", "private-option-value"), + ], + ) + .unwrap(); + let ignored = format!("{:?}", resolution.ignored); + + assert!(ignored.contains("REDIS_SSL_CIPHERS"), "{ignored}"); + assert!(!ignored.contains("private-option-value"), "{ignored}"); +} + +#[rstest] +#[case::negative("-1")] +#[case::infinite("inf")] +fn rejects_bad_socket_timeout(#[case] timeout: &str) { + let yaml = format!( + "general_settings:\n coordination_redis:\n host: redis\n socket_timeout: {timeout}" + ); + let error = resolve(&yaml, &[]).unwrap_err(); + + assert!(matches!( + error, + Error::Redis(litellm_redis_resolve::Error::Invalid { setting, expected: Expected::Seconds }) if setting == "socket_timeout" + )); +} + +#[rstest] +fn debug_redacts_redis_secrets_and_url() { + let config = Config::from_yaml( + "general_settings:\n coordination_redis:\n url: redis://user:password@redis:6379\n username: credential-user\n password: credential-password\n sentinel_kwargs: {password: credential-passthrough}", + ) + .unwrap(); + let settings = config.general_settings.coordination_redis.unwrap(); + let configured = ConfiguredRedis::coordination(&settings).unwrap(); + let resolved = resolution(&configured, &HashMap::new()).unwrap(); + let debug = format!("{settings:?} {configured:?} {resolved:?}"); + + assert!(!debug.contains("redis://user:password")); + assert!(!debug.contains("credential-user")); + assert!(!debug.contains("credential-password")); + assert!(!debug.contains("credential-passthrough")); + assert!(debug.contains("redacted") || debug.contains("REDACTED")); +} + +#[rstest] +fn legacy_unknown_settings_do_not_change_resolution() { + let resolved = resolve( + "general_settings:\n coordination_redis:\n host: redis\n redis_flush_size: 42\n service_name: primary", + &[], + ) + .unwrap(); + + assert!(matches!(resolved.target, Target::Standalone(_))); +} + +#[rstest] +#[case::configured( + "general_settings:\n coordination_redis:\n host: redis\n max_connections: 47", + &[], + "max_connections" +)] +#[case::from_the_environment( + "general_settings:\n coordination_redis:\n host: redis", + &[("REDIS_MAX_CONNECTIONS", "47")], + "REDIS_MAX_CONNECTIONS" +)] +fn ignored_settings_are_named_by_their_source( + #[case] yaml: &str, + #[case] environment: &[(&str, &str)], + #[case] source: &str, +) { + let resolution = coordination_resolution(yaml, environment).unwrap(); + + let settings: Vec<&str> = resolution + .ignored + .iter() + .map(|ignored| ignored.setting.as_str()) + .collect(); + assert_eq!(settings, vec![source]); +} + +fn snapshot(values: &[(&str, &str)]) -> SecretSnapshot { + let values: HashMap<&str, &str> = values.iter().copied().collect(); + SecretSnapshot::read( + ConfiguredRedis::environment_only().secret_names(), + &|name: &str| values.get(name).map(|value| (*value).to_owned()), + ) +} + +#[rstest] +fn secret_manager_values_reach_the_resolver_through_the_snapshot() { + let settings = redis_cache_kwargs(json!({"host": "os.environ/COORD_HOST"})); + let secrets = SecretSnapshot::read(settings.secret_names(), &|name: &str| match name { + "COORD_HOST" => Some("from-the-manager".to_owned()), + "REDIS_PASSWORD" => Some("manager-secret".to_owned()), + _ => None, + }); + + let resolved = settings.resolve(&secrets, &|_: &str| None).unwrap(); + + assert!( + matches!(&resolved.target, Target::Standalone(host) if host.host == "from-the-manager") + ); + assert_eq!(resolved.auth, static_auth(None, Some("manager-secret"))); +} + +#[rstest] +#[case::from_the_environment(&[], &[("REDIS_USERNAME", "environment-user")], Some("environment-user"))] +#[case::never_from_the_secret_manager(&[("REDIS_USERNAME", "manager-user")], &[], None)] +fn the_azure_username_comes_from_the_process_environment_like_python( + #[case] secrets: &[(&str, &str)], + #[case] environment: &[(&str, &str)], + #[case] expected: Option<&str>, +) { + let settings = redis_cache_kwargs(json!({"host": "redis", "azure_redis_ad_token": "true"})); + let environment: HashMap<&str, &str> = environment.iter().copied().collect(); + + let resolved = settings + .resolve(&snapshot(secrets), &|name: &str| { + environment.get(name).map(|value| (*value).to_owned()) + }) + .unwrap(); + + assert!( + matches!(&resolved.auth, Auth::AzureAd(azure) if azure.username.as_deref() == expected), + "{:?}", + resolved.auth + ); +} + +#[rstest] +#[case::coordination_drops_the_null( + "general_settings:\n coordination_redis:\n host: redis\n ssl: null", + false, + true +)] +#[case::cache_passes_an_unnamed_null_on( + "litellm_settings:\n cache_params:\n type: redis\n host: redis\n ssl: null", + true, + false +)] +#[case::cache_drops_a_null_it_names( + "litellm_settings:\n cache_params:\n type: redis\n host: null\n ssl: true", + true, + true +)] +fn a_null_overrides_the_environment_only_where_python_passes_it_on( + #[case] yaml: &str, + #[case] cache: bool, + #[case] tls: bool, +) { + let environment = [("REDIS_HOST", "environment"), ("REDIS_SSL", "true")]; + let resolved = match cache { + true => resolve_cache_in(yaml, &environment), + false => resolve(yaml, &environment), + } + .unwrap(); + + let Target::Standalone(host) = resolved.target else { + panic!("expected a standalone target, got {:?}", resolved.target); + }; + assert_eq!(host.tls.is_some(), tls); +} + +#[rstest] +#[case::empty_cluster_nodes_use_the_environment( + json!({"startup_nodes": []}), + &[("REDIS_CLUSTER_NODES", r#"[{"host": "node-a", "port": 7000}]"#)], + json!({"startup_nodes": "[{\"host\": \"node-a\", \"port\": 7000}]"}), + &[("startup_nodes", "REDIS_CLUSTER_NODES")], +)] +#[case::empty_sentinel_nodes_use_the_environment( + json!({"sentinel_nodes": [], "service_name": "primary"}), + &[("REDIS_SENTINEL_NODES", r#"[["sentinel-a", 26379]]"#)], + json!({"sentinel_nodes": "[[\"sentinel-a\", 26379]]", "service_name": "primary"}), + &[("sentinel_nodes", "REDIS_SENTINEL_NODES")], +)] +#[case::empty_service_name_uses_the_environment( + json!({"sentinel_nodes": [["sentinel-a", 26379]], "service_name": ""}), + &[("REDIS_SERVICE_NAME", "primary")], + json!({"sentinel_nodes": [["sentinel-a", 26379]], "service_name": "primary"}), + &[("service_name", "REDIS_SERVICE_NAME")], +)] +#[case::environment_host_port_and_password( + json!({}), + &[("REDIS_HOST", "environment"), ("REDIS_PASSWORD", "environment-secret"), ("REDIS_PORT", "6390"), ("REDIS_SSL", "True")], + json!({"port": "6390", "ssl": true, "host": "environment", "password": "environment-secret"}), + &[("host", "REDIS_HOST"), ("password", "REDIS_PASSWORD"), ("port", "REDIS_PORT"), ("ssl", "REDIS_SSL")], +)] +#[case::padded_environment_numbers_parse_like_python_int( + json!({}), + &[("REDIS_HOST", "redis"), ("REDIS_PORT", "6_380"), ("REDIS_DB", "3\n")], + json!({"port": "6_380", "host": "redis", "db": "3\n"}), + &[("db", "REDIS_DB"), ("host", "REDIS_HOST"), ("port", "REDIS_PORT")], +)] +#[case::socket_timeout_from_the_environment( + json!({"host": "redis"}), + &[("REDIS_SOCKET_TIMEOUT", "0.75")], + json!({"socket_timeout": "0.75", "host": "redis"}), + &[("socket_timeout", "REDIS_SOCKET_TIMEOUT")], +)] +#[case::environment_url( + json!({}), + &[("REDIS_URL", "redis://environment:6379/1")], + json!({"url": "redis://environment:6379/1"}), + &[("url", "REDIS_URL")], +)] +#[case::environment_url_beats_the_environment_host( + json!({}), + &[("REDIS_HOST", "ignored"), ("REDIS_URL", "redis://from-url:6379")], + json!({"host": "ignored", "url": "redis://from-url:6379"}), + &[("host", "REDIS_HOST"), ("url", "REDIS_URL")], +)] +#[case::an_explicit_host_drops_the_environment_url( + json!({"host": "configured"}), + &[("REDIS_URL", "redis://environment:6379")], + json!({"host": "configured"}), + &[], +)] +#[case::environment_cluster_nodes( + json!({}), + &[("REDIS_CLUSTER_NODES", r#"[{"host": "node-a", "port": "7000"}, {"host": "node-b", "port": 7001}]"#)], + json!({"startup_nodes": "[{\"host\": \"node-a\", \"port\": \"7000\"}, {\"host\": \"node-b\", \"port\": 7001}]"}), + &[("startup_nodes", "REDIS_CLUSTER_NODES")], +)] +#[case::sentinel_from_the_environment( + json!({}), + &[("REDIS_SENTINEL_NODES", r#"[["sentinel-a", 26379]]"#), ("REDIS_SENTINEL_PASSWORD", "sentinel-secret"), ("REDIS_SERVICE_NAME", "primary")], + json!({"sentinel_nodes": "[[\"sentinel-a\", 26379]]", "sentinel_password": "sentinel-secret", "service_name": "primary"}), + &[("sentinel_nodes", "REDIS_SENTINEL_NODES"), ("sentinel_password", "REDIS_SENTINEL_PASSWORD"), ("service_name", "REDIS_SERVICE_NAME")], +)] +#[case::an_explicit_credential_provider_beats_cloud_auth( + json!({"aws_iam_auth": true, "credential_provider": true, "gcp_service_account": "account", "host": "redis", "password": "dropped"}), + &[("REDIS_AZURE_AD_TOKEN", "true")], + json!({"aws_iam_auth": true, "credential_provider": true, "gcp_service_account": "account", "host": "redis", "password": "dropped"}), + &[], +)] +#[case::gcp_beats_azure_and_aws( + json!({"aws_iam_auth": true, "azure_redis_ad_token": "True", "host": "redis", "password": "dropped"}), + &[("REDIS_GCP_SERVICE_ACCOUNT", "projects/-/serviceAccounts/sa@project.iam.gserviceaccount.com")], + json!({"gcp_service_account": "projects/-/serviceAccounts/sa@project.iam.gserviceaccount.com", "aws_iam_auth": true, "azure_redis_ad_token": "True", "host": "redis", "password": "dropped"}), + &[("gcp_service_account", "REDIS_GCP_SERVICE_ACCOUNT")], +)] +#[case::azure_beats_aws_and_takes_the_environment_username( + json!({"aws_iam_auth": true, "azure_client_id": "client", "azure_client_secret": "secret", "azure_redis_ad_token": true, "azure_tenant_id": "tenant", "host": "redis", "username": "dropped"}), + &[("REDIS_USERNAME", "environment-user")], + json!({"aws_iam_auth": true, "azure_client_id": "client", "azure_client_secret": "secret", "azure_redis_ad_token": true, "azure_tenant_id": "tenant", "host": "redis", "username": "dropped"}), + &[], +)] +#[case::an_empty_environment_username_sends_only_the_token( + json!({"azure_client_id": "client", "azure_client_secret": "secret", "azure_redis_ad_token": true, "azure_tenant_id": "tenant", "host": "redis"}), + &[("REDIS_USERNAME", "")], + json!({"azure_client_id": "client", "azure_client_secret": "secret", "azure_redis_ad_token": true, "azure_tenant_id": "tenant", "host": "redis", "username": ""}), + &[("username", "REDIS_USERNAME")], +)] +#[case::aws_iam_over_tls_with_the_default_region( + json!({"aws_iam_auth": "yes", "aws_iam_cache_name": "cache-1", "aws_iam_user_name": "iam-user", "host": "cache-1", "password": "dropped", "ssl": true}), + &[("AWS_DEFAULT_REGION", "us-east-1")], + json!({"aws_iam_auth": "yes", "aws_iam_cache_name": "cache-1", "aws_iam_user_name": "iam-user", "host": "cache-1", "password": "dropped", "ssl": true, "aws_iam_region": "us-east-1"}), + &[("aws_iam_region", "AWS_DEFAULT_REGION")], +)] +#[case::aws_region_beats_the_default_region( + json!({"aws_iam_auth": true, "aws_iam_cache_name": "cache-1", "aws_iam_user_name": "iam-user", "host": "cache-1", "ssl": true}), + &[("AWS_DEFAULT_REGION", "us-east-1"), ("AWS_REGION", "eu-central-1")], + json!({"aws_iam_auth": true, "aws_iam_cache_name": "cache-1", "aws_iam_user_name": "iam-user", "host": "cache-1", "ssl": true, "aws_iam_region": "eu-central-1"}), + &[("aws_iam_region", "AWS_REGION")], +)] +#[case::a_null_cert_requirement_skips_verification( + json!({"host": "redis", "ssl": true, "ssl_cert_reqs": null}), + &[("REDIS_SSL_CERT_REQS", "required")], + json!({"host": "redis", "ssl": true, "ssl_cert_reqs": null}), + &[], +)] +#[case::a_cert_requirement_spelled_like_python_none_fails( + json!({"host": "redis"}), + &[("REDIS_SSL", "true"), ("REDIS_SSL_CERT_REQS", "None")], + json!({"ssl": true, "ssl_cert_reqs": "None", "host": "redis"}), + &[("ssl", "REDIS_SSL"), ("ssl_cert_reqs", "REDIS_SSL_CERT_REQS")], +)] +#[case::tls_options_from_the_environment( + json!({"host": "redis"}), + &[("REDIS_SSL", "true"), ("REDIS_SSL_CA_CERTS", "/env-ca.pem"), ("REDIS_SSL_CERT_REQS", "optional")], + json!({"ssl_ca_certs": "/env-ca.pem", "ssl": true, "ssl_cert_reqs": "optional", "host": "redis"}), + &[("ssl", "REDIS_SSL"), ("ssl_ca_certs", "REDIS_SSL_CA_CERTS"), ("ssl_cert_reqs", "REDIS_SSL_CERT_REQS")], +)] +#[case::environment_references_resolve( + json!({"host": "os.environ/COORD_HOST", "password": "os.environ/COORD_PASSWORD"}), + &[("COORD_HOST", "referenced"), ("COORD_PASSWORD", "referenced-secret")], + json!({"host": "referenced", "password": "referenced-secret"}), + &[], +)] +#[case::malformed_port_from_the_environment( + json!({}), + &[("REDIS_HOST", "redis"), ("REDIS_PORT", "abc")], + json!({"port": "abc", "host": "redis"}), + &[("host", "REDIS_HOST"), ("port", "REDIS_PORT")], +)] +#[case::retry_on_error_from_the_environment( + json!({"host": "redis"}), + &[("REDIS_RETRY_ON_ERROR", "true")], + json!({"retry_on_error": true, "host": "redis"}), + &[("retry_on_error", "REDIS_RETRY_ON_ERROR")], +)] +#[case::ssl_ca_path_from_the_environment( + json!({"host": "redis"}), + &[("REDIS_SSL", "true"), ("REDIS_SSL_CA_PATH", "/etc/ssl/private-cas")], + json!({"ssl": true, "ssl_ca_path": "/etc/ssl/private-cas", "host": "redis"}), + &[("ssl", "REDIS_SSL"), ("ssl_ca_path", "REDIS_SSL_CA_PATH")], +)] +fn merges_like_python_redis( + #[case] kwargs: serde_json::Value, + #[case] environment: &[(&str, &str)], + #[case] merged: serde_json::Value, + #[case] sources: &[(&str, &str)], +) { + let configured = redis_cache_kwargs(kwargs); + let lookup = |name: &str| { + environment + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| (*value).to_owned()) + }; + + let kwargs = configured + .merged(&SecretSnapshot::read(configured.secret_names(), &lookup)) + .unwrap(); + + let values: serde_json::Map = kwargs + .iter() + .map(|(kwarg, entry)| (kwarg.clone(), entry.value.clone())) + .collect(); + let renamed: Vec<(&str, &str)> = kwargs + .iter() + .filter(|(kwarg, entry)| *kwarg != &entry.source) + .map(|(kwarg, entry)| (kwarg.as_str(), entry.source.as_str())) + .collect(); + assert_eq!(serde_json::Value::Object(values), merged); + assert_eq!(renamed, sources); +} diff --git a/litellm-rust/crates/gateway/src/settings.rs b/litellm-rust/crates/gateway/src/settings.rs index 445071504ed..c2c8bc30934 100644 --- a/litellm-rust/crates/gateway/src/settings.rs +++ b/litellm-rust/crates/gateway/src/settings.rs @@ -4,6 +4,23 @@ use litellm_secret_value::SecretValue; use litellm_tracing::Level; use serde::{Deserialize, de::DeserializeOwned}; +#[derive(Deserialize)] +pub(super) struct LogSettings { + rust_log: Option, +} + +impl LogSettings { + pub fn log_level(&self) -> Level { + log_level(self.rust_log.as_deref()) + } +} + +fn log_level(rust_log: Option<&str>) -> Level { + rust_log + .and_then(|value| value.parse().ok()) + .unwrap_or(Level::INFO) +} + #[derive(Deserialize)] #[serde(default)] pub(super) struct Settings { @@ -26,10 +43,7 @@ impl Default for Settings { impl Settings { pub fn log_level(&self) -> Level { - self.rust_log - .as_deref() - .and_then(|value| value.parse().ok()) - .unwrap_or(Level::INFO) + log_level(self.rust_log.as_deref()) } } diff --git a/litellm-rust/crates/host-native/src/driver.rs b/litellm-rust/crates/host-native/src/driver.rs index af3248848a2..bb1a7025e10 100644 --- a/litellm-rust/crates/host-native/src/driver.rs +++ b/litellm-rust/crates/host-native/src/driver.rs @@ -84,6 +84,13 @@ where reply.send(self.interceptors.after_provider_response(raw).await); Ok(()) } + HostRequest::Intercept(InterceptRequest::ProviderRequestRetried { + retry, + reply, + }) => { + reply.send(self.interceptors.provider_request_retried(retry).await); + Ok(()) + } HostRequest::Stream(StreamDelivery::Open(head, reply)) => { self.demand = Some(reply); return Ok(Boundary::Open(head)); diff --git a/litellm-rust/crates/host-python/Cargo.toml b/litellm-rust/crates/host-python/Cargo.toml index 0b0107d0548..1fa5fb38830 100644 --- a/litellm-rust/crates/host-python/Cargo.toml +++ b/litellm-rust/crates/host-python/Cargo.toml @@ -14,6 +14,7 @@ bytes.workspace = true futures-util.workspace = true parking_lot.workspace = true serde.workspace = true +serde_json.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["rt", "sync"] } @@ -23,4 +24,3 @@ pythonize = "0.29.0" [dev-dependencies] rstest.workspace = true -serde_json.workspace = true diff --git a/litellm-rust/crates/host-python/src/binding.rs b/litellm-rust/crates/host-python/src/binding.rs index e36b7d70c28..97d6be88767 100644 --- a/litellm-rust/crates/host-python/src/binding.rs +++ b/litellm-rust/crates/host-python/src/binding.rs @@ -28,6 +28,11 @@ pub trait PythonBinding: PythonOwned { ) -> PyResult>; /// What the stream carries at hand-off, as the caller's stream receives it. + /// The unary response after the deployment transform, before success hooks run. + fn deliver_response(&mut self, _py: Python<'_>, response: Py) -> PyResult> { + Ok(response) + } + fn encode_stream_head( &mut self, py: Python<'_>, diff --git a/litellm-rust/crates/host-python/src/driver.rs b/litellm-rust/crates/host-python/src/driver.rs index dbf843a5295..ba0c174c875 100644 --- a/litellm-rust/crates/host-python/src/driver.rs +++ b/litellm-rust/crates/host-python/src/driver.rs @@ -310,6 +310,14 @@ where reply.send(Ok(())); Ok(()) } + HostRequest::Intercept(InterceptRequest::ProviderRequestRetried { retry, reply }) => { + self.notify(PythonCallEvent::Execution( + ExecutionEvent::ProviderRequestRetried { retry }, + )) + .await?; + reply.send(Ok(())); + Ok(()) + } HostRequest::Stream(StreamDelivery::Open(head, reply)) => { self.streaming = true; let head = { @@ -399,7 +407,15 @@ where Ok(step) => step, Err(error) => return self.failure(error, FailureOrigin::Host).await, }; - match self.settle(step).await { + let response = match self.settle(step).await { + Ok(response) => response, + Err(error) => return self.failure(error, FailureOrigin::Host).await, + }; + let delivered = { + let mut parts = self.parts(); + Python::attach(|py| parts.binding.deliver_response(py, response)) + }; + match delivered { Ok(response) => self.succeeded(response).await, Err(error) => self.failure(error, FailureOrigin::Host).await, } diff --git a/litellm-rust/crates/host-python/src/hooks/chain/dispatch.rs b/litellm-rust/crates/host-python/src/hooks/chain/dispatch.rs index 4939995cefa..26d5a5692d2 100644 --- a/litellm-rust/crates/host-python/src/hooks/chain/dispatch.rs +++ b/litellm-rust/crates/host-python/src/hooks/chain/dispatch.rs @@ -241,6 +241,9 @@ fn retain_event(py: Python<'_>, event: PythonCallEvent<'_>) -> OwnedEvent { CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw }) => { CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw: raw.clone() }) } + CallEvent::Execution(ExecutionEvent::ProviderRequestRetried { retry }) => { + CallEvent::Execution(ExecutionEvent::ProviderRequestRetried { retry }) + } CallEvent::Succeeded { timing, response } => CallEvent::Succeeded { timing, response: response.clone_ref(py), @@ -280,6 +283,12 @@ fn dispatch( py, CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw }), ), + CallEvent::Execution(ExecutionEvent::ProviderRequestRetried { retry }) => hooks.on_event( + py, + CallEvent::Execution(ExecutionEvent::ProviderRequestRetried { + retry: retry.clone(), + }), + ), CallEvent::Succeeded { timing, response } => hooks.on_event( py, CallEvent::Succeeded { diff --git a/litellm-rust/crates/host-python/src/lib.rs b/litellm-rust/crates/host-python/src/lib.rs index 8e37ab02669..1cd4d65e29e 100644 --- a/litellm-rust/crates/host-python/src/lib.rs +++ b/litellm-rust/crates/host-python/src/lib.rs @@ -31,13 +31,14 @@ pub use gil::{PythonContext, attach_blocking, release_count, release_gil}; pub use handle::{Execution, ExecutionBody, ExecutionStep, StepTypes, Suspension}; pub use hooks::{HookChain, HookResume, HookStep, PythonCallEvent, PythonCallHooks, PythonRuntime}; pub use marshal::{ - Pythonized, from_py, from_py_argument, json_loads, json_object_field, panic_to_pyerr, to_py, + Pythonized, from_py, from_py_argument, json_from_py, json_loads, json_object_field, json_to_py, + panic_to_pyerr, to_py, }; pub use owned::PythonOwned; pub use runtime::{ ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, enter_native, poll_async_value, - ready_future, reserve_process_for_forking, run_async, run_async_value, run_sync, - run_sync_value, runtime_started, + ready_future, reserve_process_for_forking, run_async, run_async_value, run_isolated, run_sync, + run_sync_value, runtime_handle, runtime_started, }; pub use services::PythonHostCalls; diff --git a/litellm-rust/crates/host-python/src/marshal.rs b/litellm-rust/crates/host-python/src/marshal.rs index 465521cf9d2..a6468d38d83 100644 --- a/litellm-rust/crates/host-python/src/marshal.rs +++ b/litellm-rust/crates/host-python/src/marshal.rs @@ -4,9 +4,10 @@ use std::panic::{AssertUnwindSafe, catch_unwind}; use pyo3::exceptions::PyValueError; use pyo3::panic::PanicException; use pyo3::prelude::*; -use pyo3::types::PyBytes; +use pyo3::types::{PyBool, PyBytes, PyDict, PyFloat, PyInt, PyList, PyString, PyTuple}; use serde::Serialize; use serde::de::DeserializeOwned; +use serde_json::{Map, Number, Value}; /// Converts a `#[pyo3(from_py_with = ...)]` argument, reporting failures as `ValueError` /// so a bad argument reads as a bad argument rather than as whatever the conversion hit. @@ -71,6 +72,119 @@ pub fn panic_to_pyerr(payload: Box) -> PyErr { PanicException::new_err(message.to_string()) } +pub fn json_from_py(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(Value::Null); + } + if let Ok(flag) = value.cast::() { + return Ok(Value::Bool(flag.is_true())); + } + if let Ok(int) = value.cast::() { + if let Ok(number) = int.extract::() { + return Ok(Value::from(number)); + } + if let Ok(number) = int.extract::() { + return Ok(Value::from(number)); + } + return raw_leaf(value); + } + if let Ok(float) = value.cast::() { + return match Number::from_f64(float.value()) { + Some(number) => Ok(Value::Number(number)), + None => raw_leaf(value), + }; + } + if let Ok(text) = value.cast::() { + return json_string(text); + } + if let Ok(dict) = value.cast::() { + if !dict + .keys() + .iter() + .all(|key| key.is_instance_of::()) + { + return from_py(value); + } + return dict + .iter() + .map(|(key, item)| Ok((json_key(key.cast::()?)?, json_from_py(&item)?))) + .collect::>>() + .map(Value::Object); + } + if let Ok(list) = value.cast::() { + return list + .iter() + .map(|item| json_from_py(&item)) + .collect::>() + .map(Value::Array); + } + if let Ok(tuple) = value.cast::() { + return tuple + .iter() + .map(|item| json_from_py(&item)) + .collect::>() + .map(Value::Array); + } + from_py(value) +} + +pub fn json_to_py<'py>(py: Python<'py>, value: &Value) -> PyResult> { + Ok(match value { + Value::Null => py.None().into_bound(py), + Value::Bool(flag) => PyBool::new(py, *flag).to_owned().into_any(), + Value::Number(number) => match (number.as_i64(), number.as_u64()) { + (Some(int), _) => int.into_pyobject(py)?.into_any(), + (None, Some(int)) => int.into_pyobject(py)?.into_any(), + (None, None) => PyFloat::new(py, number.as_f64().unwrap_or(f64::NAN)).into_any(), + }, + Value::String(text) => match litellm_host::raw_json::decode(text) { + Some(raw) => py.import("json")?.call_method1("loads", (raw,))?, + None => PyString::new(py, text).into_any(), + }, + Value::Array(items) => PyList::new( + py, + items + .iter() + .map(|item| json_to_py(py, item)) + .collect::>>()?, + )? + .into_any(), + Value::Object(fields) => { + let dict = PyDict::new(py); + for (key, item) in fields { + dict.set_item( + json_to_py(py, &Value::String(key.clone()))?, + json_to_py(py, item)?, + )?; + } + dict.into_any() + } + }) +} + +fn json_key(text: &Bound<'_, PyString>) -> PyResult { + match json_string(text)? { + Value::String(key) => Ok(key), + _ => unreachable!("a Python str converts to a JSON string"), + } +} + +fn json_string(text: &Bound<'_, PyString>) -> PyResult { + match text.to_str() { + Ok(text) if !litellm_host::raw_json::is_marked(text) => Ok(Value::String(text.to_owned())), + _ => raw_leaf(text.as_any()), + } +} + +fn raw_leaf(value: &Bound<'_, PyAny>) -> PyResult { + let raw: String = value + .py() + .import("json")? + .call_method1("dumps", (value,))? + .extract()?; + Ok(Value::String(litellm_host::raw_json::encode(&raw))) +} + #[cfg(test)] mod tests { use serde::Serializer; @@ -131,6 +245,72 @@ mod tests { }); } + #[rstest::rstest] + #[case::plain( + c"{'a': [1, 2.5, None, True, 'x'], 'b': (1,)}", + r#"{"a":[1,2.5,null,true,"x"],"b":[1]}"# + )] + #[case::nan(c"{'t': float('nan')}", r#"{"t":NaN}"#)] + #[case::infinities(c"[float('inf'), -float('inf')]", "[Infinity,-Infinity]")] + #[case::wide_integers( + c"[2**64, -(2**70), 2**64 - 1, -(2**63)]", + "[18446744073709551616,-1180591620717411303424,18446744073709551615,-9223372036854775808]" + )] + #[case::lone_surrogates(c"{'k\\udc00': 'a\\ud800b \\u00e9'}", r#"{"k\udc00":"a\ud800b é"}"#)] + #[case::marked_text(c"['\\uf8fflitellm-raw-json:1']", r#"["litellm-raw-json:1"]"#)] + fn json_values_keep_what_serde_json_cannot_hold( + #[case] source: &std::ffi::CStr, + #[case] wire: &str, + ) { + crate::initialize_python(); + Python::attach(|py| { + let value = py.eval(source, None, None).unwrap(); + let converted = json_from_py(&value).unwrap(); + let json = py.import("json").unwrap(); + let loads = |text: &str| json.call_method1("loads", (text,)).unwrap(); + let wire_bytes = litellm_host::raw_json::to_vec(&converted).unwrap(); + assert!( + loads(std::str::from_utf8(&wire_bytes).unwrap()) + .eq(loads(wire)) + .unwrap() + || wire.contains("NaN") + ); + assert_eq!( + json.call_method1("dumps", (loads(std::str::from_utf8(&wire_bytes).unwrap()),)) + .unwrap() + .extract::() + .unwrap(), + json.call_method1("dumps", (loads(wire),)) + .unwrap() + .extract::() + .unwrap() + ); + let back = json_to_py(py, &converted).unwrap(); + assert_eq!( + json.call_method1("dumps", (back,)) + .unwrap() + .extract::() + .unwrap(), + json.call_method1("dumps", (loads(wire),)) + .unwrap() + .extract::() + .unwrap() + ); + }); + } + + #[test] + fn values_json_cannot_hold_keep_their_old_conversion() { + crate::initialize_python(); + Python::attach(|py| { + let value = py.eval(c"{1: 'a'}", None, None).unwrap(); + assert_eq!( + json_from_py(&value).unwrap_err().to_string(), + from_py::(&value).unwrap_err().to_string() + ); + }); + } + #[test] fn depythonize_preserves_python_exception_identity_and_traceback() { crate::initialize_python(); diff --git a/litellm-rust/crates/host-python/src/runtime.rs b/litellm-rust/crates/host-python/src/runtime.rs index e8872c9340a..2a8fc598fc4 100644 --- a/litellm-rust/crates/host-python/src/runtime.rs +++ b/litellm-rust/crates/host-python/src/runtime.rs @@ -68,6 +68,23 @@ fn runtime() -> PyResult<&'static Runtime> { Ok(pyo3_async_runtimes::tokio::get_runtime()) } +/// Runs `future` on a current-thread runtime built and shut down inside the call, its blocking pool joined before +/// it returns. No thread outlives the call, so unlike every other door this one never claims the process in the fork +/// gate: a process that only used it can still fork. The future must not hold resources tied to another runtime +pub fn run_isolated(future: F) -> std::io::Result { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let output = runtime.block_on(future); + runtime.shutdown_timeout(Duration::from_secs(5)); + Ok(output) +} + +/// The native runtime, for work that must outlive the call that schedules it. +pub fn runtime_handle() -> PyResult { + Ok(runtime()?.handle().clone()) +} + #[expect(clippy::disallowed_methods, reason = "this is the gated door")] fn future_into_py(py: Python<'_>, future: F) -> PyResult> where @@ -813,4 +830,21 @@ asyncio.run(exercise()) .expect("async route contract should hold"); }); } + + #[rstest] + fn an_isolated_run_joins_blocking_work_it_left_running_before_returning() { + let finished = Arc::new(AtomicBool::new(false)); + let flag = Arc::clone(&finished); + let output = run_isolated(async move { + drop(tokio::task::spawn_blocking(move || { + thread::sleep(Duration::from_millis(50)); + flag.store(true, Ordering::SeqCst); + })); + 7 + }) + .unwrap(); + + assert_eq!(output, 7); + assert!(finished.load(Ordering::SeqCst)); + } } diff --git a/litellm-rust/crates/host-python/tests/isolated_run.rs b/litellm-rust/crates/host-python/tests/isolated_run.rs new file mode 100644 index 00000000000..7b8603229a7 --- /dev/null +++ b/litellm-rust/crates/host-python/tests/isolated_run.rs @@ -0,0 +1,9 @@ +use litellm_host_python::{reserve_process_for_forking, run_isolated, runtime_started}; + +#[test] +fn an_isolated_run_leaves_the_process_free_to_reserve_for_forking() { + run_isolated(async { tokio::task::spawn_blocking(|| ()).await.unwrap() }).unwrap(); + + assert!(!runtime_started()); + assert!(reserve_process_for_forking().is_ok()); +} diff --git a/litellm-rust/crates/host-python/tests/support/mod.rs b/litellm-rust/crates/host-python/tests/support/mod.rs index 15e864c0718..f4f0903cc45 100644 --- a/litellm-rust/crates/host-python/tests/support/mod.rs +++ b/litellm-rust/crates/host-python/tests/support/mod.rs @@ -422,6 +422,9 @@ impl CallHooks for SyntheticHooks { PythonCallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw }) => { format!("response:{}", raw.body) } + PythonCallEvent::Execution(ExecutionEvent::ProviderRequestRetried { retry }) => { + format!("retried:{}", retry.body) + } PythonCallEvent::Succeeded { response, .. } => { format!("succeeded:{}", response.bind(py)) } diff --git a/litellm-rust/crates/host/Cargo.toml b/litellm-rust/crates/host/Cargo.toml index e0ccab4e0ad..2c113bc644c 100644 --- a/litellm-rust/crates/host/Cargo.toml +++ b/litellm-rust/crates/host/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true litellm-secret-value.workspace = true futures-util.workspace = true litellm-coroutine.workspace = true +serde.workspace = true serde_json.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["sync"] } diff --git a/litellm-rust/crates/host/src/interceptors.rs b/litellm-rust/crates/host/src/interceptors.rs index 9398ea4bad6..8146448cd98 100644 --- a/litellm-rust/crates/host/src/interceptors.rs +++ b/litellm-rust/crates/host/src/interceptors.rs @@ -10,6 +10,18 @@ pub struct WireRequest { pub body: Value, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RetryReason { + ThinkingSignature, + StrictTools { dropped: usize }, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct RetriedRequest { + pub reasons: Vec, + pub body: Value, +} + /// What the route knows about the request it is sending, for a host that logs it. The /// route owns these facts; a host reads them beside the wire request and never rewrites /// them. @@ -62,6 +74,7 @@ pub struct ProviderIdentity { pub enum ResultSource { Provider, Cache { key: String }, + Host, } /// What the call cost, settled once by the route. A host records it; it never re-derives it. @@ -79,6 +92,7 @@ impl Cost { pub fn settle(source: &ResultSource, reported: Option) -> Self { match (source, reported) { (ResultSource::Cache { .. }, _) => Self::CacheHit, + (ResultSource::Host, _) => Self::Deferred, (ResultSource::Provider, Some(amount)) => Self::Reported { amount }, (ResultSource::Provider, None) => Self::Deferred, } @@ -107,6 +121,13 @@ pub trait Interceptors: Send + Sync { &self, raw: RawResponse, ) -> impl Future> + Send; + + fn provider_request_retried( + &self, + _retry: RetriedRequest, + ) -> impl Future> + Send { + async { Ok(()) } + } } impl + ?Sized> Interceptors for &T { @@ -128,6 +149,13 @@ impl + ?Sized> Interceptors for &T { ) -> impl Future> + Send { (**self).after_provider_response(raw) } + + fn provider_request_retried( + &self, + retry: RetriedRequest, + ) -> impl Future> + Send { + (**self).provider_request_retried(retry) + } } impl Interceptors for () { diff --git a/litellm-rust/crates/host/src/lib.rs b/litellm-rust/crates/host/src/lib.rs index 76fce582fb3..26c53c89f97 100644 --- a/litellm-rust/crates/host/src/lib.rs +++ b/litellm-rust/crates/host/src/lib.rs @@ -13,3 +13,4 @@ pub mod lifecycle; pub mod machine; pub mod observation; pub mod protocol; +pub mod raw_json; diff --git a/litellm-rust/crates/host/src/lifecycle.rs b/litellm-rust/crates/host/src/lifecycle.rs index e6b31381494..16456ea52d0 100644 --- a/litellm-rust/crates/host/src/lifecycle.rs +++ b/litellm-rust/crates/host/src/lifecycle.rs @@ -6,7 +6,10 @@ use std::{ use futures_util::TryStreamExt; -use crate::{call::CallOutput, interceptors::RawResponse}; +use crate::{ + call::CallOutput, + interceptors::{RawResponse, RetriedRequest}, +}; /// Seconds since the Unix epoch, on one clock for every host. pub fn epoch_seconds() -> f64 { @@ -50,7 +53,7 @@ pub enum CallEvent { }, } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq)] pub enum ExecutionEvent { ResultReady { facts: crate::interceptors::ExecutionFacts, @@ -58,6 +61,9 @@ pub enum ExecutionEvent { ProviderResponseReceived { raw: Raw, }, + ProviderRequestRetried { + retry: RetriedRequest, + }, } impl> CallEvent { @@ -76,6 +82,11 @@ impl> CallEvent { + CallEvent::Execution(ExecutionEvent::ProviderRequestRetried { + retry: retry.clone(), + }) + } Self::Succeeded { timing, .. } => CallEvent::Succeeded { timing: *timing, response: (), diff --git a/litellm-rust/crates/host/src/machine/context.rs b/litellm-rust/crates/host/src/machine/context.rs index 1e53605ed7b..dacb470e97f 100644 --- a/litellm-rust/crates/host/src/machine/context.rs +++ b/litellm-rust/crates/host/src/machine/context.rs @@ -5,7 +5,7 @@ use litellm_coroutine::Co; use super::coroutine::MachineFault; use crate::{ - interceptors::{RawResponse, RequestContext, WireRequest}, + interceptors::{RawResponse, RequestContext, RetriedRequest, WireRequest}, protocol::{HostRequest, InterceptRequest, Protocol, Reply, StreamDelivery}, }; @@ -119,6 +119,14 @@ where }) .await? } + + async fn provider_request_retried(&self, retry: RetriedRequest) -> Result<(), P::Error> { + self.0 + .request_reply(|reply| { + HostRequest::Intercept(InterceptRequest::ProviderRequestRetried { retry, reply }) + }) + .await? + } } pub struct StreamSender(Channel

); diff --git a/litellm-rust/crates/host/src/protocol.rs b/litellm-rust/crates/host/src/protocol.rs index c24557b0348..5ec566b6a65 100644 --- a/litellm-rust/crates/host/src/protocol.rs +++ b/litellm-rust/crates/host/src/protocol.rs @@ -2,7 +2,7 @@ use std::ops::ControlFlow; pub use litellm_coroutine::{Abandoned, Answer, Reply, reply}; -use crate::interceptors::{RawResponse, RequestContext, WireRequest}; +use crate::interceptors::{RawResponse, RequestContext, RetriedRequest, WireRequest}; pub trait Protocol: Send + Sync + 'static { type Request: Send + 'static; @@ -33,6 +33,10 @@ pub enum InterceptRequest { raw: RawResponse, reply: Reply>, }, + ProviderRequestRetried { + retry: RetriedRequest, + reply: Reply>, + }, } pub enum StreamDelivery { diff --git a/litellm-rust/crates/host/src/raw_json.rs b/litellm-rust/crates/host/src/raw_json.rs new file mode 100644 index 00000000000..fbf9bbc6ecb --- /dev/null +++ b/litellm-rust/crates/host/src/raw_json.rs @@ -0,0 +1,362 @@ +use std::{borrow::Cow, io}; + +use serde::Serialize; +use serde_json::ser::{CharEscape, CompactFormatter, Formatter}; + +const MARK: char = '\u{F8FF}'; +const PREFIX: &str = "\u{F8FF}litellm-raw-json:"; + +pub fn encode(raw: &str) -> String { + format!("{PREFIX}{raw}") +} + +pub fn decode(text: &str) -> Option<&str> { + text.strip_prefix(PREFIX).filter(|raw| is_scalar(raw)) +} + +pub fn is_marked(text: &str) -> bool { + text.starts_with(PREFIX) +} + +pub fn to_vec(value: &T) -> serde_json::Result> { + let mut out = Vec::with_capacity(128); + let mut serializer = serde_json::Serializer::with_formatter(&mut out, RawLeaves::default()); + value.serialize(&mut serializer)?; + Ok(out) +} + +pub fn mark_wide_numbers(text: &str) -> Cow<'_, str> { + let bytes = text.as_bytes(); + let mut out = String::new(); + let mut copied = 0; + let mut index = 0; + while index < bytes.len() { + match bytes[index] { + b'"' => { + let end = string_end(bytes, index + 1); + if let Some(escaped) = escaped_marked_string(&text[index..end]) { + out.push_str(&text[copied..index]); + out.push_str(&escaped); + copied = end; + } + index = end; + } + b'-' | b'0'..=b'9' => { + let end = bytes[index..] + .iter() + .position(|byte| { + !matches!(byte, b'0'..=b'9' | b'-' | b'+' | b'.' | b'e' | b'E') + }) + .map_or(bytes.len(), |length| index + length); + if let Some(raw) = wide_number(&text[index..end]) { + out.push_str(&text[copied..index]); + out.push_str(&serde_json::Value::String(encode(raw)).to_string()); + copied = end; + } + index = end; + } + _ => index += 1, + } + } + if copied == 0 { + return Cow::Borrowed(text); + } + out.push_str(&text[copied..]); + Cow::Owned(out) +} + +fn escaped_marked_string(token: &str) -> Option { + let inner = token.strip_prefix('"')?; + let escaped_mark = inner + .get(..6) + .is_some_and(|head| head.eq_ignore_ascii_case("\\uf8ff")); + if !inner.starts_with(MARK) && !escaped_mark { + return None; + } + let text = serde_json::from_str::(token).ok()?; + is_marked(&text) + .then(|| serde_json::Value::String(encode(&ascii_string_literal(&text))).to_string()) +} + +fn ascii_string_literal(text: &str) -> String { + let mut literal = String::with_capacity(text.len() + 2); + literal.push('"'); + for character in text.chars() { + match character { + '"' => literal.push_str("\\\""), + '\\' => literal.push_str("\\\\"), + '\n' => literal.push_str("\\n"), + '\r' => literal.push_str("\\r"), + '\t' => literal.push_str("\\t"), + ' '..='~' => literal.push(character), + _ => { + let mut units = [0; 2]; + for unit in character.encode_utf16(&mut units) { + literal.push_str(&format!("\\u{unit:04x}")); + } + } + } + } + literal.push('"'); + literal +} + +fn string_end(bytes: &[u8], mut index: usize) -> usize { + while index < bytes.len() { + match bytes[index] { + b'\\' => index += 2, + b'"' => return index + 1, + _ => index += 1, + } + } + index +} + +fn wide_number(raw: &str) -> Option<&str> { + if is_integer(raw) { + return (raw.parse::().is_err() && raw.parse::().is_err()).then_some(raw); + } + match raw.parse::() { + Ok(number) if number == f64::INFINITY => Some("Infinity"), + Ok(number) if number == f64::NEG_INFINITY => Some("-Infinity"), + _ => None, + } +} + +fn is_scalar(raw: &str) -> bool { + matches!(raw, "NaN" | "Infinity" | "-Infinity") || is_integer(raw) || is_string_literal(raw) +} + +fn is_integer(raw: &str) -> bool { + let digits = raw.strip_prefix('-').unwrap_or(raw); + match digits.as_bytes() { + [b'0'] => true, + [b'1'..=b'9', rest @ ..] => rest.iter().all(u8::is_ascii_digit), + _ => false, + } +} + +fn is_string_literal(raw: &str) -> bool { + let Some(inner) = raw + .strip_prefix('"') + .and_then(|rest| rest.strip_suffix('"')) + else { + return false; + }; + let mut bytes = inner.bytes(); + while let Some(byte) = bytes.next() { + match byte { + b'\\' => match bytes.next() { + Some(b'"' | b'\\' | b'/' | b'b' | b'f' | b'n' | b'r' | b't') => {} + Some(b'u') => { + if !(0..4).all(|_| bytes.next().is_some_and(|hex| hex.is_ascii_hexdigit())) { + return false; + } + } + _ => return false, + }, + b'"' => return false, + 0x20..=0x7e => {} + _ => return false, + } + } + true +} + +fn unescaped(escape: CharEscape) -> char { + match escape { + CharEscape::Quote => '"', + CharEscape::ReverseSolidus => '\\', + CharEscape::Solidus => '/', + CharEscape::Backspace => '\u{8}', + CharEscape::FormFeed => '\u{c}', + CharEscape::LineFeed => '\n', + CharEscape::CarriageReturn => '\r', + CharEscape::Tab => '\t', + CharEscape::AsciiControl(byte) => char::from(byte), + } +} + +#[derive(Default)] +enum Text { + #[default] + Closed, + Opened, + Plain, + Marked(String), +} + +#[derive(Default)] +struct RawLeaves { + key: bool, + text: Text, +} + +impl Formatter for RawLeaves { + fn begin_object_key( + &mut self, + writer: &mut W, + first: bool, + ) -> io::Result<()> { + self.key = true; + CompactFormatter.begin_object_key(writer, first) + } + + fn end_object_key(&mut self, writer: &mut W) -> io::Result<()> { + self.key = false; + CompactFormatter.end_object_key(writer) + } + + fn begin_string(&mut self, _writer: &mut W) -> io::Result<()> { + self.text = Text::Opened; + Ok(()) + } + + fn write_string_fragment( + &mut self, + writer: &mut W, + fragment: &str, + ) -> io::Result<()> { + match &mut self.text { + Text::Opened if fragment.starts_with(MARK) => { + self.text = Text::Marked(fragment.to_owned()); + Ok(()) + } + Text::Opened => { + self.text = Text::Plain; + writer.write_all(b"\"")?; + writer.write_all(fragment.as_bytes()) + } + Text::Marked(held) => { + held.push_str(fragment); + Ok(()) + } + Text::Plain | Text::Closed => writer.write_all(fragment.as_bytes()), + } + } + + fn write_char_escape( + &mut self, + writer: &mut W, + escape: CharEscape, + ) -> io::Result<()> { + match &mut self.text { + Text::Opened => { + self.text = Text::Plain; + writer.write_all(b"\"")?; + CompactFormatter.write_char_escape(writer, escape) + } + Text::Marked(held) => { + held.push(unescaped(escape)); + Ok(()) + } + Text::Plain | Text::Closed => CompactFormatter.write_char_escape(writer, escape), + } + } + + fn end_string(&mut self, writer: &mut W) -> io::Result<()> { + match std::mem::take(&mut self.text) { + Text::Opened => writer.write_all(b"\"\""), + Text::Plain | Text::Closed => writer.write_all(b"\""), + Text::Marked(held) => match decode(&held) { + Some(raw) if !self.key || raw.starts_with('"') => writer.write_all(raw.as_bytes()), + _ => serde_json::to_writer(writer, &held).map_err(io::Error::other), + }, + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::{Value, json}; + + use super::*; + + #[rstest] + #[case::nan("NaN")] + #[case::infinity("Infinity")] + #[case::negative_infinity("-Infinity")] + #[case::wide_integer("18446744073709551616")] + #[case::negative_wide_integer("-1180591620717411303424")] + #[case::zero("0")] + #[case::surrogate_string(r#""sys \ud800 lone""#)] + #[case::escaped_string(r#""a\"b\\c\n\u00e9""#)] + fn scalar_texts_are_written_back_verbatim(#[case] raw: &str) { + let body = json!({"leaf": encode(raw), "list": [encode(raw)]}); + assert_eq!( + String::from_utf8(to_vec(&body).unwrap()).unwrap(), + format!(r#"{{"leaf":{raw},"list":[{raw}]}}"#) + ); + } + + #[rstest] + #[case::structure("1, \"injected\": true")] + #[case::object("{}")] + #[case::leading_zero("012")] + #[case::float("1.5")] + #[case::unquoted_quote(r#""a"b""#)] + #[case::non_ascii("\"\u{e9}\"")] + #[case::bad_escape(r#""\x""#)] + #[case::short_unicode_escape(r#""\u12""#)] + #[case::empty("")] + fn other_marked_texts_stay_strings(#[case] raw: &str) { + let marked = encode(raw); + assert_eq!(decode(&marked), None); + assert_eq!( + to_vec(&json!([marked])).unwrap(), + serde_json::to_vec(&json!([marked])).unwrap() + ); + } + + #[test] + fn keys_take_only_string_texts() { + let body = Value::Object( + [ + (encode(r#""k\udc00""#), json!(1)), + (encode("NaN"), json!(2)), + ] + .into_iter() + .collect(), + ); + assert_eq!( + String::from_utf8(to_vec(&body).unwrap()).unwrap(), + format!( + r#"{{"k\udc00":1,{}:2}}"#, + serde_json::to_string(&encode("NaN")).unwrap() + ) + ); + } + + #[rstest] + #[case::wide_integer(r#"{"a":18446744073709551616}"#, json!({"a": encode("18446744073709551616")}))] + #[case::negative_wide_integer(r#"[-12345678901234567890123]"#, json!([encode("-12345678901234567890123")]))] + #[case::overflowing_float(r#"{"a":[1e400,-1E+400]}"#, json!({"a": [encode("Infinity"), encode("-Infinity")]}))] + #[case::numbers_that_fit(r#"{"a":[18446744073709551615,-9223372036854775808,-0.0,5e-324,1E2,0.30000000000000004]}"#, json!({"a": [18_446_744_073_709_551_615_u64, i64::MIN, -0.0, 5e-324, 100.0, 0.300_000_000_000_000_04]}))] + #[case::provider_marked_text("{\"a\":\"\u{F8FF}litellm-raw-json:42\"}", json!({"a": encode(r#""\uf8fflitellm-raw-json:42""#)}))] + #[case::provider_escaped_mark(r#"{"\uF8FFlitellm-raw-json:1":["\uf8fflitellm-raw-json:\"x\"\n\u00e9"]}"#, json!({encode(r#""\uf8fflitellm-raw-json:1""#): [encode(r#""\uf8fflitellm-raw-json:\"x\"\n\u00e9""#)]}))] + #[case::mark_not_leading(r#"["x\uf8fflitellm-raw-json:NaN"]"#, json!(["x\u{F8FF}litellm-raw-json:NaN"]))] + #[case::numbers_inside_strings(r#"{"18446744073709551616":"1e400 \" 18446744073709551616"}"#, json!({"18446744073709551616": "1e400 \" 18446744073709551616"}))] + fn wide_numbers_are_read_as_marked_texts(#[case] text: &str, #[case] read: Value) { + assert_eq!( + serde_json::from_str::(&mark_wide_numbers(text)).unwrap(), + read + ); + } + + #[test] + fn a_text_without_wide_numbers_is_borrowed() { + assert!(matches!( + mark_wide_numbers(r#"{"a":[1,2.5,"18446744073709551616"]}"#), + Cow::Borrowed(_) + )); + } + + #[rstest] + #[case::plain(json!({"a": "text", "b": ["", "\"quoted\"\n", "\u{F8FF}"], "c": 1.5, "d": null}))] + #[case::mark_inside(json!(["x\u{F8FF}litellm-raw-json:NaN"]))] + #[case::control(json!(["\u{1}\t"]))] + fn unmarked_documents_serialize_as_serde_json_does(#[case] body: Value) { + assert_eq!(to_vec(&body).unwrap(), serde_json::to_vec(&body).unwrap()); + } +} diff --git a/litellm-rust/crates/http/src/config.rs b/litellm-rust/crates/http/src/config.rs index 5e5f2d66fa2..8e9a7000e69 100644 --- a/litellm-rust/crates/http/src/config.rs +++ b/litellm-rust/crates/http/src/config.rs @@ -94,6 +94,8 @@ impl From<&HttpSettings> for Resolution { } } +const MAX_REDIRECTS: usize = 20; + impl TryFrom<&HttpClientConfig> for reqwest::ClientBuilder { type Error = Error; @@ -101,7 +103,9 @@ impl TryFrom<&HttpClientConfig> for reqwest::ClientBuilder { let base = reqwest::Client::builder() .tls_backend_preconfigured(rustls::ClientConfig::try_from(config)?) .connect_timeout(config.connect_timeout) - .pool_idle_timeout(config.pool_idle_timeout); + .pool_idle_timeout(config.pool_idle_timeout) + .referer(false) + .redirect(reqwest::redirect::Policy::limited(MAX_REDIRECTS)); let with_keepalive = match config.tcp_keepalive { None => base, Some(keepalive) => base @@ -186,17 +190,23 @@ mod tests { assert_eq!(resolution.unsupported, []); } - #[test] - fn unsupported_ecdh_curve_keeps_the_defaults_and_is_reported() { + #[rstest] + #[case::openssl_lacks_in_rustls("secp521r1")] + #[case::nist_name("P-256")] + #[case::nist_p384("P-384")] + #[case::sec_name("secp256r1")] + #[case::lowercase("x25519")] + #[case::padded(" X25519 ")] + fn unsupported_ecdh_curve_keeps_the_defaults_and_is_reported(#[case] curve: &str) { let settings = HttpSettings { - ssl_ecdh_curve: Some("secp521r1".into()), + ssl_ecdh_curve: Some(curve.into()), ..HttpSettings::default() }; let resolution = Resolution::from(&settings); assert_eq!(resolution.config.key_exchange_group, None); assert_eq!( resolution.unsupported, - [Unsupported::EcdhCurve("secp521r1".into())] + [Unsupported::EcdhCurve(curve.into())] ); } diff --git a/litellm-rust/crates/http/src/error.rs b/litellm-rust/crates/http/src/error.rs index 7078d4aba2a..d9fdfb5486f 100644 --- a/litellm-rust/crates/http/src/error.rs +++ b/litellm-rust/crates/http/src/error.rs @@ -16,6 +16,15 @@ pub enum TlsSource { #[error(transparent)] pub struct Error(Arc); +// A streamed body the connection ended early, after the response head arrived; `expected` is +// the declared `content-length`, `None` for a chunked body. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BodyCut { + pub received: u64, + pub expected: Option, + pub reset: bool, +} + #[derive(Debug, thiserror::Error)] pub(crate) enum Repr { #[error("could not read {}", path.display())] @@ -56,6 +65,12 @@ pub(crate) enum Repr { Connect(#[source] reqwest::Error), #[error("upstream network error")] Network(#[source] reqwest::Error), + #[error("upstream network error")] + BodyCut { + #[source] + source: reqwest::Error, + cut: BodyCut, + }, #[error("upstream request timed out")] Timeout(#[source] reqwest::Error), /// No bytes arrived within the read timeout, which the caller enforces per read; `body` says @@ -201,10 +216,45 @@ impl Error { matches!(&*self.0, Repr::ReadTimeout { body: true, .. }) } + pub fn body_read(error: reqwest::Error, received: u64, expected: Option) -> Self { + let mut source: Option<&(dyn std::error::Error + 'static)> = Some(&error); + let kind = std::iter::from_fn(|| { + let current = source?; + source = current.source(); + Some(current) + }) + .find_map(|cause| cause.downcast_ref::().map(io::Error::kind)); + let reset = match kind { + Some(io::ErrorKind::UnexpectedEof) => false, + Some(io::ErrorKind::ConnectionReset) => true, + _ => return error.into(), + }; + Repr::BodyCut { + source: error.without_url(), + cut: BodyCut { + received, + expected, + reset, + }, + } + .into() + } + + pub fn body_cut(&self) -> Option { + match &*self.0 { + Repr::BodyCut { cut, .. } => Some(*cut), + _ => None, + } + } + pub fn computed_header(name: impl Into) -> Self { Repr::ComputedHeader(name.into()).into() } + pub fn request_body(error: serde_json::Error) -> Self { + Repr::RequestBody(error).into() + } + pub fn unsignable_body() -> Self { Repr::UnsignableBody.into() } @@ -226,6 +276,7 @@ impl Error { pub fn dispatched(&self) -> bool { match &*self.0 { Repr::Network(_) + | Repr::BodyCut { .. } | Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::Status { .. } @@ -285,6 +336,7 @@ impl Error { | Repr::Signature(_) | Repr::Connect(_) | Repr::Network(_) + | Repr::BodyCut { .. } | Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::Status { .. } @@ -318,6 +370,7 @@ impl Error { | Repr::Signature(_) | Repr::Connect(_) | Repr::Network(_) + | Repr::BodyCut { .. } | Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::Status { .. } @@ -355,6 +408,7 @@ impl Error { | Repr::Signature(_) | Repr::Connect(_) | Repr::Network(_) + | Repr::BodyCut { .. } | Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::Status { .. } @@ -394,6 +448,7 @@ impl Error { | Repr::UnsignableHeader(_) | Repr::Signature(_) | Repr::Network(_) + | Repr::BodyCut { .. } | Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::Status { .. } @@ -437,6 +492,7 @@ impl Error { | Repr::Signature(_) | Repr::Connect(_) | Repr::Network(_) + | Repr::BodyCut { .. } | Repr::Status { .. } | Repr::WebSocketUrl(_) | Repr::WebSocketScheme(_) @@ -473,6 +529,7 @@ impl Error { | Repr::Signature(_) | Repr::Connect(_) | Repr::Network(_) + | Repr::BodyCut { .. } | Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::WebSocketUrl(_) @@ -513,6 +570,7 @@ impl Error { | Repr::Signature(_) | Repr::Connect(_) | Repr::Network(_) + | Repr::BodyCut { .. } | Repr::WebSocketUrl(_) | Repr::WebSocketScheme(_) | Repr::WebSocketSchemeRewrite(_) @@ -548,6 +606,7 @@ impl Error { | Repr::Signature(_) | Repr::Connect(_) | Repr::Network(_) + | Repr::BodyCut { .. } | Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::Status { .. } @@ -742,4 +801,53 @@ mod tests { assert_eq!(error.upstream_response(), Some((429, body.as_str()))); assert_eq!(error.status(), Some(429)); } + + #[rstest] + #[case::chunked("transfer-encoding: chunked\r\n\r\n5\r\nhello\r\n", None)] + #[case::content_length("content-length: 100\r\n\r\nhello", Some(100))] + #[tokio::test] + async fn a_connection_that_ends_mid_body_is_a_body_cut( + #[case] tail: &'static str, + #[case] expected: Option, + ) { + use std::io::{Read as _, Write as _}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let address = listener.local_addr().expect("address"); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("accept"); + let mut request = [0_u8; 1024]; + let _ = stream.read(&mut request).expect("request"); + stream + .write_all(format!("HTTP/1.1 200 OK\r\n{tail}").as_bytes()) + .expect("response"); + }); + let mut response = reqwest::Client::builder() + .no_proxy() + .build() + .expect("client") + .get(format!("http://{address}/")) + .send() + .await + .expect("head"); + server.join().expect("server"); + let mut received = 0_u64; + let error = loop { + match response.chunk().await { + Ok(Some(chunk)) => received += chunk.len() as u64, + Ok(None) => panic!("the body must end with an error"), + Err(error) => break crate::Error::body_read(error, received, expected), + } + }; + assert_eq!( + error.body_cut(), + Some(super::BodyCut { + received: 5, + expected, + reset: false, + }) + ); + assert!(error.dispatched()); + assert_eq!(error.to_string(), "upstream network error"); + } } diff --git a/litellm-rust/crates/http/src/lib.rs b/litellm-rust/crates/http/src/lib.rs index 02129c62083..1aa11843b46 100644 --- a/litellm-rust/crates/http/src/lib.rs +++ b/litellm-rust/crates/http/src/lib.rs @@ -18,7 +18,7 @@ pub mod websocket; pub use client::Client; pub use config::{ClientIdentity, HttpClientConfig, Resolution, Verify}; -pub use error::{Error, TlsSource}; +pub use error::{BodyCut, Error, TlsSource}; pub use pool::{ClientVariant, HttpClientPool}; pub use proxy::EnvironmentProxies; pub use settings::{HttpSettings, HttpSettingsLayer, SslVerify, TcpKeepalive}; diff --git a/litellm-rust/crates/http/src/outbound.rs b/litellm-rust/crates/http/src/outbound.rs index 20b0b1be972..b11041151af 100644 --- a/litellm-rust/crates/http/src/outbound.rs +++ b/litellm-rust/crates/http/src/outbound.rs @@ -27,7 +27,20 @@ pub fn json( body: &impl Serialize, timeout: Option, ) -> Result { - let body = serde_json::to_vec(body).map_err(Repr::RequestBody)?; + json_bytes( + url, + headers, + serde_json::to_vec(body).map_err(Repr::RequestBody)?, + timeout, + ) +} + +pub fn json_bytes( + url: &str, + headers: &[(String, String)], + body: Vec, + timeout: Option, +) -> Result { let url = Url::parse(url).map_err(Repr::ParseUrl)?; let with_headers = headers.iter().fold( http::Request::post(url.as_str()), diff --git a/litellm-rust/crates/http/src/pool.rs b/litellm-rust/crates/http/src/pool.rs index 3bdc63453ef..c56c5de4a11 100644 --- a/litellm-rust/crates/http/src/pool.rs +++ b/litellm-rust/crates/http/src/pool.rs @@ -16,6 +16,7 @@ use crate::{ #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum ClientVariant { Provider, + Undecoded, NoRedirect, Media, UnpinnedMedia, @@ -55,36 +56,13 @@ impl HttpClientPool { config: &HttpClientConfig, variant: ClientVariant, ) -> Result { - let effective = match variant { - ClientVariant::Media => HttpClientConfig { - client_certificate: None, - proxies: EnvironmentProxies::default(), - ..config.clone() - }, - ClientVariant::UnpinnedMedia => HttpClientConfig { - client_certificate: None, - ..config.clone() - }, - ClientVariant::WebSocket => HttpClientConfig { - http2: false, - verify: match &config.verify { - Verify::BuiltInRoots => Verify::NativeRoots, - verify => verify.clone(), - }, - ..config.clone() - }, - ClientVariant::Provider | ClientVariant::NoRedirect => config.clone(), - }; - let key = (effective, variant); + let key = (effective(config, variant), variant); if let Some(pooled) = self.lock().get(&key) && pooled.built_at.elapsed() < self.ttl { return Ok(Client::new(pooled.client.clone())); } - let client = self - .apply(variant, reqwest::ClientBuilder::try_from(&key.0)?) - .build() - .map_err(Repr::Client)?; + let client = self.build(&key.0, variant)?; self.lock().insert( key, PooledClient { @@ -95,6 +73,26 @@ impl HttpClientPool { Ok(Client::new(client)) } + /// A client nobody else holds, for a caller that drives it on a runtime of its own and drops both together + pub fn unpooled( + &self, + config: &HttpClientConfig, + variant: ClientVariant, + ) -> Result { + self.build(&effective(config, variant), variant) + .map(Client::new) + } + + fn build( + &self, + config: &HttpClientConfig, + variant: ClientVariant, + ) -> Result { + self.apply(variant, reqwest::ClientBuilder::try_from(config)?) + .build() + .map_err(|error| Error::from(Repr::Client(error))) + } + fn lock(&self) -> MutexGuard<'_, Clients> { self.clients.lock().unwrap_or_else(PoisonError::into_inner) } @@ -106,6 +104,7 @@ impl HttpClientPool { ) -> reqwest::ClientBuilder { match variant { ClientVariant::Provider => builder, + ClientVariant::Undecoded => builder.no_gzip().no_deflate().no_brotli().no_zstd(), ClientVariant::NoRedirect | ClientVariant::UnpinnedMedia | ClientVariant::WebSocket => { builder.redirect(reqwest::redirect::Policy::none()) } @@ -116,6 +115,31 @@ impl HttpClientPool { } } +fn effective(config: &HttpClientConfig, variant: ClientVariant) -> HttpClientConfig { + match variant { + ClientVariant::Media => HttpClientConfig { + client_certificate: None, + proxies: EnvironmentProxies::default(), + ..config.clone() + }, + ClientVariant::UnpinnedMedia => HttpClientConfig { + client_certificate: None, + ..config.clone() + }, + ClientVariant::WebSocket => HttpClientConfig { + http2: false, + verify: match &config.verify { + Verify::BuiltInRoots => Verify::NativeRoots, + verify => verify.clone(), + }, + ..config.clone() + }, + ClientVariant::Provider | ClientVariant::Undecoded | ClientVariant::NoRedirect => { + config.clone() + } + } +} + #[cfg(test)] mod tests { use std::{ @@ -347,6 +371,28 @@ mod tests { assert!(request.contains("user-agent: litellm-test/9"), "{request}"); } + #[tokio::test] + async fn provider_client_follows_twenty_redirects_without_a_referer() { + let (address, _, requests) = serve("HTTP/1.1 307 Temporary Redirect").await; + let followed = pool() + .client(&config("a"), ClientVariant::Provider) + .unwrap() + .get(format!("http://{address}/start")) + .timeout(Duration::from_secs(5)) + .send() + .await; + assert!(followed.is_err()); + let requests = requests.lock().unwrap().clone(); + assert_eq!(requests.len(), 21, "{requests:?}"); + assert!(requests[1].starts_with("GET /elsewhere "), "{requests:?}"); + assert!( + requests + .iter() + .all(|request| !request.to_ascii_lowercase().contains("referer:")), + "{requests:?}" + ); + } + #[tokio::test] async fn no_redirect_variant_returns_the_redirect_instead_of_following_it() { let (address, _, _) = serve("HTTP/1.1 302 Found").await; diff --git a/litellm-rust/crates/http/src/tls.rs b/litellm-rust/crates/http/src/tls.rs index a83b0018b2b..47a1429f1a4 100644 --- a/litellm-rust/crates/http/src/tls.rs +++ b/litellm-rust/crates/http/src/tls.rs @@ -23,10 +23,10 @@ impl FromStr for KeyExchangeGroup { type Err = Unsupported; fn from_str(name: &str) -> Result { - match name.trim().to_ascii_lowercase().as_str() { - "x25519" => Ok(Self::X25519), - "prime256v1" | "secp256r1" | "p-256" => Ok(Self::Secp256r1), - "secp384r1" | "p-384" => Ok(Self::Secp384r1), + match name { + "X25519" => Ok(Self::X25519), + "prime256v1" => Ok(Self::Secp256r1), + "secp384r1" => Ok(Self::Secp384r1), _ => Err(Unsupported::EcdhCurve(name.to_owned())), } } diff --git a/litellm-rust/crates/inference-chat/tests/caching.rs b/litellm-rust/crates/inference-chat/tests/caching.rs index dd229d3d657..d8d3a8956c8 100644 --- a/litellm-rust/crates/inference-chat/tests/caching.rs +++ b/litellm-rust/crates/inference-chat/tests/caching.rs @@ -117,6 +117,7 @@ async fn the_same_route_entrypoint_reports_facts_with_or_without_caching( assert!(caching); assert!(!key.is_empty()); } + ResultSource::Host => panic!("the host never answers a chat call"), } let summaries = traces.summaries("litellm.route"); assert_eq!(summaries.len(), 2); diff --git a/litellm-rust/crates/inference-messages/Cargo.toml b/litellm-rust/crates/inference-messages/Cargo.toml index 0c5be52550d..0ca371c3ce7 100644 --- a/litellm-rust/crates/inference-messages/Cargo.toml +++ b/litellm-rust/crates/inference-messages/Cargo.toml @@ -8,7 +8,9 @@ repository.workspace = true [dependencies] litellm-secret-value.workspace = true bytes.workspace = true +flate2 = { workspace = true, features = ["zlib-rs"] } futures-util.workspace = true +http.workspace = true litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] } litellm-auth-aws.workspace = true litellm-cache-response.workspace = true @@ -30,9 +32,11 @@ tracing.workspace = true [dev-dependencies] litellm-error.workspace = true litellm-framer = { workspace = true, features = ["sse"] } +litellm-cache.workspace = true litellm-cache-memory.workspace = true litellm-host-native.workspace = true litellm-http = { workspace = true, features = ["test-support"] } litellm-inference-testing.workspace = true rstest.workspace = true +thiserror.workspace = true wiremock.workspace = true diff --git a/litellm-rust/crates/inference-messages/src/common_utils.rs b/litellm-rust/crates/inference-messages/src/common_utils.rs index e5ece2837d8..d0f109c1e99 100644 --- a/litellm-rust/crates/inference-messages/src/common_utils.rs +++ b/litellm-rust/crates/inference-messages/src/common_utils.rs @@ -1,5 +1,5 @@ use litellm_core_utils::get_llm_provider_logic::LlmProviders; -use litellm_http::request::string_headers as shared_string_headers; +use litellm_llms::ErrorDetail; use litellm_llms::tencent::messages::transformation::TENCENT_MESSAGES_CONFIG; use litellm_llms::{ anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG, @@ -19,8 +19,6 @@ use serde_json::{Map, Value}; use super::Error; -const HEADER_CONTEXT: &str = "messages"; - #[derive(Clone, Copy)] pub(crate) struct MessagesProvider { provider: LlmProviders, @@ -75,7 +73,52 @@ pub(crate) fn messages_provider(provider: LlmProviders, model: &str) -> Option>, ) -> Result, Error> { - shared_string_headers(HEADER_CONTEXT, extra_headers).map_err(Error::from) + extra_headers + .unwrap_or_default() + .into_iter() + .map(|(name, value)| match value { + Value::String(value) => Ok((name, value)), + value => Err(Error::invalid_request(ErrorDetail::invalid( + "extra_headers", + NonStringHeader { + actual: python_type_name(&value), + name, + }, + ))), + }) + .collect() +} + +/// A header value the caller set to something other than a string, named by its Python type +/// since the caller wrote it in Python. +#[derive(Debug)] +struct NonStringHeader { + name: String, + actual: &'static str, +} + +impl std::fmt::Display for NonStringHeader { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "header {} must be a string, got {}", + self.name, self.actual + ) + } +} + +impl std::error::Error for NonStringHeader {} + +fn python_type_name(value: &Value) -> &'static str { + match value { + Value::Null => "NoneType", + Value::Bool(_) => "bool", + Value::Number(number) if number.is_f64() => "float", + Value::Number(_) => "int", + Value::String(_) => "str", + Value::Array(_) => "list", + Value::Object(_) => "dict", + } } #[cfg(test)] @@ -85,7 +128,6 @@ mod tests { use rstest::rstest; use super::{messages_provider, string_headers}; - use crate::Error; use litellm_core_utils::get_llm_provider_logic::LlmProviders; #[rstest] @@ -113,17 +155,28 @@ mod tests { assert!(messages_provider(provider, model).is_none()); } - #[test] - fn string_headers_rejects_non_string_values() { - let headers = json!({"x-count": 3}).as_object().unwrap().clone(); - let err = string_headers(Some(headers)).expect_err("non-string header rejected"); + #[rstest] + #[case::int(json!(12345), "int")] + #[case::float(json!(1.5), "float")] + #[case::bool(json!(true), "bool")] + #[case::null(json!(null), "NoneType")] + #[case::list(json!([1]), "list")] + #[case::dict(json!({"a": 1}), "dict")] + fn non_string_header_is_a_bad_request_naming_the_header_and_its_type( + #[case] value: serde_json::Value, + #[case] python_type: &str, + ) { + let headers = json!({"x-kept": "v", "x-count": value}) + .as_object() + .unwrap() + .clone(); + let error = string_headers(Some(headers)).expect_err("non-string header refused"); + assert!(error.is_request(), "{error:?}"); + assert_eq!(error.upstream_response(), None); + let rejection = error.rejected_request().expect("a rejected request"); assert_eq!( - err, - Error::Headers(litellm_http::request::HeaderError { - context: "messages", - name: "x-count".to_string(), - actual: "number", - }) + litellm_error::Report::new(rejection).to_string(), + format!("invalid extra_headers: header x-count must be a string, got {python_type}") ); } } diff --git a/litellm-rust/crates/inference-messages/src/constants.rs b/litellm-rust/crates/inference-messages/src/constants.rs index c0b023be7c7..5fae6e45dad 100644 --- a/litellm-rust/crates/inference-messages/src/constants.rs +++ b/litellm-rust/crates/inference-messages/src/constants.rs @@ -2,3 +2,6 @@ /// seconds. Mirrors the Python Anthropic Messages default. The per-request /// timeout from the caller still overrides this on the request builder. pub(crate) const MESSAGES_TIMEOUT_SECS: u64 = 600; + +pub(crate) const MESSAGES_HTTP_ERROR_ATTEMPTS: usize = 2; +pub(crate) const MESSAGES_STREAM_ERROR_BODY_READ_SECS: u64 = 5; diff --git a/litellm-rust/crates/inference-messages/src/encoding.rs b/litellm-rust/crates/inference-messages/src/encoding.rs new file mode 100644 index 00000000000..c315deb1a33 --- /dev/null +++ b/litellm-rust/crates/inference-messages/src/encoding.rs @@ -0,0 +1,437 @@ +use bytes::Bytes; +use flate2::{Decompress, DecompressError, FlushDecompress, Status}; +use futures_util::StreamExt; +use reqwest::header::{CONTENT_ENCODING, HeaderMap}; + +type BoxError = Box; + +const MAX_WINDOW_BITS: u8 = 15; +const MAX_DECOMPRESS_SIZE: usize = 1 << 16; + +#[derive(Debug)] +pub struct DecodingError(String); + +impl std::fmt::Display for DecodingError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.0) + } +} + +impl std::error::Error for DecodingError {} + +impl From for DecodingError { + fn from(error: DecompressError) -> Self { + Self(match (error.needs_dictionary(), error.message()) { + (Some(_), _) => "Error 2 while decompressing data".to_owned(), + (None, Some(message)) => format!("Error -3 while decompressing data: {message}"), + (None, None) => "Error -3 while decompressing data: invalid input data".to_owned(), + }) + } +} + +enum Codec { + Gzip, + Deflate, +} + +enum FirstAttempt { + Unread, + Reading(Vec), + Over, +} + +struct Inflater { + zlib: Decompress, + first_attempt: FirstAttempt, + pending: Vec, + saturated: bool, + stalled: bool, + finished: bool, +} + +impl Inflater { + fn new(codec: Codec) -> Self { + let (zlib, first_attempt) = match codec { + Codec::Gzip => (Decompress::new_gzip(MAX_WINDOW_BITS), FirstAttempt::Over), + Codec::Deflate => (Decompress::new(true), FirstAttempt::Unread), + }; + Self { + zlib, + first_attempt, + pending: Vec::new(), + saturated: false, + stalled: false, + finished: false, + } + } + + fn push(&mut self, data: &[u8]) { + self.first_attempt = match self.first_attempt { + FirstAttempt::Unread => FirstAttempt::Reading(data.to_vec()), + FirstAttempt::Reading(_) | FirstAttempt::Over => FirstAttempt::Over, + }; + if self.finished { + return; + } + self.pending.extend_from_slice(data); + self.stalled = false; + } + + fn has_work(&self) -> bool { + !self.finished && (self.saturated || (!self.pending.is_empty() && !self.stalled)) + } + + fn step(&mut self) -> Result, DecodingError> { + match (self.inflate(), &mut self.first_attempt) { + (Err(_), FirstAttempt::Reading(first)) => { + self.pending = std::mem::take(first); + self.first_attempt = FirstAttempt::Over; + self.zlib = Decompress::new_with_window_bits(false, MAX_WINDOW_BITS); + self.saturated = false; + self.stalled = false; + self.step() + } + (inflated, attempt) => { + if self.pending.is_empty() && !self.saturated { + *attempt = FirstAttempt::Over; + } + inflated + } + } + } + + fn inflate(&mut self) -> Result, DecodingError> { + let mut output = Vec::with_capacity(MAX_DECOMPRESS_SIZE); + let (read, written) = (self.zlib.total_in(), self.zlib.total_out()); + let status = self + .zlib + .decompress_vec(&self.pending, &mut output, FlushDecompress::Sync)?; + self.pending.drain(..(self.zlib.total_in() - read) as usize); + self.finished = matches!(status, Status::StreamEnd); + if self.finished { + self.pending.clear(); + } + self.saturated = !self.finished && output.len() == MAX_DECOMPRESS_SIZE; + self.stalled = self.zlib.total_in() == read && self.zlib.total_out() == written; + Ok(output) + } +} + +struct Decoder(Vec); + +impl Decoder { + fn detect(headers: &HeaderMap) -> Option { + let inflaters: Vec = header_pairs(headers) + .into_iter() + .filter(|(name, _)| name.as_str() == CONTENT_ENCODING.as_str()) + .flat_map(|(_, value)| { + value + .split(',') + .map(|coding| coding.trim().to_lowercase()) + .collect::>() + }) + .filter_map(|coding| match coding.as_str() { + "gzip" => Some(Codec::Gzip), + "deflate" => Some(Codec::Deflate), + _ => None, + }) + .rev() + .map(Inflater::new) + .collect(); + (!inflaters.is_empty()).then_some(Self(inflaters)) + } + + fn feed(&mut self, chunk: &[u8]) { + self.0[0].push(chunk); + } + + fn next(&mut self) -> Result, DecodingError> { + self.pull(self.0.len() - 1) + .map(|decoded| decoded.map(Bytes::from)) + } + + fn pull(&mut self, layer: usize) -> Result>, DecodingError> { + loop { + if self.0[layer].has_work() { + let decoded = self.0[layer].step()?; + if !decoded.is_empty() { + return Ok(Some(decoded)); + } + continue; + } + let Some(outer) = layer.checked_sub(1) else { + return Ok(None); + }; + match self.pull(outer)? { + Some(decoded) => self.0[layer].push(&decoded), + None => return Ok(None), + } + } + } +} + +pub(crate) fn header_pairs(headers: &HeaderMap) -> Vec<(String, String)> { + let utf8 = headers + .values() + .all(|value| std::str::from_utf8(value.as_bytes()).is_ok()); + headers + .iter() + .map(|(name, value)| { + let bytes = value.as_bytes(); + let text = if utf8 { + String::from_utf8_lossy(bytes).into_owned() + } else { + bytes.iter().copied().map(char::from).collect() + }; + (name.to_string(), text) + }) + .collect() +} + +pub(crate) fn decoded(response: reqwest::Response) -> reqwest::Response { + let Some(decoder) = Decoder::detect(response.headers()) else { + return response; + }; + let status = response.status(); + let version = response.version(); + let headers = response.headers().clone(); + let body = futures_util::stream::unfold( + Some((response.bytes_stream().boxed(), decoder)), + |state| async move { + let (mut body, mut decoder) = state?; + loop { + match decoder.next() { + Ok(Some(decoded)) => return Some((Ok(decoded), Some((body, decoder)))), + Ok(None) => {} + Err(error) => return Some((Err(BoxError::from(error)), None)), + } + match body.next().await? { + Ok(chunk) => decoder.feed(&chunk), + Err(error) => return Some((Err(BoxError::from(error)), None)), + } + } + }, + ); + let mut rebuilt = http::Response::new(reqwest::Body::wrap_stream(body)); + *rebuilt.status_mut() = status; + *rebuilt.version_mut() = version; + *rebuilt.headers_mut() = headers; + reqwest::Response::from(rebuilt) +} + +#[cfg(test)] +mod tests { + use std::io::Write; + + use flate2::{Compression, write::GzEncoder, write::ZlibEncoder}; + use futures_util::TryStreamExt; + use rstest::rstest; + + use reqwest::header::{HeaderMap, HeaderValue}; + + use super::{decoded, header_pairs}; + + fn encoded(encoding: &str, body: &[u8]) -> Vec { + match encoding { + "gzip" => { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).unwrap(); + encoder.finish().unwrap() + } + "deflate" => { + let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).unwrap(); + encoder.finish().unwrap() + } + _ => body.to_vec(), + } + } + + fn response(encoding: &str, body: Vec) -> reqwest::Response { + let length = body.len().to_string(); + let mut response = http::Response::new(reqwest::Body::from(body)); + let headers = response.headers_mut(); + headers.insert("content-encoding", encoding.parse().unwrap()); + headers.insert("content-length", length.parse().unwrap()); + reqwest::Response::from(response) + } + + #[rstest] + #[case::gzip("gzip")] + #[case::deflate("deflate")] + #[case::unknown("br")] + #[tokio::test] + async fn decoded_bodies_keep_the_head_the_provider_sent(#[case] encoding: &str) { + let body = br#"{"type":"message"}"#; + let wire = encoded(encoding, body); + let length = wire.len().to_string(); + let response = decoded(response(encoding, wire.clone())); + + assert_eq!(response.headers()["content-encoding"], encoding); + assert_eq!(response.headers()["content-length"], length.as_str()); + let expected = if encoding == "br" { + wire + } else { + body.to_vec() + }; + assert_eq!( + response.bytes().await.unwrap().as_ref(), + expected.as_slice() + ); + } + + fn raw_deflate(body: &[u8]) -> Vec { + let mut encoder = flate2::write::DeflateEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).unwrap(); + encoder.finish().unwrap() + } + + fn chunked(encoding: &str, wire: Vec, size: usize) -> reqwest::Response { + let chunks: Vec, std::io::Error>> = + wire.chunks(size).map(|chunk| Ok(chunk.to_vec())).collect(); + let mut response = http::Response::new(reqwest::Body::wrap_stream( + futures_util::stream::iter(chunks), + )); + response + .headers_mut() + .insert("content-encoding", encoding.parse().unwrap()); + reqwest::Response::from(response) + } + + async fn read(response: reqwest::Response) -> Result, String> { + response + .bytes() + .await + .map(|bytes| bytes.to_vec()) + .map_err(|error| { + litellm_error::find::(&error) + .map_or_else(|| error.to_string(), ToString::to_string) + }) + } + + const BODY: &[u8] = br#"{"type":"message","content":[{"type":"text","text":"Paris"}]}"#; + + #[rstest] + #[case::padded_upper_case(" GZIP ", encoded("gzip", BODY))] + #[case::listed_with_an_unknown_coding("gzip, br", encoded("gzip", BODY))] + #[case::applied_twice("gzip, gzip", encoded("gzip", &encoded("gzip", BODY)))] + #[case::deflate_without_its_zlib_header("deflate", raw_deflate(BODY))] + #[case::gzip_with_data_past_its_end("gzip", [encoded("gzip", BODY), b"\x1f\x8b trailing".to_vec()].concat())] + #[tokio::test] + async fn bodies_decode_as_httpx_decodes_them( + #[case] encoding: &str, + #[case] wire: Vec, + #[values(7, 4096)] chunk: usize, + ) { + assert_eq!( + read(decoded(chunked(encoding, wire, chunk))).await.unwrap(), + BODY + ); + } + + #[rstest] + #[case::x_gzip_alias(" X-Gzip ")] + #[case::unknown_only("br, identity")] + #[tokio::test] + async fn codings_httpx_has_no_decoder_for_pass_through(#[case] encoding: &str) { + let wire = encoded("gzip", BODY); + assert_eq!( + read(decoded(chunked(encoding, wire.clone(), 4096))) + .await + .unwrap(), + wire + ); + } + + #[rstest] + #[case::not_deflate("deflate", b"not deflate at all".to_vec(), "Error -3 while decompressing data: invalid block type")] + #[case::bad_gzip_trailer( + "gzip", + { let wire = encoded("gzip", BODY); [&wire[..wire.len() - 8], &[0; 8]].concat() }, + "Error -3 while decompressing data: incorrect data check" + )] + #[case::not_gzip("gzip", b"\x1f\x8b\x08\x00not really gzip data at all".to_vec(), "Error -3 while decompressing data: invalid stored block lengths")] + #[case::no_gzip_magic("gzip", b"plain text".to_vec(), "Error -3 while decompressing data: incorrect header check")] + #[tokio::test] + async fn corrupt_bodies_fail_with_zlib_s_error( + #[case] encoding: &str, + #[case] wire: Vec, + #[case] expected: &str, + ) { + assert_eq!( + read(decoded(chunked(encoding, wire, 4096))) + .await + .unwrap_err(), + expected + ); + } + + #[rstest] + #[case::gzip_in_gzip("gzip, gzip")] + #[case::deflate_in_gzip("deflate, gzip")] + #[tokio::test] + async fn a_nested_bomb_is_decoded_in_bounded_reads(#[case] encoding: &str) { + let zeros = vec![0; 8 << 20]; + let inner = if encoding.starts_with("deflate") { + encoded("deflate", &zeros) + } else { + encoded("gzip", &zeros) + }; + let wire = encoded("gzip", &inner); + let sizes: Vec = decoded(chunked(encoding, wire, 4096)) + .bytes_stream() + .map_ok(|chunk| chunk.len()) + .try_collect() + .await + .unwrap(); + assert!( + sizes.iter().all(|size| *size <= super::MAX_DECOMPRESS_SIZE), + "{sizes:?}" + ); + assert_eq!(sizes.iter().sum::(), zeros.len()); + } + + #[rstest] + #[tokio::test] + async fn a_truncated_body_yields_what_it_holds_without_an_error() { + let wire = encoded("gzip", BODY); + let decoded_body = read(decoded(chunked( + "gzip", + wire[..wire.len() - 8].to_vec(), + 4096, + ))) + .await + .unwrap(); + assert_eq!(decoded_body, BODY); + } + + #[rstest] + #[case::ascii(&[b"plain".as_slice()], &["plain"])] + #[case::utf_8(&[b"plain".as_slice(), b"caf\xc3\xa9"], &["plain", "caf\u{e9}"])] + #[case::latin_1_once_any_value_is_not_utf_8( + &[b"caf\xc3\xa9".as_slice(), b"caf\xe9"], + &["caf\u{c3}\u{a9}", "caf\u{e9}"] + )] + fn header_values_decode_as_httpx_decodes_them( + #[case] values: &[&[u8]], + #[case] expected: &[&str], + ) { + let headers: HeaderMap = values + .iter() + .enumerate() + .map(|(index, value)| { + ( + format!("x-fz-{index}") + .parse::() + .unwrap(), + HeaderValue::from_bytes(value).unwrap(), + ) + }) + .collect(); + let decoded: Vec = header_pairs(&headers) + .into_iter() + .map(|(_, value)| value) + .collect(); + assert_eq!(decoded, expected); + } +} diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index e0a7aa5adb8..4796a640e25 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -18,7 +18,7 @@ use litellm_llms::base_llm::{ messages::{ context::MessagesTransformContext, streaming::{self, ByteStream, StreamDecoder}, - transformation::BaseMessagesConfig, + transformation::{BaseMessagesConfig, FederatedEnvironment}, }, }; use litellm_llms_types::formats::messages::{ @@ -26,16 +26,24 @@ use litellm_llms_types::formats::messages::{ }; use litellm_secrets::source::Secrets; use litellm_tracing::ByteChunk; -use serde_json::Value; +use serde_json::{Map, Value}; use super::{ - Error, MessagesCall, MessagesCallResponse, MessagesRoute, + Delivered, Error, HostAnswer, MessagesBody, MessagesCall, MessagesCallResponse, MessagesRoute, + Transport, TransportBody, TransportRequest, TransportResponse, UnencodableHeader, + WorkloadIdentity, common_utils::{MessagesProvider, merge_messages_header_sources, string_headers}, - constants::MESSAGES_TIMEOUT_SECS, + constants::{ + MESSAGES_HTTP_ERROR_ATTEMPTS, MESSAGES_STREAM_ERROR_BODY_READ_SECS, MESSAGES_TIMEOUT_SECS, + }, route::Messages, types::invalid_request, }; -use litellm_inference::{caching::CallCache, context::CallContext, outbound::outbound_request}; +use litellm_inference::{ + caching::{CacheRequest, CallCache}, + context::CallContext, + outbound::raw_leaf_outbound_request, +}; struct ProviderCall { identity: ProviderIdentity, @@ -44,6 +52,12 @@ struct ProviderCall { signer: Option, timeout: Option, stream: bool, + whole_sse_frames: bool, + joined_reads: bool, + transform_context: MessagesTransformContext, + ascii_response_headers: bool, + unencodable_request_header: Option>, + ascii_request_headers: bool, } impl ProviderCall { @@ -52,16 +66,122 @@ impl ProviderCall { } } +pub(super) async fn cached( + route: &MessagesRoute, + provider: MessagesProvider, + model: &str, + context: &CallContext<'_, impl Interceptors>, +) -> Result, Error> { + let Some(cache) = caller_keyed(route) else { + return Ok(None); + }; + let identity = ProviderIdentity { + model: model.to_owned(), + provider: provider.as_str().to_string(), + }; + let Some((output, source)) = CallCache::::new( + Some(cache), + context.cache, + CacheRequest::from_wire(identity.clone(), None), + ) + .lookup() + .await + else { + return Ok(None); + }; + context + .result_ready(execution_facts(provider, identity, &output, &source)) + .await?; + Ok(Some(output)) +} + +pub(super) async fn mocked( + route: &MessagesRoute, + provider: MessagesProvider, + model: String, + context: &CallContext<'_, impl Interceptors>, + mock: Map, + stream: bool, +) -> MessagesCallResponse { + let output = MessagesCallResponse::Complete(MessagesBody::Unchecked(mock)); + let Some(cache) = caller_keyed(route).filter(|_| !stream) else { + return output; + }; + let identity = ProviderIdentity { + model, + provider: provider.as_str().to_string(), + }; + CallCache::::new( + Some(cache), + context.cache, + CacheRequest::from_wire(identity, None), + ) + .finish(output, &ResultSource::Provider) + .await +} + +pub(super) async fn answered( + route: &MessagesRoute, + host_answer: &dyn HostAnswer, + provider: MessagesProvider, + model: String, + stream: bool, + context: &CallContext<'_, impl Interceptors>, +) -> Result { + host_answer.answer(stream).await?; + deliver_answer(route, provider, model, stream, context).await +} + +pub(super) async fn deliver_answer( + route: &MessagesRoute, + provider: MessagesProvider, + model: String, + stream: bool, + context: &CallContext<'_, impl Interceptors>, +) -> Result { + let identity = ProviderIdentity { + model, + provider: provider.as_str().to_string(), + }; + let cache = CallCache::::new( + caller_keyed(route), + context.cache, + CacheRequest::from_wire(identity.clone(), None), + ); + context + .result_ready(ExecutionFacts { + provider: identity, + source: ResultSource::Host, + cost: Cost::Deferred, + }) + .await?; + let output = MessagesCallResponse::Complete(MessagesBody::Unchecked(Map::new())); + if stream { + return Ok(output); + } + Ok(cache.finish(output, &ResultSource::Host).await) +} + +fn caller_keyed(route: &MessagesRoute) -> Option<&litellm_cache_response::ScopedCache> { + route + .cache + .as_ref() + .filter(|cache| cache.service.keyed_by_caller()) +} + pub(super) async fn execute( route: &MessagesRoute, provider: MessagesProvider, model: String, call: MessagesCall, secrets: Secrets, + requested_stream: bool, context: &CallContext<'_, impl Interceptors>, ) -> Result { + let caller_keyed = caller_keyed(route).is_some(); let request = provider_call( &route.auth, + route.workload_identity.as_deref(), provider, model, call, @@ -71,35 +191,64 @@ pub(super) async fn execute( .boxed() .await?; let cache = CallCache::::from_wire( - route.cache.as_ref().filter(|_| request.cacheable()), + route + .cache + .as_ref() + .filter(|_| caller_keyed || request.cacheable()), context.cache, &request.identity, &request.wire, ); let identity = request.identity.clone(); - let (output, source) = match cache.lookup().await { + let hit = match caller_keyed { + true => None, + false => cache.lookup().await, + }; + let (output, source) = match hit { Some(hit) => hit, None => ( - call_provider(&route.http, request, context).await?, + call_provider(&route.http, route.transport.as_deref(), request, context).await?, ResultSource::Provider, ), }; - let reported = match &output { - MessagesCallResponse::Complete(response) => provider.config().reported_cost(response), + context + .result_ready(execution_facts(provider, identity, &output, &source)) + .await?; + let output = match source { + ResultSource::Provider => { + super::safeguards::stamped(route.safeguards.as_ref(), output).await? + } + ResultSource::Cache { .. } | ResultSource::Host => output, + }; + if !requested_stream && matches!(output, MessagesCallResponse::Stream { .. }) { + return Ok(output); + } + Ok(cache.finish(output, &source).await) +} + +fn execution_facts( + provider: MessagesProvider, + identity: ProviderIdentity, + output: &MessagesCallResponse, + source: &ResultSource, +) -> ExecutionFacts { + let reported = match output { + MessagesCallResponse::Complete(response) => match response { + MessagesBody::Message(message) => provider.config().reported_cost(message), + MessagesBody::Unchecked(_) => None, + }, MessagesCallResponse::Stream { .. } => None, }; - context - .result_ready(ExecutionFacts { - provider: identity, - cost: Cost::settle(&source, reported), - source: source.clone(), - }) - .await?; - Ok(cache.finish(output, &source).await) + ExecutionFacts { + provider: identity, + cost: Cost::settle(source, reported), + source: source.clone(), + } } async fn provider_call( auth: &AuthServices, + workload_identity: Option<&dyn WorkloadIdentity>, provider: MessagesProvider, model: String, call: MessagesCall, @@ -126,13 +275,18 @@ async fn provider_call( let sanitized = config.shape_request( MessagesRequest { model, ..body }, - shaping.settings.reasoning_auto_summary, + shaping.settings.shape_options(), )?; - let transform_context = - MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params); - let filtered = requested_optional_params(sanitized, provider, &transform_context)?; + let transform_context = MessagesTransformContext { + drops_unsupported_params: shaping.settings.drops_unsupported_params, + ..MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params) + }; + let dropped_keys = top_level_paths(&shaping.settings.additional_drop_params); + let without_keys = without_additional_drop_params(sanitized, &dropped_keys)?; + let requested = requested_optional_params(without_keys, provider, &transform_context)?; + let requested_params = serde_json::to_value(&requested.params).map_err(serialize_failure)?; let trimmed = - without_additional_drop_params(filtered, &shaping.settings.additional_drop_params)?; + without_additional_drop_params(requested, &shaping.settings.additional_drop_params)?; let scoped = get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str()); @@ -140,40 +294,63 @@ async fn provider_call( extra_headers.unwrap_or_default(), scoped, ])))?; - let validated = config.validate_environment( + let validated = match config.validate_federated_environment( forwarded, api_key.as_deref(), &trimmed.model, &trimmed.params, &connection_params, &env_lookup, - )?; + )? { + FederatedEnvironment::Validated(validated) => validated, + FederatedEnvironment::Unauthenticated { + headers, + missing, + federate, + } => match workload_identity { + Some(identity) => match identity + .token(&connection_params, api_base.as_deref()) + .await? + { + Some(token) => federate(headers, token), + None => return Err(missing.into()), + }, + None => return Err(missing.into()), + }, + }; + let tool_custom_field = config.tool_custom_field(&trimmed.model); + let requested_output_format = trimmed.params.output_format.clone(); let body = config.request_body( - config.transform_anthropic_messages_request(trimmed, &transform_context)?, + tool_custom_field.apply( + config.transform_anthropic_messages_request(trimmed, &transform_context)?, + shaping.settings.drops_unsupported_params, + ), &validated.headers, shaping.settings.drop_params, )?; - let environment = ValidatedEnvironment { - headers: config.request_headers( - with_python_default_headers(validated.headers, config.default_headers()), - &body, - ), - auth: validated.auth, - }; - let url = config.get_complete_url( api_base.as_deref(), &body.model, &connection_params, - body.params.stream == Some(true), + body.params.streams(), &env_lookup, )?; + let environment = ValidatedEnvironment { + headers: config.headers_for_url( + config.request_headers( + with_python_default_headers(validated.headers, config.default_headers()), + &with_requested_output_format(&body, requested_output_format), + ), + &url, + ), + auth: validated.auth, + }; let request_context = RequestContext { model: body.model.clone(), custom_llm_provider: provider.as_str().to_string(), - optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?, + optional_params: requested_params, secret_fields: Vec::new(), api_key: api_key.map(SecretValue::new), }; @@ -193,19 +370,10 @@ async fn provider_call( request_context, ) .await?; - let stream = match wire.body.get("stream") { - None | Some(Value::Null) => false, - Some(Value::Bool(stream)) => *stream, - Some(value) => { - return Err(Error::invalid_request( - litellm_llms::ErrorDetail::InvalidValue { - field: "stream", - expected: "a boolean", - actual: value.clone(), - }, - )); - } - }; + let stream = wire + .body + .get("stream") + .is_some_and(litellm_llms_types::recognized::python_truthy); Ok(ProviderCall { identity, wire, @@ -213,9 +381,33 @@ async fn provider_call( signer: authenticated.signer, timeout: timeout.or_else(|| shaping.settings.request_timeout()), stream, + whole_sse_frames: shaping.settings.whole_sse_frames, + joined_reads: shaping.settings.joined_reads, + transform_context, + ascii_response_headers: shaping.settings.ascii_response_headers, + unencodable_request_header: shaping.settings.unencodable_request_header, + ascii_request_headers: shaping.settings.ascii_request_headers, }) } +fn with_requested_output_format( + body: &MessagesRequest, + requested: Option, +) -> std::borrow::Cow<'_, MessagesRequest> { + match requested { + Some(output_format) if body.params.output_format.is_none() => { + std::borrow::Cow::Owned(MessagesRequest { + params: MessagesOptionalParams { + output_format: Some(output_format), + ..body.params.clone() + }, + ..body.clone() + }) + } + _ => std::borrow::Cow::Borrowed(body), + } +} + /// Python sends its header dict through `httpx.Headers`, which folds names that differ only in /// case into one lowercase header whose values are joined with `", "`. fn joined_header_values(headers: Vec<(String, String)>) -> Vec<(String, String)> { @@ -274,6 +466,14 @@ fn requested_optional_params( }) } +fn top_level_paths(paths: &[String]) -> Vec { + paths + .iter() + .filter(|path| !path.contains(['.', '['])) + .cloned() + .collect() +} + fn without_additional_drop_params( request: MessagesRequest, paths: &[String], @@ -291,52 +491,57 @@ fn without_additional_drop_params( }) } +enum Upstream { + Native(reqwest::Response), + Hosted(TransportResponse), +} + async fn call_provider( http: &litellm_http::Client, + transport: Option<&dyn Transport>, request: ProviderCall, context: &CallContext<'_, impl Interceptors>, ) -> Result { + let timeout = request + .timeout + .unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS)); + let response = match send_recovering(http, transport, &request, timeout, context).await? { + Upstream::Native(response) => response, + Upstream::Hosted(response) => return hosted_response(response, request, context).await, + }; let ProviderCall { identity, - wire, provider, - signer, - timeout, stream, + whole_sse_frames, + joined_reads, + .. } = request; let provider_name = provider.as_str(); - log_request_body(provider_name, stream, &wire.body); - let timeout = timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS)); - let response = send( - http, - Authenticated { - headers: wire.headers, - signer, - }, - &wire.url, - &wire.body, - timeout, - stream, - ) - .await?; - if !response.status().is_success() { - return Err(provider_error(response, timeout).await); - } let config = provider.config(); + let head = ResponseHead { + status: response.status().as_u16(), + headers: super::encoding::header_pairs(response.headers()), + }; if stream { - return Ok(streaming_response( + context + .raw_response_received(RawResponse::new(String::new()).with_head(head)) + .await?; + let streamed = streaming_response( response, config.stream_decoder(), provider_name, timeout, - )); + joined_reads, + ); + return Ok(if whole_sse_frames { + framed(streamed) + } else { + streamed + }); } - let head = ResponseHead { - status: response.status().as_u16(), - headers: litellm_http::request::header_pairs(response.headers()), - }; let status = head.status; - let text = response.text().await.map_err(litellm_http::Error::from)?; + let text = read_body(response, timeout).await?; log_response_body(&text); context .raw_response_received(RawResponse::new(text.clone()).with_head(head)) @@ -346,8 +551,198 @@ async fn call_provider( // body it cannot parse, and the public mapping classifies it by that status. return Err(litellm_http::Error::upstream(status, text).into()); } - decode_response(config, &identity.model, &text) - .map(|message| MessagesCallResponse::Complete(Box::new(message))) + decode_response(config, &identity.model, &text).map(MessagesCallResponse::Complete) +} + +async fn hosted_response( + response: TransportResponse, + request: ProviderCall, + context: &CallContext<'_, impl Interceptors>, +) -> Result { + let TransportResponse { + status, + headers, + delivered, + } = response; + let ProviderCall { + identity, + provider, + whole_sse_frames, + .. + } = request; + let provider_name = provider.as_str(); + let config = provider.config(); + let text = match delivered { + Delivered::Complete(text) => text, + Delivered::Stream(chunks) => { + context + .raw_response_received(RawResponse::new(String::new()).with_head(ResponseHead { + status, + headers: headers.clone(), + })) + .await?; + let streamed = + hosted_streaming_response(headers, chunks, config.stream_decoder(), provider_name); + return Ok(if whole_sse_frames { + framed(streamed) + } else { + streamed + }); + } + Delivered::Rejected { error, .. } => return Err(error), + }; + log_response_body(&text); + context + .raw_response_received( + RawResponse::new(text.clone()).with_head(ResponseHead { status, headers }), + ) + .await?; + if serde_json::from_str::(&text).is_err() { + return Err(litellm_http::Error::upstream(status, text).into()); + } + decode_response(config, &identity.model, &text).map(MessagesCallResponse::Complete) +} + +fn hosted_request( + authenticated: Authenticated, + url: &str, + body: &Value, + stream: bool, +) -> Result { + if authenticated.signer.is_none() { + return Ok(TransportRequest { + url: url.to_owned(), + headers: authenticated.headers, + body: TransportBody::Json(body.clone()), + stream, + }); + } + let names: Vec = authenticated + .headers + .iter() + .map(|(name, _)| name.clone()) + .collect(); + let signed = raw_leaf_outbound_request(authenticated, url, body, None)?; + let headers = signed + .headers() + .iter() + .filter_map(|(name, value)| { + let spelled = names + .iter() + .find(|known| known.eq_ignore_ascii_case(name.as_str())) + .cloned() + .unwrap_or_else(|| name.to_string()); + Some((spelled, value.to_str().ok()?.to_owned())) + }) + .collect(); + let bytes = signed + .body() + .and_then(reqwest::Body::as_bytes) + .map(Bytes::copy_from_slice) + .unwrap_or_default(); + Ok(TransportRequest { + url: signed.url().to_string(), + headers, + body: TransportBody::Signed(bytes), + stream, + }) +} + +async fn send_recovering( + http: &litellm_http::Client, + transport: Option<&dyn Transport>, + request: &ProviderCall, + timeout: Duration, + context: &CallContext<'_, impl Interceptors>, +) -> Result { + let config = request.provider.config(); + let mut body = std::borrow::Cow::Borrowed(&request.wire.body); + let mut attempt = 1; + loop { + log_request_body(request.provider.as_str(), request.stream, &body); + if let Some(value) = &request.unencodable_request_header { + return Err(Error::transport(UnencodableHeader::Sent(value.clone()))); + } + // The credential header built from the caller's api_key never passes the caller header + // check, so it is checked on the wire headers as Python's client encodes them. + if request.ascii_request_headers + && let Some((_, value)) = request + .wire + .headers + .iter() + .find(|(_, value)| !value.is_ascii()) + { + return Err(Error::transport(UnencodableHeader::Sent( + value.as_bytes().to_vec(), + ))); + } + let authenticated = Authenticated { + headers: request.wire.headers.clone(), + signer: request.signer.clone(), + }; + let response = match transport { + None => Upstream::Native( + send(http, authenticated, &request.wire.url, &body, timeout).await?, + ), + Some(transport) => Upstream::Hosted( + transport + .send(hosted_request( + authenticated, + &request.wire.url, + &body, + request.stream, + )?) + .await?, + ), + }; + if let Upstream::Native(response) = &response + && request.ascii_response_headers + && let Some(unencodable) = UnencodableHeader::received(response.headers()) + { + return Err(Error::transport(unencodable)); + } + let failure = match response { + Upstream::Native(response) if response.status().is_success() => { + return Ok(Upstream::Native(response)); + } + Upstream::Native(response) => { + provider_failure(response, timeout, request.stream).await? + } + Upstream::Hosted(TransportResponse { + status, + headers, + delivered: Delivered::Rejected { text, error }, + }) => { + log_error_body(status, &text); + ProviderFailure { + status, + text, + headers, + raised: Some(error), + } + } + hosted @ Upstream::Hosted(_) => return Ok(hosted), + }; + let retried = if attempt == MESSAGES_HTTP_ERROR_ATTEMPTS { + None + } else { + config.retry_after_http_error( + failure.status, + &failure.text, + &body, + &request.transform_context, + ) + }; + let Some(retried) = retried else { + return Err(failure.into()); + }; + attempt += 1; + tracing::debug!( + "Anthropic /v1/messages: retrying after a recoverable HTTP error (attempt {attempt}/{MESSAGES_HTTP_ERROR_ATTEMPTS})." + ); + body = std::borrow::Cow::Owned(retried.body.clone()); + context.provider_request_retried(retried).await?; + } } fn serialize_failure(err: serde_json::Error) -> Error { @@ -357,71 +752,154 @@ fn serialize_failure(err: serde_json::Error) -> Error { )) } -/// Python's httpx timeout bounds each phase and each read, never the whole exchange. A complete -/// response keeps the client's overall deadline; a stream is bounded until its head arrives and -/// then per read (see [`timed_chunks`]), so a long stream that keeps sending is not cut off. async fn send( http: &litellm_http::Client, authenticated: Authenticated, url: &str, body: &Value, timeout: Duration, - stream: bool, ) -> Result { - let request = outbound_request(authenticated, url, body, (!stream).then_some(timeout))?; - if !stream { - return Ok(litellm_inference::outbound::send(request, http).await?); - } + let request = raw_leaf_outbound_request(authenticated, url, body, None)?; tokio::time::timeout(timeout, litellm_inference::outbound::send(request, http)) .await .map_err(litellm_http::Error::read_timeout)? + .map(super::encoding::decoded) .map_err(Error::from) } +async fn read_body( + mut response: reqwest::Response, + timeout: Duration, +) -> Result { + let expected = response.content_length(); + let mut body = Vec::new(); + while let Some(chunk) = tokio::time::timeout(timeout, response.chunk()) + .await + .map_err(litellm_http::Error::read_timeout)? + .map_err(|error| litellm_http::Error::body_read(error, body.len() as u64, expected))? + { + body.extend_from_slice(&chunk); + } + Ok(String::from_utf8_lossy(&body).into_owned()) +} + /// The stream's body, each read bounded by `timeout` as httpx bounds each read. fn timed_chunks( response: reqwest::Response, timeout: Duration, ) -> impl futures_util::Stream> + Send + 'static { - futures_util::stream::try_unfold(response, move |mut response| async move { - let chunk = tokio::time::timeout(timeout, response.chunk()) - .await - .map_err(litellm_http::Error::body_read_timeout)? - .map_err(litellm_http::Error::from)?; - Ok(chunk.map(|chunk| (chunk, response))) - }) + let expected = response.content_length(); + futures_util::stream::try_unfold( + (response, 0_u64), + move |(mut response, received)| async move { + let chunk = tokio::time::timeout(timeout, response.chunk()) + .await + .map_err(litellm_http::Error::body_read_timeout)? + .map_err(|error| litellm_http::Error::body_read(error, received, expected))?; + Ok(chunk.map(|chunk| { + let received = received + chunk.len() as u64; + (chunk, (response, received)) + })) + }, + ) } -/// The provider's error, its body read under `timeout`: a stream has no overall deadline once -/// its head arrives, so an error body that stalls would otherwise hold the call forever. -async fn provider_error(response: reqwest::Response, timeout: Duration) -> Error { - let status = response.status().as_u16(); - let headers = litellm_http::request::header_pairs(response.headers()); - match tokio::time::timeout(timeout, response.text()).await { - Ok(Ok(text)) => { - log_error_body(status, &text); - litellm_http::Error::upstream_with_headers(status, text, headers).into() +struct ProviderFailure { + status: u16, + text: String, + headers: Vec<(String, String)>, + raised: Option, +} + +impl From for Error { + fn from(failure: ProviderFailure) -> Self { + match failure.raised { + Some(raised) => raised, + None => litellm_http::Error::upstream_with_headers( + failure.status, + failure.text, + failure.headers, + ) + .into(), } - Ok(Err(error)) => litellm_http::Error::from(error).into(), - Err(elapsed) => litellm_http::Error::read_timeout(elapsed).into(), } } +async fn provider_failure( + response: reqwest::Response, + timeout: Duration, + stream: bool, +) -> Result { + let status = response.status().as_u16(); + let headers = super::encoding::header_pairs(response.headers()); + let text = if stream { + tokio::time::timeout( + Duration::from_secs(MESSAGES_STREAM_ERROR_BODY_READ_SECS), + read_body(response, timeout), + ) + .await + .ok() + .and_then(Result::ok) + .unwrap_or_default() + } else { + read_body(response, timeout).await? + }; + log_error_body(status, &text); + Ok(ProviderFailure { + status, + text, + headers, + raised: None, + }) +} + fn decode_response( config: &dyn BaseMessagesConfig, model: &str, text: &str, -) -> Result { - let response = serde_json::from_str(text).map_err(|err| { - Error::invalid_response(litellm_llms::ErrorDetail::invalid( - "messages response JSON", - err, - )) - })?; - config +) -> Result { + let marked = litellm_host::raw_json::mark_wide_numbers(text); + let response = match serde_json::from_str::(&marked) { + Ok(response) => response, + Err(err) => { + let Ok(mut body) = serde_json::from_str::>(&marked) else { + return Err(Error::invalid_response(litellm_llms::ErrorDetail::invalid( + "messages response JSON", + err, + ))); + }; + body.entry("stop_sequence").or_insert(Value::Null); + return Ok(MessagesBody::Unchecked(body)); + } + }; + let message = config .transform_anthropic_messages_response(model, response) .map_err(Error::from) - .map_err(Error::after_send) + .map_err(Error::after_send)?; + if text_has_stop_reason(text) { + return Ok(MessagesBody::Message(Box::new(message))); + } + match serde_json::to_value(message).map_err(serialize_failure)? { + Value::Object(mut body) => { + body.remove("stop_reason"); + Ok(MessagesBody::Unchecked(body)) + } + _ => unreachable!("a message serializes as a JSON object"), + } +} + +#[derive(serde::Deserialize)] +struct StopReasonKey { + #[serde(default, deserialize_with = "key_present")] + stop_reason: bool, +} + +fn key_present<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + >::deserialize(deserializer).map(|_| true) +} + +fn text_has_stop_reason(text: &str) -> bool { + serde_json::from_str::(text).is_ok_and(|key| key.stop_reason) } fn streaming_response( @@ -429,14 +907,15 @@ fn streaming_response( decoder: Option, provider: &'static str, timeout: Duration, + joined: bool, ) -> MessagesCallResponse { - let headers = response - .headers() - .iter() - .filter_map(|(name, value)| Some((name.to_string(), value.to_str().ok()?.to_string()))) - .collect(); - let chunks = timed_chunks(response, timeout) - .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk)); + let headers = super::encoding::header_pairs(response.headers()); + let chunks = timed_chunks(response, timeout); + let chunks = match (joined, &decoder) { + (true, None) => super::reads::joined_reads(chunks), + _ => chunks.boxed(), + } + .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk)); let chunks = match decoder { None => chunks.map_err(Error::from).boxed(), Some(decode) => decoded_chunks( @@ -451,6 +930,73 @@ fn streaming_response( } } +fn hosted_streaming_response( + headers: Vec<(String, String)>, + chunks: BoxStream<'static, Result>, + decoder: Option, + provider: &'static str, +) -> MessagesCallResponse { + let chunks = chunks.inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk)); + let chunks = match decoder { + None => chunks.boxed(), + Some(decode) => decoded_chunks( + chunks.map_err(std::io::Error::other).boxed(), + decode, + provider, + ), + }; + MessagesCallResponse::Stream { + head: super::route::MessagesStreamHead { headers }, + chunks, + } +} + +fn framed(response: MessagesCallResponse) -> MessagesCallResponse { + let MessagesCallResponse::Stream { head, chunks } = response else { + return response; + }; + MessagesCallResponse::Stream { + head, + chunks: whole_sse_frames(chunks), + } +} + +fn whole_sse_frames( + chunks: BoxStream<'static, Result>, +) -> BoxStream<'static, Result> { + futures_util::stream::try_unfold( + (Some(chunks), Vec::new()), + |(chunks, mut pending)| async move { + let Some(mut chunks) = chunks else { + return Ok(None); + }; + while let Some(chunk) = chunks.try_next().await? { + pending.extend_from_slice(&chunk); + let end = complete_frames_end(&pending); + if end > 0 { + let rest = pending.split_off(end); + return Ok(Some((Bytes::from(pending), (Some(chunks), rest)))); + } + } + Ok((!pending.is_empty()).then(|| (Bytes::from(pending), (None, Vec::new())))) + }, + ) + .boxed() +} + +fn complete_frames_end(pending: &[u8]) -> usize { + [b"\r\n\r\n".as_slice(), b"\n\n", b"\r\r"] + .iter() + .filter_map(|delimiter| { + pending + .windows(delimiter.len()) + .rposition(|window| window == *delimiter) + .map(|at| at + delimiter.len()) + }) + .max() + .unwrap_or(0) +} + fn decoded_chunks( bytes: ByteStream, decode: StreamDecoder, @@ -511,6 +1057,14 @@ mod tests { async fn provider_call_with_secrets( call: MessagesCall, secrets: &(dyn Lookup + Send + Sync), + ) -> Result { + provider_call_federating(call, secrets, None).await + } + + async fn provider_call_federating( + call: MessagesCall, + secrets: &(dyn Lookup + Send + Sync), + workload_identity: Option<&dyn WorkloadIdentity>, ) -> Result { let provider = resolve_llm_provider( &call.body.model, @@ -524,6 +1078,7 @@ mod tests { let context = CallContext::new(&interceptors, litellm_inference::CallOptions::default()); provider_call( &AuthServices::default(), + workload_identity, config, model, call, @@ -717,6 +1272,178 @@ mod tests { assert!(error.is_missing_credential(), "{error:?}"); } + struct ScriptedIdentity { + minted: Result, ()>, + asked: std::sync::Mutex, Option)>>, + } + + impl ScriptedIdentity { + fn new(minted: Result, ()>) -> Self { + Self { + minted, + asked: std::sync::Mutex::new(Vec::new()), + } + } + + fn asked(&self) -> Vec<(Option, Option)> { + self.asked.lock().unwrap().clone() + } + } + + impl WorkloadIdentity for ScriptedIdentity { + fn token<'a>( + &'a self, + litellm_params: &'a litellm_router_types::LitellmParams, + api_base: Option<&'a str>, + ) -> futures_util::future::BoxFuture<'a, Result, litellm_auth::Error>> + { + self.asked.lock().unwrap().push(( + litellm_params + .extra + .get("anthropic_federation_rule_id") + .cloned(), + api_base.map(str::to_owned), + )); + let minted = self.minted; + Box::pin(async move { + minted + .map(|token| token.map(SecretValue::new)) + .map_err(|()| litellm_auth::Error::InvalidHeader) + }) + } + } + + fn federated_call( + shaping: MessagesShaping, + api_key: Option<&str>, + extra_headers: Option, + ) -> MessagesCall { + MessagesCall { + body: body( + json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}), + ), + api_key: api_key.map(str::to_owned), + api_base: Some("https://anthropic.test".into()), + custom_llm_provider: Some("anthropic".into()), + litellm_params: crate::litellm_params( + serde_json::from_value( + json!({"model": "claude-test", "anthropic_federation_rule_id": "fdrl_1"}), + ) + .unwrap(), + ) + .unwrap(), + extra_headers: extra_headers.map(|headers| serde_json::from_value(headers).unwrap()), + provider_specific_header: None, + timeout: None, + shaping, + } + } + + fn sent_headers<'a>(call: &'a ProviderCall, names: &[&str]) -> Vec<(&'a str, &'a str)> { + call.wire + .headers + .iter() + .filter(|(name, _)| names.contains(&name.as_str())) + .map(|(name, value)| (name.as_str(), value.as_str())) + .collect() + } + + #[rstest] + #[case::alone(None, &[("anthropic-beta", "oauth-2025-04-20"), ("authorization", "Bearer sk-ant-oat01-minted")])] + #[case::beside_a_caller_beta( + Some(json!({"anthropic-beta": "files-api-2025-04-14"})), + &[("anthropic-beta", "files-api-2025-04-14,oauth-2025-04-20"), ("authorization", "Bearer sk-ant-oat01-minted")] + )] + #[tokio::test] + async fn a_minted_federation_token_is_sent_as_an_oauth_bearer( + shaping: MessagesShaping, + #[case] extra_headers: Option, + #[case] expected: &[(&str, &str)], + ) { + let identity = ScriptedIdentity::new(Ok(Some("sk-ant-oat01-minted"))); + let call = provider_call_federating( + federated_call(shaping, None, extra_headers), + &|_: &str| None, + Some(&identity), + ) + .await + .unwrap(); + let mut sent = sent_headers( + &call, + &[ + "anthropic-beta", + "authorization", + "x-api-key", + "anthropic-dangerous-direct-browser-access", + ], + ); + sent.sort_unstable(); + assert_eq!( + (sent.as_slice(), identity.asked()), + ( + expected, + vec![(Some(json!("fdrl_1")), Some("https://anthropic.test".into()))] + ) + ); + } + + #[rstest] + #[case::static_key(Some("sk-static"), None, &[("x-api-key", "sk-static")])] + #[case::caller_credential( + None, + Some(json!({"x-api-key": "sk-caller"})), + &[("x-api-key", "sk-caller")] + )] + #[tokio::test] + async fn a_configured_credential_is_never_federated( + shaping: MessagesShaping, + #[case] api_key: Option<&str>, + #[case] extra_headers: Option, + #[case] expected: &[(&str, &str)], + ) { + let identity = ScriptedIdentity::new(Ok(Some("sk-ant-oat01-minted"))); + let call = provider_call_federating( + federated_call(shaping, api_key, extra_headers), + &|_: &str| None, + Some(&identity), + ) + .await + .unwrap(); + assert_eq!( + ( + sent_headers(&call, &["authorization", "x-api-key"]), + identity.asked() + ), + (expected.to_vec(), Vec::new()) + ); + } + + #[rstest] + #[case::nothing_minted(Ok(None), true)] + #[case::minting_failed(Err(()), false)] + #[tokio::test] + async fn an_unminted_federation_token_fails_the_call( + shaping: MessagesShaping, + #[case] minted: Result, ()>, + #[case] missing_credential: bool, + ) { + let identity = ScriptedIdentity::new(minted); + let Err(error) = provider_call_federating( + federated_call(shaping, None, None), + &|_: &str| None, + Some(&identity), + ) + .await + else { + panic!("a call without a minted token fails"); + }; + assert_eq!( + error.is_missing_credential(), + missing_credential, + "{error:?}" + ); + } + #[rstest] #[case::nothing_configured(None, None, None)] #[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))] @@ -1019,7 +1746,7 @@ mod tests { .await .unwrap(); assert_eq!(call.wire.url, format!("{base}{path}")); - let signed = outbound_request( + let signed = raw_leaf_outbound_request( Authenticated { headers: call.wire.headers, signer: call.signer, @@ -1063,6 +1790,7 @@ mod tests { Some(streaming::event_stream), "test", Duration::from_secs(5), + false, ) else { panic!("a streaming response returns chunks"); }; @@ -1074,4 +1802,48 @@ mod tests { } assert!(chunks.next().await.is_none()); } + + #[rstest] + #[case::no_delimiter(b"data: a".as_slice(), 0)] + #[case::lf(b"data: a\n\ndata: b", 9)] + #[case::crlf(b"data: a\r\n\r\ndata: b", 11)] + #[case::cr(b"data: a\r\rdata: b", 9)] + #[case::last_of_several(b"data: a\n\ndata: b\n\ndata: c", 18)] + fn complete_frames_end_is_after_the_last_delimiter(#[case] pending: &[u8], #[case] end: usize) { + assert_eq!(complete_frames_end(pending), end); + } + + #[rstest] + #[case::split_frame( + vec![&b"data: a\n"[..], b"\ndata: ", b"b\n\n"], + vec![&b"data: a\n\n"[..], b"data: b\n\n"] + )] + #[case::several_frames_in_one_chunk( + vec![&b"data: a\n\ndata: b\n\nda"[..], b"ta: c\n\n"], + vec![&b"data: a\n\ndata: b\n\n"[..], b"data: c\n\n"] + )] + #[case::unterminated_tail_is_flushed( + vec![&b"data: a\n\nda"[..], b"ta: b"], + vec![&b"data: a\n\n"[..], b"data: b"] + )] + #[tokio::test] + async fn whole_sse_frames_regroups_chunks_at_frame_boundaries( + #[case] chunks: Vec<&'static [u8]>, + #[case] expected: Vec<&'static [u8]>, + ) { + let upstream = futures_util::stream::iter( + chunks + .into_iter() + .map(|chunk| Ok::<_, Error>(Bytes::from_static(chunk))), + ) + .boxed(); + let framed: Vec = whole_sse_frames(upstream).try_collect().await.unwrap(); + assert_eq!( + framed, + expected + .into_iter() + .map(Bytes::from_static) + .collect::>() + ); + } } diff --git a/litellm-rust/crates/inference-messages/src/host_answer.rs b/litellm-rust/crates/inference-messages/src/host_answer.rs new file mode 100644 index 00000000000..07d183406ea --- /dev/null +++ b/litellm-rust/crates/inference-messages/src/host_answer.rs @@ -0,0 +1,11 @@ +use futures_util::future::BoxFuture; + +use crate::Error; + +pub trait HostAnswer: Send + Sync { + fn answer(&self, stream: bool) -> BoxFuture<'_, Result<(), Error>>; + + fn intercept(&self, stream: bool) -> BoxFuture<'_, Result> { + Box::pin(async move { self.answer(stream).await.map(|()| true) }) + } +} diff --git a/litellm-rust/crates/inference-messages/src/lib.rs b/litellm-rust/crates/inference-messages/src/lib.rs index 745442d0314..cd312ee61b1 100644 --- a/litellm-rust/crates/inference-messages/src/lib.rs +++ b/litellm-rust/crates/inference-messages/src/lib.rs @@ -1,25 +1,41 @@ mod common_utils; mod constants; +mod encoding; mod handler; +mod host_answer; +mod pre_request; +mod reads; pub mod route; +mod safeguards; +mod transport; mod types; +mod workload_identity; use futures_util::FutureExt; use litellm_auth::AuthServices; use litellm_host::interceptors::Interceptors; use litellm_inference::context::CallContext; use litellm_inference::provider::resolve_llm_provider; +use litellm_llms_types::recognized::Recognized; use litellm_secrets::source::SecretSource; use std::sync::Arc; use crate::common_utils::messages_provider; pub use common_utils::merge_messages_header_sources; +pub use encoding::DecodingError; +pub use host_answer::HostAnswer; pub use litellm_inference::Error; -pub use types::{ - MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params, - messages_body, +pub use pre_request::PreRequestHooks; +pub use safeguards::Safeguards; +pub use transport::{ + Delivered, Transport, TransportBody, TransportRequest, TransportResponse, UnencodableHeader, }; +pub use types::{ + MessagesBody, MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, + litellm_params, messages_body, +}; +pub use workload_identity::WorkloadIdentity; #[derive(Clone)] pub struct MessagesRoute { @@ -27,6 +43,11 @@ pub struct MessagesRoute { auth: Arc, secrets: Arc, cache: Option, + workload_identity: Option>, + transport: Option>, + pre_request_hooks: Option>, + safeguards: Option>, + host_answer: Option>, } impl MessagesRoute { @@ -40,6 +61,51 @@ impl MessagesRoute { auth, secrets, cache: None, + workload_identity: None, + transport: None, + pre_request_hooks: None, + safeguards: None, + host_answer: None, + } + } + + #[must_use] + pub fn with_workload_identity(self, workload_identity: Arc) -> Self { + Self { + workload_identity: Some(workload_identity), + ..self + } + } + + #[must_use] + pub fn with_transport(self, transport: Arc) -> Self { + Self { + transport: Some(transport), + ..self + } + } + + #[must_use] + pub fn with_pre_request_hooks(self, hooks: Arc) -> Self { + Self { + pre_request_hooks: Some(hooks), + ..self + } + } + + #[must_use] + pub fn with_safeguards(self, safeguards: Arc) -> Self { + Self { + safeguards: Some(safeguards), + ..self + } + } + + #[must_use] + pub fn with_host_answer(self, host_answer: Arc) -> Self { + Self { + host_answer: Some(host_answer), + ..self } } @@ -67,7 +133,7 @@ impl MessagesRoute { model = %call.body.model, provider, resolved_model, - stream = call.body.params.stream == Some(true), + stream = call.body.params.streams(), outcome ))] async fn run( @@ -76,24 +142,69 @@ impl MessagesRoute { context: CallContext<'_, impl Interceptors>, ) -> Result { litellm_inference::diagnostic::call(async { - let provider = resolve_llm_provider( - &call.body.model, - call.custom_llm_provider.as_deref(), - "messages", - )?; - let config = messages_provider(provider.provider, provider.model).ok_or_else(|| { - Error::InvalidProvider(<&str>::from(provider.provider).to_string()) - })?; - litellm_inference::diagnostic::provider( - provider.model, - <&'static str>::from(provider.provider), - ); - let model = provider.model.to_owned(); + let requested_stream = call.body.params.streams(); + let client_streams = call.body.params.stream == Some(Recognized::Known(true)); + let call = match &self.pre_request_hooks { + None => call, + Some(hooks) => { + let (config, model, _) = Self::provider(&call)?; + if let Some(output) = handler::cached(self, config, &model, &context).await? { + return Ok(output); + } + hooks.run(call).await? + } + }; + let (config, model, provider) = Self::provider(&call)?; + litellm_inference::diagnostic::provider(&model, provider); + if self.pre_request_hooks.is_none() + && let Some(output) = handler::cached(self, config, &model, &context).await? + { + return Ok(output); + } + if let Some(host_answer) = self + .host_answer + .as_deref() + .filter(|_| call.shaping.settings.intercepted) + && host_answer.intercept(requested_stream).await? + { + return handler::deliver_answer(self, config, model, requested_stream, &context) + .await; + } + if let Some(mock) = call.shaping.settings.mock_response { + return Ok( + handler::mocked(self, config, model, &context, mock, requested_stream).await, + ); + } + if let Some(host_answer) = self + .host_answer + .as_deref() + .filter(|_| call.shaping.settings.mcp_gateway) + { + let stream = call.body.params.streams(); + return handler::answered(self, host_answer, config, model, stream, &context).await; + } let secrets = self.secrets.resolve(config.config().secret_names()).await?; - handler::execute(self, config, model, call, secrets, &context) + handler::execute(self, config, model, call, secrets, client_streams, &context) .boxed() .await }) .await } + + fn provider( + call: &MessagesCall, + ) -> Result<(common_utils::MessagesProvider, String, &'static str), Error> { + let provider = resolve_llm_provider( + &call.body.model, + call.custom_llm_provider.as_deref(), + "messages", + )?; + let config = messages_provider(provider.provider, provider.model) + .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?; + Ok(( + config, + provider.model.to_owned(), + <&'static str>::from(provider.provider), + )) + } } diff --git a/litellm-rust/crates/inference-messages/src/pre_request.rs b/litellm-rust/crates/inference-messages/src/pre_request.rs new file mode 100644 index 00000000000..949dcd6585b --- /dev/null +++ b/litellm-rust/crates/inference-messages/src/pre_request.rs @@ -0,0 +1,7 @@ +use futures_util::future::BoxFuture; + +use super::{Error, MessagesCall}; + +pub trait PreRequestHooks: Send + Sync { + fn run(&self, call: MessagesCall) -> BoxFuture<'_, Result>; +} diff --git a/litellm-rust/crates/inference-messages/src/reads.rs b/litellm-rust/crates/inference-messages/src/reads.rs new file mode 100644 index 00000000000..0e127a42a13 --- /dev/null +++ b/litellm-rust/crates/inference-messages/src/reads.rs @@ -0,0 +1,394 @@ +use std::{collections::VecDeque, sync::Arc}; + +use bytes::Bytes; +use futures_util::{FutureExt, Stream, StreamExt, stream::BoxStream}; +use std::sync::Mutex; +use tokio::{sync::Notify, task::JoinHandle}; + +const READ_SIZE: usize = 16 * 1024; +const LOW_WATER: usize = 64 * 1024; +const HIGH_WATER: usize = 2 * LOW_WATER; +const SETTLE_YIELDS: usize = 64; + +struct Buffer { + chunks: VecDeque, + size: usize, + quiet: bool, + failure: Option, + ended: bool, +} + +impl Buffer { + fn take(&mut self) -> Bytes { + let mut joined = Vec::new(); + while let Some(mut chunk) = self.chunks.pop_front() { + let room = READ_SIZE - joined.len(); + if chunk.len() > room { + let rest = chunk.split_off(room); + self.chunks.push_front(rest); + } + if joined.is_empty() && (chunk.len() == READ_SIZE || self.chunks.is_empty()) { + self.size -= chunk.len(); + return chunk; + } + joined.extend_from_slice(&chunk); + self.size -= chunk.len(); + if joined.len() == READ_SIZE { + break; + } + } + Bytes::from(joined) + } +} + +struct Reads { + buffer: Mutex>, + received: Notify, + drained: Notify, +} + +enum Read { + Chunk(Bytes), + Failure(E), + End, +} + +impl Reads { + fn readable(&self) -> bool { + let buffer = self.buffer(); + buffer.failure.is_some() + || buffer.ended + || (!buffer.chunks.is_empty() && (buffer.quiet || buffer.size >= READ_SIZE)) + } + + fn quiet(&self) { + self.buffer().quiet = true; + self.received.notify_one(); + } + + fn read(&self) -> Option> { + let mut buffer = self.buffer(); + if !buffer.chunks.is_empty() { + let chunk = buffer.take(); + drop(buffer); + self.drained.notify_one(); + return Some(Read::Chunk(chunk)); + } + if let Some(failure) = buffer.failure.take() { + return Some(Read::Failure(failure)); + } + buffer.ended.then_some(Read::End) + } + + fn buffer(&self) -> std::sync::MutexGuard<'_, Buffer> { + self.buffer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +struct Reader(JoinHandle<()>); + +impl Drop for Reader { + fn drop(&mut self) { + self.0.abort(); + } +} + +pub(crate) fn joined_reads( + chunks: impl Stream> + Send + 'static, +) -> BoxStream<'static, Result> { + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return chunks.boxed(); + }; + let reads = Arc::new(Reads { + buffer: Mutex::new(Buffer { + chunks: VecDeque::new(), + size: 0, + quiet: false, + failure: None, + ended: false, + }), + received: Notify::new(), + drained: Notify::new(), + }); + let reader = Reader(runtime.spawn(fill(chunks.boxed(), Arc::clone(&reads)))); + futures_util::stream::unfold(Some((reads, reader)), |state| async move { + let (reads, reader) = state?; + loop { + let received = reads.received.notified(); + if reads.readable() { + break; + } + received.await; + } + let read = reads.read()?; + match read { + Read::Chunk(chunk) => Some((Ok(chunk), Some((reads, reader)))), + Read::Failure(failure) => Some((Err(failure), None)), + Read::End => None, + } + }) + .boxed() +} + +async fn fill(mut chunks: BoxStream<'static, Result>, reads: Arc>) { + loop { + let next = match settled_next(&mut chunks).await { + Some(next) => next, + None => { + reads.quiet(); + chunks.next().await + } + }; + let paused = { + let mut buffer = reads.buffer(); + buffer.quiet = false; + match next { + Some(Ok(chunk)) => { + buffer.size += chunk.len(); + if !chunk.is_empty() { + buffer.chunks.push_back(chunk); + } + } + Some(Err(failure)) => { + buffer.failure = Some(failure); + buffer.ended = true; + } + None => buffer.ended = true, + } + if buffer.ended { + drop(buffer); + reads.received.notify_one(); + return; + } + buffer.size > HIGH_WATER + }; + reads.received.notify_one(); + if paused { + reads.quiet(); + loop { + let drained = reads.drained.notified(); + if reads.buffer().size < LOW_WATER { + break; + } + drained.await; + } + } + } +} + +async fn settled_next( + chunks: &mut BoxStream<'static, Result>, +) -> Option>> { + for _ in 0..SETTLE_YIELDS { + if let Some(next) = chunks.next().now_or_never() { + return Some(next); + } + tokio::task::yield_now().await; + } + None +} + +#[cfg(test)] +mod tests { + use std::{ + pin::Pin, + sync::atomic::{AtomicUsize, Ordering}, + task::{Context, Poll}, + time::Duration, + }; + + use futures_util::{FutureExt, StreamExt, TryStreamExt}; + use tokio::sync::mpsc; + + use super::*; + + struct Upstream { + receiver: mpsc::UnboundedReceiver>, + pulled: Arc, + } + + impl Stream for Upstream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let next = self.receiver.poll_recv(cx); + if let Poll::Ready(Some(_)) = next { + self.pulled.fetch_add(1, Ordering::SeqCst); + } + next + } + } + + type Item = Result; + + fn upstream() -> ( + mpsc::UnboundedSender, + BoxStream<'static, Item>, + Arc, + ) { + let (sender, receiver) = mpsc::unbounded_channel(); + let pulled = Arc::new(AtomicUsize::new(0)); + let reads = joined_reads(Upstream { + receiver, + pulled: Arc::clone(&pulled), + }); + (sender, reads, pulled) + } + + async fn settled() { + tokio::time::sleep(Duration::from_millis(20)).await; + } + + #[rstest::rstest] + #[tokio::test] + async fn chunks_that_arrived_together_are_read_as_one() { + let (sender, mut reads, _) = upstream(); + sender.send(Ok(Bytes::from("event: a\n\n"))).unwrap(); + sender.send(Ok(Bytes::from("event: b\n\n"))).unwrap(); + sender.send(Ok(Bytes::from("event: c\n\n"))).unwrap(); + settled().await; + assert_eq!( + reads.try_next().await.unwrap().unwrap(), + "event: a\n\nevent: b\n\nevent: c\n\n" + ); + sender.send(Ok(Bytes::from("event: d\n\n"))).unwrap(); + settled().await; + assert_eq!(reads.try_next().await.unwrap().unwrap(), "event: d\n\n"); + drop(sender); + assert!(reads.try_next().await.unwrap().is_none()); + } + + #[rstest::rstest] + #[tokio::test] + async fn chunks_still_being_handed_over_join_the_read() { + let handed_over = futures_util::stream::unfold(0, |sent| async move { + tokio::task::yield_now().await; + (sent < 50).then(|| (Ok::<_, &str>(Bytes::from("event: ping\n\n")), sent + 1)) + }); + let read: Vec = joined_reads(handed_over) + .map_ok(|chunk| chunk.len()) + .try_collect() + .await + .unwrap(); + assert_eq!(read, [50 * "event: ping\n\n".len()]); + } + + #[rstest::rstest] + #[tokio::test] + async fn a_read_returns_the_first_chunk_without_waiting_for_more() { + let (sender, mut reads, _) = upstream(); + let first = tokio::spawn(async move { reads.try_next().await }); + settled().await; + sender.send(Ok(Bytes::from("event: a\n\n"))).unwrap(); + assert_eq!(first.await.unwrap().unwrap().unwrap(), "event: a\n\n"); + } + + #[rstest::rstest] + #[tokio::test] + async fn reads_are_cut_at_sixteen_kibibytes() { + let (sender, reads, _) = upstream(); + sender.send(Ok(Bytes::from(vec![b'a'; 10_000]))).unwrap(); + sender.send(Ok(Bytes::from(vec![b'b'; 30_000]))).unwrap(); + drop(sender); + settled().await; + let sizes: Vec = reads + .map_ok(|chunk| chunk.len()) + .try_collect() + .await + .unwrap(); + assert_eq!(sizes, [READ_SIZE, READ_SIZE, 40_000 - 2 * READ_SIZE]); + } + + #[rstest::rstest] + #[tokio::test] + async fn the_bytes_still_buffered_are_read_before_a_failure() { + let (sender, mut reads, _) = upstream(); + sender.send(Ok(Bytes::from("event: a\n\n"))).unwrap(); + sender.send(Ok(Bytes::from("event: b\n\n"))).unwrap(); + sender.send(Err("reset")).unwrap(); + settled().await; + assert_eq!( + reads.try_next().await.unwrap().unwrap(), + "event: a\n\nevent: b\n\n" + ); + assert_eq!(reads.try_next().await.unwrap_err(), "reset"); + assert!(reads.next().await.is_none()); + } + + #[rstest::rstest] + #[tokio::test] + async fn reading_pauses_above_the_high_water_mark_until_below_the_low_one() { + let (sender, mut reads, pulled) = upstream(); + std::iter::repeat_with(|| Bytes::from(vec![b'a'; READ_SIZE])) + .take(12) + .try_for_each(|chunk| sender.send(Ok(chunk))) + .unwrap(); + settled().await; + assert_eq!(pulled.load(Ordering::SeqCst), 9); + let read: Vec = (&mut reads).take(5).try_collect().await.unwrap(); + assert_eq!(read.len(), 5); + settled().await; + assert_eq!(pulled.load(Ordering::SeqCst), 9); + reads.try_next().await.unwrap(); + settled().await; + assert_eq!(pulled.load(Ordering::SeqCst), 12); + } + + async fn chunked_upstream(frames: usize) -> String { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = [0_u8; 4096]; + let _ = socket.read(&mut request).await.unwrap(); + let frame = "event: ping\ndata: {\"type\": \"ping\"}\n\n"; + let body: String = + std::iter::repeat_n(format!("{:x}\r\n{frame}\r\n", frame.len()), frames).collect(); + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ntransfer-encoding: chunked\r\nconnection: close\r\n\r\n{body}0\r\n\r\n" + ); + socket.write_all(response.as_bytes()).await.unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + }); + format!("http://{address}") + } + + #[rstest::rstest] + #[case::fifty(50)] + #[case::a_thousand(1000)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn http_chunks_written_together_are_read_together(#[case] frames: usize) { + let url = chunked_upstream(frames).await; + let response = litellm_http::Client::for_test() + .get(url) + .send() + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + let sizes: Vec = joined_reads(response.bytes_stream()) + .map_ok(|chunk| chunk.len()) + .try_collect() + .await + .unwrap(); + let total = frames * "event: ping\ndata: {\"type\": \"ping\"}\n\n".len(); + let expected: Vec = std::iter::repeat_n(READ_SIZE, total / READ_SIZE) + .chain((!total.is_multiple_of(READ_SIZE)).then_some(total % READ_SIZE)) + .collect(); + assert_eq!(sizes, expected); + } + + #[rstest::rstest] + fn without_a_runtime_the_chunks_pass_through() { + let chunks = + futures_util::stream::iter([Ok::<_, &str>(Bytes::from("a")), Ok(Bytes::from("b"))]); + let read: Vec = joined_reads(chunks) + .try_collect() + .now_or_never() + .unwrap() + .unwrap(); + assert_eq!(read, [Bytes::from("a"), Bytes::from("b")]); + } +} diff --git a/litellm-rust/crates/inference-messages/src/route.rs b/litellm-rust/crates/inference-messages/src/route.rs index 7ff886689da..fa850b23bf2 100644 --- a/litellm-rust/crates/inference-messages/src/route.rs +++ b/litellm-rust/crates/inference-messages/src/route.rs @@ -5,11 +5,10 @@ use litellm_host::{ call::{HostedCompletion, HostedMachine, hosted_call}, protocol::Protocol, }; -use litellm_llms_types::formats::messages::MessagesResponse; -use super::{Error, MessagesCall}; +use super::{Error, MessagesBody, MessagesCall}; -pub type MessagesOutput = HostedCompletion>; +pub type MessagesOutput = HostedCompletion; /// The upstream response as the caller sees it at stream hand-off, before any chunk. pub struct MessagesStreamHead { @@ -19,7 +18,7 @@ pub struct MessagesStreamHead { pub struct Messages; impl Protocol for Messages { - type Response = Box; + type Response = MessagesBody; type Error = Error; type Request = MessagesCall; type HostCall = Infallible; @@ -75,4 +74,103 @@ impl litellm_inference::caching::StreamCachable for Messages { fn bytes(chunk: &Self::Chunk) -> &[u8] { chunk.as_ref() } + + fn storable_stream(text: &str) -> bool { + let lines: Vec<&str> = sse_lines(text).collect(); + lines.contains(&"event: message_stop") + && !carries_provider_error(&lines) + && lines.iter().any(|line| line_carries_output(line)) + } +} + +fn sse_lines(text: &str) -> impl Iterator { + let mut rest = Some(text); + std::iter::from_fn(move || { + let current = rest?; + match current.find(['\r', '\n']) { + Some(at) => { + let skip = if current[at..].starts_with("\r\n") { + 2 + } else { + 1 + }; + rest = Some(¤t[at + skip..]); + Some(¤t[..at]) + } + None => { + rest = None; + (!current.is_empty()).then_some(current) + } + } + }) +} + +fn sse_data(line: &str, stripped: bool) -> Option> { + let data = line.strip_prefix("data:")?; + let data = if stripped { + data.trim_matches(|c: char| matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c')) + } else { + data + }; + match serde_json::from_str(data) { + Ok(serde_json::Value::Object(map)) => Some(map), + _ => None, + } +} + +fn carries_provider_error(lines: &[&str]) -> bool { + lines + .iter() + .filter_map(|line| sse_data(line, true)) + .find(|event| event.get("type").and_then(serde_json::Value::as_str) == Some("error")) + .is_some_and(|event| event.get("error").is_some_and(serde_json::Value::is_object)) +} + +fn line_carries_output(line: &str) -> bool { + let Some(event) = sse_data(line, false) else { + return false; + }; + let block = match event.get("type").and_then(serde_json::Value::as_str) { + Some("content_block_start") => event.get("content_block"), + Some("content_block_delta") => event.get("delta"), + _ => None, + }; + block + .and_then(serde_json::Value::as_object) + .is_some_and(|fields| { + fields.iter().any(|(name, value)| { + name != "type" + && !matches!(value, serde_json::Value::Null) + && value.as_str() != Some("") + && value.as_array().is_none_or(|items| !items.is_empty()) + }) + }) +} + +#[cfg(test)] +mod tests { + use litellm_inference::caching::StreamCachable; + use rstest::rstest; + + use super::Messages; + + const TEXT: &str = "event: message_start\ndata: {\"type\":\"message_start\"}\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Paris\"}}\n\n"; + const EMPTY: &str = "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"\"}}\n\n"; + const STOP: &str = "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"; + + #[rstest] + #[case::complete(format!("{TEXT}{STOP}"), true)] + #[case::crlf_lines(format!("{TEXT}{STOP}").replace('\n', "\r\n"), true)] + #[case::done_sentinel(format!("{TEXT}{STOP}data: [DONE]\n\n"), true)] + #[case::data_only_frames(format!("{TEXT}{STOP}").replace("event: message_stop\n", ""), false)] + #[case::event_without_space(format!("{TEXT}{STOP}").replace("event: message_stop", "event:message_stop"), false)] + #[case::provider_error(format!("{TEXT}{STOP}event: error\ndata: {{\"type\":\"error\",\"error\":{{\"type\":\"overloaded_error\"}}}}\n\n"), false)] + #[case::error_without_body(format!("{TEXT}{STOP}data: {{\"type\":\"error\",\"error\":\"overloaded\"}}\n\n"), true)] + #[case::no_output(format!("{EMPTY}{STOP}"), false)] + fn a_stream_is_stored_by_python_s_message_stop_error_and_output_tests( + #[case] text: String, + #[case] stored: bool, + ) { + assert_eq!(Messages::storable_stream(&text), stored); + } } diff --git a/litellm-rust/crates/inference-messages/src/safeguards.rs b/litellm-rust/crates/inference-messages/src/safeguards.rs new file mode 100644 index 00000000000..78e4a752536 --- /dev/null +++ b/litellm-rust/crates/inference-messages/src/safeguards.rs @@ -0,0 +1,248 @@ +use std::{collections::VecDeque, sync::Arc}; + +use bytes::Bytes; +use futures_util::{StreamExt, TryStreamExt, future::BoxFuture, stream::BoxStream}; +use serde_json::Value; + +use super::{Error, MessagesBody, MessagesCallResponse}; + +const SAFEGUARD_RESULTS: &str = "safeguard_results"; + +pub trait Safeguards: Send + Sync { + fn results<'a>( + &'a self, + response: &'a MessagesBody, + ) -> BoxFuture<'a, Result, Error>>; + + fn open_stream(&self) -> BoxFuture<'_, Result>; + + fn frames(&self, chunk: Bytes) -> BoxFuture<'_, Result, Error>>; + + fn flush(&self) -> BoxFuture<'_, Result, Error>>; +} + +pub(super) async fn stamped( + safeguards: Option<&Arc>, + output: MessagesCallResponse, +) -> Result { + let Some(safeguards) = safeguards else { + return Ok(output); + }; + match output { + MessagesCallResponse::Complete(response) => { + let results = safeguards.results(&response).await?; + Ok(MessagesCallResponse::Complete(match results { + Some(results) => with_results(response, results), + None => response, + })) + } + MessagesCallResponse::Stream { head, chunks } => { + if !safeguards.open_stream().await? { + return Ok(MessagesCallResponse::Stream { head, chunks }); + } + Ok(MessagesCallResponse::Stream { + head, + chunks: stamped_frames(Arc::clone(safeguards), chunks), + }) + } + } +} + +fn with_results(response: MessagesBody, results: Value) -> MessagesBody { + match response { + MessagesBody::Message(mut message) => { + message.extra.insert(SAFEGUARD_RESULTS.to_owned(), results); + MessagesBody::Message(message) + } + MessagesBody::Unchecked(mut body) => { + body.insert(SAFEGUARD_RESULTS.to_owned(), results); + MessagesBody::Unchecked(body) + } + } +} + +struct Framing { + chunks: Option>>, + safeguards: Arc, + ready: VecDeque, +} + +fn stamped_frames( + safeguards: Arc, + chunks: BoxStream<'static, Result>, +) -> BoxStream<'static, Result> { + let framing = Framing { + chunks: Some(chunks), + safeguards, + ready: VecDeque::new(), + }; + futures_util::stream::try_unfold(framing, |mut framing| async move { + loop { + if let Some(frame) = framing.ready.pop_front() { + return Ok(Some((frame, framing))); + } + let Some(mut chunks) = framing.chunks.take() else { + return Ok(None); + }; + let frames = match chunks.try_next().await? { + Some(chunk) => { + framing.chunks = Some(chunks); + framing.safeguards.frames(chunk).await? + } + None => framing.safeguards.flush().await?, + }; + framing.ready.extend(frames); + } + }) + .boxed() +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use litellm_llms_types::formats::messages::MessagesResponse; + use rstest::rstest; + use serde_json::{Map, json}; + + use super::*; + use crate::route::MessagesStreamHead; + + struct Scripted { + results: Option, + streams: bool, + fed: Mutex>, + } + + impl Scripted { + fn new(results: Option, streams: bool) -> Arc { + Arc::new(Self { + results, + streams, + fed: Mutex::new(Vec::new()), + }) + } + } + + impl Safeguards for Scripted { + fn results<'a>( + &'a self, + _: &'a MessagesBody, + ) -> BoxFuture<'a, Result, Error>> { + Box::pin(async move { Ok(self.results.clone()) }) + } + + fn open_stream(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { Ok(self.streams) }) + } + + fn frames(&self, chunk: Bytes) -> BoxFuture<'_, Result, Error>> { + Box::pin(async move { + self.fed.lock().unwrap().push(chunk.clone()); + Ok(chunk + .split_inclusive(|byte| *byte == b';') + .map(Bytes::copy_from_slice) + .collect()) + }) + } + + fn flush(&self) -> BoxFuture<'_, Result, Error>> { + Box::pin(async move { Ok(vec![Bytes::from_static(b"tail")]) }) + } + } + + fn message() -> MessagesBody { + let response: MessagesResponse = serde_json::from_value(json!({ + "id": "msg_1", "type": "message", "role": "assistant", "model": "m", + "content": [], "stop_reason": "end_turn", "stop_sequence": null, + })) + .unwrap(); + MessagesBody::Message(Box::new(response)) + } + + fn stream(chunks: &[&'static [u8]]) -> MessagesCallResponse { + let chunks: Vec> = chunks + .iter() + .map(|chunk| Ok(Bytes::from_static(chunk))) + .collect(); + MessagesCallResponse::Stream { + head: MessagesStreamHead { + headers: Vec::new(), + }, + chunks: futures_util::stream::iter(chunks).boxed(), + } + } + + async fn collected(output: MessagesCallResponse) -> Vec { + let MessagesCallResponse::Stream { chunks, .. } = output else { + panic!("expected a stream"); + }; + chunks.try_collect().await.unwrap() + } + + #[rstest] + #[case::message_gains_the_results(message(), Some(json!([{"type": "dangerous_tool_use"}])))] + #[case::unchecked_gains_the_results( + MessagesBody::Unchecked(Map::new()), + Some(json!([{"type": "dangerous_tool_use"}])) + )] + #[case::declined_leaves_it(message(), None)] + #[tokio::test] + async fn a_complete_response_carries_what_the_classifier_answered( + #[case] response: MessagesBody, + #[case] results: Option, + ) { + let safeguards: Arc = Scripted::new(results.clone(), false); + let MessagesCallResponse::Complete(stamped) = stamped( + Some(&safeguards), + MessagesCallResponse::Complete(response.clone()), + ) + .await + .unwrap() else { + panic!("expected a complete response"); + }; + let body = serde_json::to_value(&stamped).unwrap(); + assert_eq!(body.get(SAFEGUARD_RESULTS), results.as_ref()); + let mut unstamped = body; + unstamped.as_object_mut().unwrap().remove(SAFEGUARD_RESULTS); + assert_eq!(unstamped, serde_json::to_value(&response).unwrap()); + } + + #[tokio::test] + async fn a_stream_is_reframed_through_the_classifier_and_flushed_at_its_end() { + let scripted = Scripted::new(None, true); + let safeguards: Arc = scripted.clone(); + let frames = collected( + stamped(Some(&safeguards), stream(&[b"a;b", b";c;"])) + .await + .unwrap(), + ) + .await; + assert_eq!( + frames, + ["a;", "b", ";", "c;", "tail"] + .map(|frame| Bytes::from_static(frame.as_bytes())) + .to_vec() + ); + assert_eq!(scripted.fed.lock().unwrap().len(), 2); + } + + #[rstest] + #[case::declined(Some(Scripted::new(None, false) as Arc))] + #[case::not_classified(None)] + #[tokio::test] + async fn a_stream_the_classifier_does_not_answer_passes_through( + #[case] safeguards: Option>, + ) { + let frames = collected( + stamped(safeguards.as_ref(), stream(&[b"a;b", b";c;"])) + .await + .unwrap(), + ) + .await; + assert_eq!( + frames, + [b"a;b".as_slice(), b";c;"].map(Bytes::from_static).to_vec() + ); + } +} diff --git a/litellm-rust/crates/inference-messages/src/transport.rs b/litellm-rust/crates/inference-messages/src/transport.rs new file mode 100644 index 00000000000..1353592447b --- /dev/null +++ b/litellm-rust/crates/inference-messages/src/transport.rs @@ -0,0 +1,59 @@ +use bytes::Bytes; +use futures_util::{future::BoxFuture, stream::BoxStream}; +use serde_json::Value; + +use crate::Error; + +pub enum TransportBody { + Json(Value), + Signed(Bytes), +} + +pub struct TransportRequest { + pub url: String, + pub headers: Vec<(String, String)>, + pub body: TransportBody, + pub stream: bool, +} + +pub enum Delivered { + Complete(String), + Stream(BoxStream<'static, Result>), + Rejected { text: String, error: Error }, +} + +pub struct TransportResponse { + pub status: u16, + pub headers: Vec<(String, String)>, + pub delivered: Delivered, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum UnencodableHeader { + Sent(Vec), + Received(Vec), +} + +impl std::fmt::Display for UnencodableHeader { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a header value is not ASCII, which Python's HTTP client cannot encode") + } +} + +impl std::error::Error for UnencodableHeader {} + +impl UnencodableHeader { + pub(crate) fn received(headers: &reqwest::header::HeaderMap) -> Option { + headers + .values() + .find(|value| !value.as_bytes().is_ascii()) + .map(|value| Self::Received(value.as_bytes().to_vec())) + } +} + +pub trait Transport: Send + Sync { + fn send( + &self, + request: TransportRequest, + ) -> BoxFuture<'static, Result>; +} diff --git a/litellm-rust/crates/inference-messages/src/types.rs b/litellm-rust/crates/inference-messages/src/types.rs index ae346259559..9dbc171ecb0 100644 --- a/litellm-rust/crates/inference-messages/src/types.rs +++ b/litellm-rust/crates/inference-messages/src/types.rs @@ -2,7 +2,10 @@ use std::time::Duration; use bytes::Bytes; use litellm_host::call::CallOutput; -use litellm_llms::{ErrorDetail, base_llm::messages::context::MessagesModelCapabilities}; +use litellm_llms::{ + ErrorDetail, + base_llm::messages::context::{MessagesModelCapabilities, MessagesShapeOptions}, +}; use litellm_llms_types::{ formats::messages::{MessagesRequest, MessagesResponse}, headers::ProviderSpecificHeaders, @@ -42,7 +45,16 @@ pub(super) fn invalid_request(err: serde_json::Error) -> Error { } pub type MessagesCallResponse = - CallOutput, super::route::MessagesStreamHead, Bytes, Error>; + CallOutput; + +// Python's `transform_anthropic_messages_response` returns any JSON object the provider sends +// with a 200, message or not, so one that is not a message passes through as sent. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum MessagesBody { + Message(Box), + Unchecked(Map), +} #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct MessagesShaping { @@ -62,9 +74,38 @@ pub struct MessagesSettings { pub additional_drop_params: Vec, #[serde(default)] pub request_timeout: Option, + #[serde(default)] + pub mock_response: Option>, + #[serde(default)] + pub whole_sse_frames: bool, + #[serde(default)] + pub joined_reads: bool, + #[serde(default)] + pub presanitized: bool, + #[serde(default)] + pub drops_unsupported_params: bool, + #[serde(default)] + pub intercepted: bool, + #[serde(default)] + pub mcp_gateway: bool, + #[serde(default)] + pub ascii_response_headers: bool, + #[serde(default)] + pub unencodable_request_header: Option>, + /// Python's HTTP client encodes every header value it sends as ASCII, so a value it cannot + /// encode fails the call before the send. + #[serde(default)] + pub ascii_request_headers: bool, } impl MessagesSettings { + pub fn shape_options(&self) -> MessagesShapeOptions { + MessagesShapeOptions { + reasoning_auto_summary: self.reasoning_auto_summary, + presanitized: self.presanitized, + } + } + pub fn request_timeout(&self) -> Option { self.request_timeout .and_then(|secs| Duration::try_from_secs_f64(secs).ok()) @@ -131,7 +172,8 @@ mod tests { "drop_params": true, "reasoning_auto_summary": true, "additional_drop_params": ["metadata.user_id", "thinking"], - "request_timeout": 1800.0 + "request_timeout": 1800.0, + "drops_unsupported_params": true }), MessagesShaping { settings: MessagesSettings { @@ -139,11 +181,14 @@ mod tests { reasoning_auto_summary: true, additional_drop_params: vec!["metadata.user_id".to_string(), "thinking".to_string()], request_timeout: Some(1800.0), + drops_unsupported_params: true, + ..MessagesSettings::default() }, capabilities: MessagesModelCapabilities { supports_reasoning: true, supports_adaptive_thinking: true, thinking_always_on: false, + thinking_on_by_default: false, supports_legacy_thinking: false, supports_output_config: true, supports_sampling_params: false, @@ -175,6 +220,10 @@ mod tests { serialized["reasoning_auto_summary"], json!(expected.settings.reasoning_auto_summary) ); + assert_eq!( + serialized["drops_unsupported_params"], + json!(expected.settings.drops_unsupported_params) + ); assert_eq!( serialized["additional_drop_params"], json!(expected.settings.additional_drop_params) diff --git a/litellm-rust/crates/inference-messages/src/workload_identity.rs b/litellm-rust/crates/inference-messages/src/workload_identity.rs new file mode 100644 index 00000000000..56445de6eea --- /dev/null +++ b/litellm-rust/crates/inference-messages/src/workload_identity.rs @@ -0,0 +1,11 @@ +use futures_util::future::BoxFuture; +use litellm_router_types::LitellmParams; +use litellm_secret_value::SecretValue; + +pub trait WorkloadIdentity: Send + Sync { + fn token<'a>( + &'a self, + litellm_params: &'a LitellmParams, + api_base: Option<&'a str>, + ) -> BoxFuture<'a, Result, litellm_auth::Error>>; +} diff --git a/litellm-rust/crates/inference-messages/tests/bedrock_native/main.rs b/litellm-rust/crates/inference-messages/tests/bedrock_native/main.rs index f91d5c79f7c..5207f4e9d58 100644 --- a/litellm-rust/crates/inference-messages/tests/bedrock_native/main.rs +++ b/litellm-rust/crates/inference-messages/tests/bedrock_native/main.rs @@ -1,7 +1,9 @@ use std::{sync::Arc, time::Duration}; use futures_util::TryStreamExt; -use litellm_inference_messages::{Error, MessagesCall, MessagesCallResponse, MessagesShaping}; +use litellm_inference_messages::{ + Error, MessagesBody, MessagesCall, MessagesCallResponse, MessagesShaping, +}; use litellm_inference_testing::{RecordingSecrets, no_secrets}; use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse}; use rstest::{fixture, rstest}; @@ -74,7 +76,10 @@ async fn run(call: MessagesCall) -> Result { async fn run_message(call: MessagesCall) -> MessagesResponse { match run(call).await.unwrap() { - MessagesCallResponse::Complete(message) => *message, + MessagesCallResponse::Complete(message) => match message { + MessagesBody::Message(message) => *message, + MessagesBody::Unchecked(_) => panic!("expected a message"), + }, MessagesCallResponse::Stream { .. } => panic!("expected a completed message"), } } diff --git a/litellm-rust/crates/inference-messages/tests/caching.rs b/litellm-rust/crates/inference-messages/tests/caching.rs index 663c87453a9..ff83f4e41e1 100644 --- a/litellm-rust/crates/inference-messages/tests/caching.rs +++ b/litellm-rust/crates/inference-messages/tests/caching.rs @@ -50,42 +50,47 @@ async fn messages_cache_identity_includes_provider_native_parameters( #[case] original: Value, #[case] changed: Value, ) { - use litellm_inference_messages::route::Messages; + use litellm_inference_messages::{MessagesBody, route::Messages}; use litellm_llms_types::formats::messages::MessagesResponse; let calls = AtomicUsize::new(0); for (value, expected_call) in [(original.clone(), 0), (changed, 1), (original, 0)] { - let response = - execute_unary::( - CacheRequest::from_wire( - ProviderIdentity { - model: "test".into(), - provider: "anthropic".into(), - }, - Some(&WireRequest { - url: "https://example.test/v1/messages".into(), - headers: vec![], - body: json!({ - "model":"test", "messages":[{"role":"user","content":"hello"}], - "max_tokens":32, (field):value - }), - }), - ), - Some(cache.clone()), - Some(CacheOptions::new(CacheScope::Shared)), - &(), - None, - || async { - let call = calls.fetch_add(1, Ordering::SeqCst); - Ok(Box::new(serde_json::from_value::(json!({ - "id":call.to_string(), "type":"message", "role":"assistant", "model":"test", - "content":[{"type":"text","text":format!("answer {call}")}], - "stop_reason":"end_turn", "stop_sequence":null - })).unwrap())) + let response = execute_unary::( + CacheRequest::from_wire( + ProviderIdentity { + model: "test".into(), + provider: "anthropic".into(), }, - ) - .await - .unwrap(); + Some(&WireRequest { + url: "https://example.test/v1/messages".into(), + headers: vec![], + body: json!({ + "model":"test", "messages":[{"role":"user","content":"hello"}], + "max_tokens":32, (field):value + }), + }), + ), + Some(cache.clone()), + Some(CacheOptions::new(CacheScope::Shared)), + &(), + None, + || async { + let call = calls.fetch_add(1, Ordering::SeqCst); + Ok(MessagesBody::Message(Box::new( + serde_json::from_value::(json!({ + "id":call.to_string(), "type":"message", "role":"assistant", "model":"test", + "content":[{"type":"text","text":format!("answer {call}")}], + "stop_reason":"end_turn", "stop_sequence":null + })) + .unwrap(), + ))) + }, + ) + .await + .unwrap(); + let MessagesBody::Message(response) = response else { + panic!("expected a message"); + }; assert_eq!(response.id, expected_call.to_string()); assert_eq!( response.content[0]["text"], @@ -253,7 +258,7 @@ async fn provider_reported_cost_is_delivered_once_and_never_charged_for_a_cache_ #[case] cost: Value, #[case] expected: Option, ) { - use litellm_inference_messages::{MessagesCall, MessagesCallResponse}; + use litellm_inference_messages::{MessagesBody, MessagesCall, MessagesCallResponse}; let response = json!({"id": "msg_cost", "type": "message", "role": "assistant", "model": "native-test", "content": [{"type": "text", "text": "answer"}], "stop_reason": "end_turn", "stop_sequence": null, "usage": {"input_tokens": 3, "output_tokens": 2}, "cost": cost}); @@ -277,6 +282,9 @@ async fn provider_reported_cost_is_delivered_once_and_never_charged_for_a_cache_ panic!("expected Messages responses"); }; assert_eq!(first, second); + let MessagesBody::Message(first) = first else { + panic!("expected a message"); + }; assert_eq!(first.extra.get("cost"), Some(&cost)); assert_eq!(support::received(&upstream).await.len(), 1); let facts = hooks.facts.lock().unwrap(); @@ -293,3 +301,135 @@ async fn provider_reported_cost_is_delivered_once_and_never_charged_for_a_cache_ assert!(matches!(cached.source, ResultSource::Cache { .. })); assert_eq!(cached.cost, Cost::CacheHit); } + +#[derive(Default)] +struct CallerKeyed { + config: ResponseCacheConfig, + entry: std::sync::Mutex>, +} + +impl ResponseCacheService for CallerKeyed { + fn config(&self) -> &ResponseCacheConfig { + &self.config + } + + fn keyed_by_caller(&self) -> bool { + true + } + + fn lookup<'a>( + &'a self, + _: &'a litellm_cache_response::ResponseCacheRequest, + _: Duration, + ) -> std::pin::Pin< + Box< + dyn std::future::Future, litellm_cache::Error>> + + Send + + 'a, + >, + > { + Box::pin(async move { Ok(self.entry.lock().unwrap().clone()) }) + } + + fn store<'a>( + &'a self, + _: &'a litellm_cache_response::ResponseCacheRequest, + value: Value, + _: Duration, + ) -> std::pin::Pin< + Box> + Send + 'a>, + > { + Box::pin(async move { + *self.entry.lock().unwrap() = Some(value); + Ok(()) + }) + } +} + +#[tokio::test] +async fn a_caller_keyed_hit_is_served_before_the_request_is_built_or_intercepted() { + use litellm_inference_messages::{MessagesCall, MessagesCallResponse}; + + let response = json!({"id": "msg_caller", "type": "message", "role": "assistant", "model": "native-test", + "content": [{"type": "text", "text": "answer"}], "stop_reason": "end_turn", "stop_sequence": null, + "usage": {"input_tokens": 3, "output_tokens": 2}}); + let upstream = support::upstream([support::json_response(response)]).await; + let hooks = ChangingHooks::default(); + let route = support::messages_route(litellm_inference_testing::no_secrets()).with_cache( + ScopedCache::new(Arc::new(CallerKeyed::default()), CacheScope::Shared), + ); + let request = |api_key: Option<&str>| { + MessagesCall { + body: serde_json::from_value(json!({"model": "native-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})).unwrap(), + api_key: api_key.map(Into::into), api_base: Some(upstream.uri()), custom_llm_provider: Some("anthropic".into()), + litellm_params: Default::default(), extra_headers: None, provider_specific_header: None, + timeout: None, shaping: Default::default(), + } + }; + let first = route + .execute(request(Some("key")), &hooks, None) + .await + .unwrap(); + let second = route.execute(request(None), &hooks, None).await.unwrap(); + let (MessagesCallResponse::Complete(first), MessagesCallResponse::Complete(second)) = + (first, second) + else { + panic!("expected Messages responses"); + }; + assert_eq!(first, second); + assert_eq!(support::received(&upstream).await.len(), 1); + assert_eq!(hooks.calls.load(Ordering::SeqCst), 1); + let facts = hooks.facts.lock().unwrap(); + let [provider, cached] = facts.as_slice() else { + panic!("expected provider and cache results"); + }; + assert_eq!(provider.source, ResultSource::Provider); + assert!(matches!(cached.source, ResultSource::Cache { .. })); + assert_eq!(cached.cost, Cost::CacheHit); +} + +#[rstest] +#[case::unary(false, true)] +#[case::streamed(true, false)] +#[tokio::test] +async fn a_mock_is_stored_under_the_caller_key_only_when_unstreamed( + #[case] stream: bool, + #[case] stored: bool, +) { + use litellm_inference_messages::{ + MessagesBody, MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, + }; + + let mock = json!({"id": "msg_mock", "type": "message", "role": "assistant", "model": "native-test", + "content": [{"type": "text", "text": "mocked"}], "stop_reason": "end_turn", "stop_sequence": null, + "usage": {"input_tokens": 3, "output_tokens": 2}}); + let upstream = support::upstream([]).await; + let hooks = ChangingHooks::default(); + let service = Arc::new(CallerKeyed::default()); + let route = support::messages_route(litellm_inference_testing::no_secrets()) + .with_cache(ScopedCache::new(service.clone(), CacheScope::Shared)); + let call = MessagesCall { + body: serde_json::from_value(json!({"model": "native-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16, "stream": stream})).unwrap(), + api_key: Some("key".into()), api_base: Some(upstream.uri()), custom_llm_provider: Some("anthropic".into()), + litellm_params: Default::default(), extra_headers: None, provider_specific_header: None, + timeout: None, + shaping: MessagesShaping { + settings: MessagesSettings { + mock_response: serde_json::from_value(mock.clone()).unwrap(), + ..Default::default() + }, + ..Default::default() + }, + }; + + let output = route.execute(call, &hooks, None).await.unwrap(); + + let MessagesCallResponse::Complete(MessagesBody::Unchecked(body)) = output else { + panic!("expected the mock as a whole response"); + }; + assert_eq!(Value::Object(body), mock); + assert!(support::received(&upstream).await.is_empty()); + assert_eq!(hooks.calls.load(Ordering::SeqCst), 0); + assert!(hooks.facts.lock().unwrap().is_empty()); + assert_eq!(service.entry.lock().unwrap().is_some(), stored); +} diff --git a/litellm-rust/crates/inference-messages/tests/live/support.rs b/litellm-rust/crates/inference-messages/tests/live/support.rs index 81e295e34ad..52684928f1f 100644 --- a/litellm-rust/crates/inference-messages/tests/live/support.rs +++ b/litellm-rust/crates/inference-messages/tests/live/support.rs @@ -4,7 +4,9 @@ use futures_util::{FutureExt, TryStreamExt}; use litellm_host::interceptors::{ ExecutionFacts, Interceptors, RawResponse, RequestContext, ResultSource, WireRequest, }; -use litellm_inference_messages::{Error, MessagesCall, MessagesCallResponse, MessagesRoute}; +use litellm_inference_messages::{ + Error, MessagesBody, MessagesCall, MessagesCallResponse, MessagesRoute, +}; use litellm_inference_testing::live::{LiveResources, model, within_deadline}; use litellm_llms_types::formats::messages::MessagesResponse; use serde_json::{Value, json}; @@ -111,7 +113,10 @@ pub(super) fn call(provider: &str, payload: Value) -> MessagesCall { pub(super) fn complete(response: MessagesCallResponse) -> Box { match response { - MessagesCallResponse::Complete(message) => message, + MessagesCallResponse::Complete(message) => match message { + MessagesBody::Message(message) => message, + MessagesBody::Unchecked(_) => panic!("a complete request returned no message"), + }, MessagesCallResponse::Stream { .. } => panic!("a complete request returned a stream"), } } diff --git a/litellm-rust/crates/inference-messages/tests/messages/gateway.rs b/litellm-rust/crates/inference-messages/tests/messages/gateway.rs new file mode 100644 index 00000000000..d6e591c723c --- /dev/null +++ b/litellm-rust/crates/inference-messages/tests/messages/gateway.rs @@ -0,0 +1,99 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + +use futures_util::future::BoxFuture; +use litellm_inference_messages::{HostAnswer, MessagesRoute}; +use rstest::rstest; + +use super::*; + +#[derive(Default)] +struct CountingGateway(AtomicUsize); + +impl HostAnswer for CountingGateway { + fn answer(&self, _stream: bool) -> BoxFuture<'_, Result<(), Error>> { + self.0.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok(()) }) + } +} + +fn gateway_call(call: MessagesCall, api_base: String, mcp_gateway: bool) -> MessagesCall { + MessagesCall { + api_key: Some("sk-ant-test".into()), + api_base: Some(api_base), + shaping: MessagesShaping { + settings: MessagesSettings { + mcp_gateway, + ..MessagesSettings::default() + }, + ..MessagesShaping::default() + }, + ..call + } +} + +async fn run_through(route: MessagesRoute, call: MessagesCall) -> Result { + let host = LocalMessagesHost::new(call); + litellm_host_native::in_process::run_hosted( + route.machine(host.request()?, None), + host.runtime(), + ) + .await +} + +#[rstest] +#[tokio::test] +async fn a_call_the_host_hands_to_the_mcp_gateway_never_reaches_the_provider(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let gateway = Arc::new(CountingGateway::default()); + let route = + messages_route(Arc::new(RecordingSecrets::empty())).with_host_answer(gateway.clone()); + + let output = run_through(route, gateway_call(call, upstream.uri(), true)) + .await + .unwrap(); + + let MessagesOutput::Complete(MessagesBody::Unchecked(body)) = output else { + panic!("the host answers in place of the provider"); + }; + assert!(body.is_empty()); + assert_eq!(gateway.0.load(Ordering::SeqCst), 1); + assert!(received(&upstream).await.is_empty()); +} + +#[rstest] +#[tokio::test] +async fn a_call_the_host_keeps_reaches_the_provider_past_the_gateway(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let gateway = Arc::new(CountingGateway::default()); + let route = + messages_route(Arc::new(RecordingSecrets::empty())).with_host_answer(gateway.clone()); + + let output = run_through(route, gateway_call(call, upstream.uri(), false)) + .await + .unwrap(); + + assert!(matches!( + output, + MessagesOutput::Complete(MessagesBody::Message(_)) + )); + assert_eq!(gateway.0.load(Ordering::SeqCst), 0); + only_request(&upstream).await; +} + +#[rstest] +#[tokio::test] +async fn a_mock_answers_before_the_gateway(call: MessagesCall) { + let gateway = Arc::new(CountingGateway::default()); + let route = + messages_route(Arc::new(RecordingSecrets::empty())).with_host_answer(gateway.clone()); + let mut mocked = gateway_call(call, UNREACHABLE_BASE.into(), true); + mocked.shaping.settings.mock_response = Some(object(json!({"mocked": true}))); + + let output = run_through(route, mocked).await.unwrap(); + + let MessagesOutput::Complete(MessagesBody::Unchecked(body)) = output else { + panic!("the mock is the call's response"); + }; + assert_eq!(body, object(json!({"mocked": true}))); + assert_eq!(gateway.0.load(Ordering::SeqCst), 0); +} diff --git a/litellm-rust/crates/inference-messages/tests/messages/host.rs b/litellm-rust/crates/inference-messages/tests/messages/host.rs index c3d06027c32..7254830a2b0 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/host.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/host.rs @@ -3,12 +3,14 @@ use litellm_host::lifecycle::ExecutionEvent; use std::sync::Mutex; use litellm_host::{ - interceptors::{ExecutionFacts, RequestContext, ResultSource, WireRequest}, + interceptors::{ExecutionFacts, RawResponse, RequestContext, ResultSource, WireRequest}, lifecycle::CallEvent, }; -use litellm_inference_messages::{MessagesCallResponse, route::Messages}; +use litellm_inference_messages::{MessagesBody, MessagesCallResponse, route::Messages}; use litellm_inference_testing::{RecordingSecrets, no_secrets}; -use litellm_llms::base_llm::messages::context::MessagesModelCapabilities as AnthropicModelCapabilities; +use litellm_llms::base_llm::messages::context::{ + MessagesModelCapabilities as AnthropicModelCapabilities, SupportedEffortTiers, +}; use rstest::rstest; use super::*; @@ -44,19 +46,23 @@ impl RecordingHost { Self::new(call, Box::new(Ok)) } - fn raw_responses(&self) -> Vec { + fn raw_heads(&self) -> Vec { self.events .lock() .unwrap() .iter() .filter_map(|event| match event { CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw }) => { - Some(raw.body.clone()) + Some(raw.clone()) } _ => None, }) .collect() } + + fn raw_responses(&self) -> Vec { + self.raw_heads().into_iter().map(|raw| raw.body).collect() + } } impl RecordingHost { @@ -135,7 +141,7 @@ async fn rejected_results_are_not_delivered_or_cached( let response = if streaming { ResponseTemplate::new(200).set_body_raw( - "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Hi\"}}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", "text/event-stream", ) } else { @@ -289,22 +295,23 @@ async fn what_before_send_returns_is_what_the_provider_receives(call: MessagesCa } #[rstest] -#[case::enable(false, json!(true), Some(true))] -#[case::disable(true, json!(false), Some(false))] -#[case::null(true, Value::Null, Some(false))] -#[case::invalid(false, json!("true"), None)] +#[case::enable(false, json!(true), true)] +#[case::disable(true, json!(false), false)] +#[case::null(true, Value::Null, false)] +#[case::truthy_string(false, json!("false"), true)] +#[case::falsy_integer(true, json!(0), false)] #[tokio::test] async fn response_mode_follows_the_intercepted_request( call: MessagesCall, traces: TraceCapture, #[case] original_stream: bool, #[case] rewritten_stream: Value, - #[case] expected_stream: Option, + #[case] expected_stream: bool, ) { use futures_util::TryStreamExt; let sse = "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"; - let response = if expected_stream == Some(true) { + let response = if expected_stream { ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream") } else { message_response() @@ -330,15 +337,18 @@ async fn response_mode_follows_the_intercepted_request( .await?; match output { MessagesCallResponse::Stream { chunks, .. } => { - assert_eq!(expected_stream, Some(true)); + assert!(expected_stream); assert_eq!( chunks.try_collect::>().await?.concat(), sse.as_bytes() ); } MessagesCallResponse::Complete(message) => { - assert_eq!(expected_stream, Some(false)); - assert_eq!(*message, serde_json::from_value(message_body()).unwrap()); + assert!(!expected_stream); + assert_eq!( + message, + MessagesBody::Message(serde_json::from_value(message_body()).unwrap()) + ); } } Ok::<_, Error>(()) @@ -347,18 +357,12 @@ async fn response_mode_follows_the_intercepted_request( let summaries = traces.summaries("litellm.route"); assert_eq!(summaries.len(), 1); - let Some(expected_stream) = expected_stream else { - assert!(result.unwrap_err().is_request()); - assert!(received(&upstream).await.is_empty()); - assert_eq!(summaries[0]["outcome"], "failure"); - return; - }; result.unwrap(); assert_eq!( only_request(&upstream).await.json()["stream"], rewritten_stream ); - assert_eq!(host.raw_responses().len(), usize::from(!expected_stream)); + assert_eq!(host.raw_responses().len(), 1); assert_eq!(summaries[0]["stream"], expected_stream); assert_eq!(summaries[0]["outcome"], "success"); } @@ -400,14 +404,9 @@ async fn the_raw_upstream_text_is_emitted_once_for_a_message(call: MessagesCall) } #[rstest] -#[case::upstream_error(ResponseTemplate::new(500).set_body_string("boom"))] -#[case::stream(ResponseTemplate::new(200).set_body_raw("event: message_stop\ndata: {}\n\n", "text/event-stream"))] #[tokio::test] -async fn no_raw_response_is_emitted_for_a_stream_or_a_failure( - call: MessagesCall, - #[case] response: ResponseTemplate, -) { - let upstream = upstream([response]).await; +async fn no_raw_response_is_emitted_for_a_failure(call: MessagesCall) { + let upstream = upstream([ResponseTemplate::new(500).set_body_string("boom")]).await; let host = RecordingHost::passthrough(authenticated( with_fields(call, json!({"stream": true})), upstream.uri(), @@ -419,7 +418,33 @@ async fn no_raw_response_is_emitted_for_a_stream_or_a_failure( assert!(host.raw_responses().is_empty()); } -/// Python logs `optional_params` as what it is about to send, so a dropped param must +#[rstest] +#[tokio::test] +async fn a_stream_emits_its_head_without_a_body_before_any_chunk(call: MessagesCall) { + let upstream = upstream([ResponseTemplate::new(200) + .insert_header("request-id", "req_stream") + .set_body_raw("event: message_stop\ndata: {}\n\n", "text/event-stream")]) + .await; + let host = RecordingHost::passthrough(authenticated( + with_fields(call, json!({"stream": true})), + upstream.uri(), + )); + + let _ = run_through(&host).await; + + let [raw] = <[RawResponse; 1]>::try_from(host.raw_heads()) + .unwrap_or_else(|raws| panic!("expected one raw response, got {}", raws.len())); + assert_eq!(raw.body, ""); + let head = raw.head.expect("the stream's head is forwarded"); + assert_eq!(head.status, 200); + assert!( + head.headers + .iter() + .any(|(name, value)| name == "request-id" && value == "req_stream") + ); +} + +/// Python logs `optional_params` after its requested-param gate, so a dropped param must /// not resurface in callbacks. #[rstest] #[tokio::test] @@ -450,3 +475,209 @@ async fn the_request_context_carries_the_shaped_params_without_model_or_messages .unwrap_or_else(|seen| panic!("before_provider_request runs once, saw {}", seen.len())); assert_eq!(optional_params, json!({"max_tokens": 16})); } + +#[rstest] +#[tokio::test] +async fn the_request_context_keeps_the_nested_fields_additional_drop_params_strip( + call: MessagesCall, +) { + let upstream = upstream([message_response()]).await; + let tool = json!({"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}); + let host = RecordingHost::passthrough(authenticated( + MessagesCall { + shaping: MessagesShaping { + settings: MessagesSettings { + additional_drop_params: vec!["tools[*].input_examples".into()], + ..MessagesSettings::default() + }, + ..MessagesShaping::default() + }, + ..with_fields(call, json!({"tools": [tool.clone()]})) + }, + upstream.uri(), + )); + + run_through(&host).await.expect("messages call succeeds"); + + let [optional_params] = <[Value; 1]>::try_from(host.optional_params.into_inner().unwrap()) + .unwrap_or_else(|seen| panic!("before_provider_request runs once, saw {}", seen.len())); + assert_eq!(optional_params["tools"], json!([tool])); + assert_eq!( + only_request(&upstream).await.json()["tools"][0].get("input_examples"), + None + ); +} + +#[rstest] +#[tokio::test] +async fn the_request_context_leaves_out_a_listed_param_the_model_refuses(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let host = RecordingHost::passthrough(authenticated( + MessagesCall { + shaping: MessagesShaping { + settings: MessagesSettings { + additional_drop_params: vec!["temperature".into()], + ..MessagesSettings::default() + }, + capabilities: AnthropicModelCapabilities { + supports_sampling_params: false, + ..AnthropicModelCapabilities::default() + }, + }, + ..with_fields(call, json!({"temperature": 0.2})) + }, + upstream.uri(), + )); + + run_through(&host).await.expect("messages call succeeds"); + + let [optional_params] = <[Value; 1]>::try_from(host.optional_params.into_inner().unwrap()) + .unwrap_or_else(|seen| panic!("before_provider_request runs once, saw {}", seen.len())); + assert_eq!(optional_params, json!({"max_tokens": 16})); +} + +#[rstest] +#[case::reasoning_effort_on_an_adaptive_model( + AnthropicModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + supports_output_config: true, + effort_tiers: SupportedEffortTiers { low: true, ..SupportedEffortTiers::default() }, + ..AnthropicModelCapabilities::default() + }, + json!({"reasoning_effort": "low"}) +)] +#[case::adaptive_payload_on_a_legacy_model( + AnthropicModelCapabilities { + supports_reasoning: true, + ..AnthropicModelCapabilities::default() + }, + json!({"max_tokens": 3000, "thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}}) +)] +#[tokio::test] +async fn the_request_context_carries_the_requested_reasoning_not_its_translation( + call: MessagesCall, + #[case] capabilities: AnthropicModelCapabilities, + #[case] fields: Value, +) { + let upstream = upstream([message_response()]).await; + let host = RecordingHost::passthrough(authenticated( + MessagesCall { + shaping: MessagesShaping { + capabilities, + ..MessagesShaping::default() + }, + ..with_fields(call, fields.clone()) + }, + upstream.uri(), + )); + + run_through(&host).await.expect("messages call succeeds"); + + let [optional_params] = <[Value; 1]>::try_from(host.optional_params.into_inner().unwrap()) + .unwrap_or_else(|seen| panic!("before_provider_request runs once, saw {}", seen.len())); + assert_eq!( + optional_params, + Value::Object( + object(json!({"max_tokens": 16})) + .into_iter() + .chain(object(fields)) + .collect() + ) + ); +} + +#[derive(Default)] +struct RecordingAnswer(Mutex>); + +impl litellm_inference_messages::HostAnswer for RecordingAnswer { + fn answer(&self, stream: bool) -> futures_util::future::BoxFuture<'_, Result<(), Error>> { + self.0.lock().unwrap().push(stream); + Box::pin(async { Ok(()) }) + } +} + +#[rstest] +#[case::unary(false)] +#[case::stream(true)] +#[tokio::test] +async fn an_intercepted_call_is_answered_by_the_host_before_the_mock_and_the_provider( + call: MessagesCall, + #[case] streaming: bool, +) { + let upstream = upstream([message_response()]).await; + let answer = Arc::new(RecordingAnswer::default()); + let route = messages_route(no_secrets()).with_host_answer(answer.clone()); + let mut request = authenticated( + with_fields(call, json!({"stream": streaming})), + upstream.uri(), + ); + request.shaping.settings.intercepted = true; + request.shaping.settings.mock_response = Some(object(json!({"id": "mocked"}))); + let host = RecordingHost::passthrough(request); + + let output = route + .execute(host.request().unwrap(), &host, None) + .await + .unwrap(); + + assert!(matches!( + output, + MessagesCallResponse::Complete(MessagesBody::Unchecked(body)) if body.is_empty() + )); + assert_eq!(*answer.0.lock().unwrap(), vec![streaming]); + assert!(received(&upstream).await.is_empty()); + let facts = host.facts.lock().unwrap(); + assert_eq!(facts.len(), 1); + assert_eq!(facts[0].source, ResultSource::Host); +} + +#[rstest] +#[tokio::test] +async fn a_call_no_interceptor_owns_never_asks_the_host(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let answer = Arc::new(RecordingAnswer::default()); + let route = messages_route(no_secrets()).with_host_answer(answer.clone()); + let host = RecordingHost::passthrough(authenticated(call, upstream.uri())); + + route + .execute(host.request().unwrap(), &host, None) + .await + .unwrap(); + + assert!(answer.0.lock().unwrap().is_empty()); + assert_eq!(received(&upstream).await.len(), 1); +} + +#[derive(Default)] +struct DecliningAnswer(Mutex>); + +impl litellm_inference_messages::HostAnswer for DecliningAnswer { + fn answer(&self, _stream: bool) -> futures_util::future::BoxFuture<'_, Result<(), Error>> { + panic!("a declined interception never asks for an answer") + } + + fn intercept(&self, stream: bool) -> futures_util::future::BoxFuture<'_, Result> { + self.0.lock().unwrap().push(stream); + Box::pin(async { Ok(false) }) + } +} + +#[rstest] +#[tokio::test] +async fn an_interception_no_interceptor_takes_reaches_the_provider(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let answer = Arc::new(DecliningAnswer::default()); + let route = messages_route(no_secrets()).with_host_answer(answer.clone()); + let mut request = authenticated(call, upstream.uri()); + request.shaping.settings.intercepted = true; + let host = RecordingHost::passthrough(request); + + route + .execute(host.request().unwrap(), &host, None) + .await + .unwrap(); + + assert_eq!(*answer.0.lock().unwrap(), vec![false]); + assert_eq!(received(&upstream).await.len(), 1); +} diff --git a/litellm-rust/crates/inference-messages/tests/messages/main.rs b/litellm-rust/crates/inference-messages/tests/messages/main.rs index a01e571ec75..1d515e87b90 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/main.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/main.rs @@ -5,7 +5,7 @@ use std::{ use litellm_http::{HttpSettings, Resolution}; use litellm_inference_messages::{ - Error, MessagesCall, MessagesSettings, MessagesShaping, + Error, MessagesBody, MessagesCall, MessagesSettings, MessagesShaping, route::{Messages, MessagesMachine, MessagesOutput}, }; use litellm_inference_testing::RecordingSecrets; @@ -19,12 +19,16 @@ use wiremock::ResponseTemplate; mod support; use support::*; +mod gateway; mod host; +mod pre_request; mod request; mod response; +mod retry; mod secrets; mod stream; mod tencent; +mod transport; const MODEL: &str = "claude-sonnet-4-5"; @@ -119,7 +123,10 @@ async fn run(call: MessagesCall) -> Result { async fn run_message(call: MessagesCall) -> MessagesResponse { match run(call).await.expect("messages call succeeds") { - MessagesOutput::Complete(message) => *message, + MessagesOutput::Complete(message) => match message { + MessagesBody::Message(message) => *message, + MessagesBody::Unchecked(_) => panic!("a call returned no message"), + }, MessagesOutput::StreamEnded | MessagesOutput::Detached => { panic!("a non-streaming call returned a stream") } diff --git a/litellm-rust/crates/inference-messages/tests/messages/pre_request.rs b/litellm-rust/crates/inference-messages/tests/messages/pre_request.rs new file mode 100644 index 00000000000..13a632bdecd --- /dev/null +++ b/litellm-rust/crates/inference-messages/tests/messages/pre_request.rs @@ -0,0 +1,141 @@ +use futures_util::future::BoxFuture; +use litellm_inference_messages::PreRequestHooks; +use rstest::rstest; + +use super::*; + +struct Hooks { + rewrite: Box Result + Send + Sync>, + runs: Mutex, +} + +impl PreRequestHooks for Hooks { + fn run(&self, call: MessagesCall) -> BoxFuture<'_, Result> { + *self.runs.lock().unwrap() += 1; + let rewritten = (self.rewrite)(call); + Box::pin(async move { rewritten }) + } +} + +fn hooks( + rewrite: impl Fn(MessagesCall) -> Result + Send + Sync + 'static, +) -> Arc { + Arc::new(Hooks { + rewrite: Box::new(rewrite), + runs: Mutex::new(0), + }) +} + +async fn run_hooked(hooks: Arc, call: MessagesCall) -> Result { + let host = LocalMessagesHost::new(call); + let route = messages_route(Arc::new(RecordingSecrets::empty())).with_pre_request_hooks(hooks); + litellm_host_native::in_process::run_hosted( + route.machine(host.request()?, None), + host.runtime(), + ) + .await +} + +fn mocked(call: MessagesCall, mock: Map) -> MessagesCall { + MessagesCall { + shaping: MessagesShaping { + settings: MessagesSettings { + mock_response: Some(mock), + ..call.shaping.settings + }, + ..call.shaping + }, + ..call + } +} + +fn routed(call: MessagesCall, base: String) -> MessagesCall { + MessagesCall { + api_key: Some("sk-test".into()), + api_base: Some(base), + ..call + } +} + +#[rstest] +#[tokio::test] +async fn the_provider_receives_the_call_the_hooks_leave(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let hooks = hooks(|call| { + Ok(with_fields( + call, + json!({"system": "hooked", "max_tokens": 7}), + )) + }); + + run_hooked(hooks.clone(), routed(call, upstream.uri())) + .await + .expect("messages call succeeds"); + + let sent: Value = only_request(&upstream).await.body_json().unwrap(); + assert_eq!(sent["system"], "hooked"); + assert_eq!(sent["max_tokens"], 7); + assert_eq!(*hooks.runs.lock().unwrap(), 1); +} + +#[rstest] +#[case::hook_sets_the_mock(false, true, true)] +#[case::hook_keeps_the_callers_mock(true, false, true)] +#[case::hook_clears_the_callers_mock(true, true, false)] +#[tokio::test] +async fn the_mock_is_read_from_the_hooked_call( + call: MessagesCall, + #[case] called_with_mock: bool, + #[case] hook_rewrites_mock: bool, + #[case] mocked_response: bool, +) { + let upstream = upstream([message_response()]).await; + let mock = object(json!({"id": "mocked"})); + let call = routed(call, upstream.uri()); + let call = match called_with_mock { + true => mocked(call, mock.clone()), + false => call, + }; + let hooked_mock = (!called_with_mock).then(|| mock.clone()); + let hooks = hooks(move |call| { + Ok(match (hook_rewrites_mock, &hooked_mock) { + (false, _) => call, + (true, Some(mock)) => mocked(call, mock.clone()), + (true, None) => MessagesCall { + shaping: MessagesShaping { + settings: MessagesSettings { + mock_response: None, + ..call.shaping.settings + }, + ..call.shaping + }, + ..call + }, + }) + }); + + let output = run_hooked(hooks, call) + .await + .expect("messages call succeeds"); + + let mocked_output = matches!( + output, + MessagesOutput::Complete(MessagesBody::Unchecked(ref body)) if *body == mock + ); + assert_eq!(mocked_output, mocked_response); + assert_eq!(received(&upstream).await.is_empty(), mocked_response); +} + +#[rstest] +#[tokio::test] +async fn a_failing_hook_fails_the_call_before_the_provider(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let hooks = hooks(|_| Err(Error::InvalidProvider("refused".into()))); + + let Err(error) = run_hooked(hooks, routed(call, upstream.uri())).await else { + panic!("the hook's failure fails the call"); + }; + + assert!(matches!(error, Error::InvalidProvider(ref provider) if provider == "refused")); + assert!(received(&upstream).await.is_empty()); +} diff --git a/litellm-rust/crates/inference-messages/tests/messages/request.rs b/litellm-rust/crates/inference-messages/tests/messages/request.rs index 216aa96fbe6..3784eb6cc68 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/request.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/request.rs @@ -530,6 +530,30 @@ async fn unsupported_params_are_dropped_under_drop_params_and_rejected_without_i assert_eq!(sent["max_tokens"], 16); } +#[rstest] +#[tokio::test] +async fn additional_drop_params_clear_a_param_the_model_does_not_support(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + + run_message(MessagesCall { + api_key: Some("sk".into()), + api_base: Some(upstream.uri()), + shaping: MessagesShaping { + capabilities: sampling_removed(), + settings: MessagesSettings { + additional_drop_params: vec!["top_k".into()], + ..MessagesSettings::default() + }, + }, + ..with_fields(call, json!({"top_k": 5})) + }) + .await; + + let sent = only_request(&upstream).await.json(); + assert_eq!(sent.get("top_k"), None); + assert_eq!(sent["max_tokens"], 16); +} + #[rstest] #[case::adaptive_thinking(json!({"type": "adaptive"}), json!({"type": "adaptive", "display": "summarized"}))] #[case::disabled_thinking(json!({"type": "disabled"}), json!({"type": "disabled"}))] diff --git a/litellm-rust/crates/inference-messages/tests/messages/response.rs b/litellm-rust/crates/inference-messages/tests/messages/response.rs index 76813fe0a34..7c2f3250acf 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/response.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/response.rs @@ -2,7 +2,9 @@ use litellm_host::{ interceptors::{ExecutionFacts, ResultSource}, lifecycle::ExecutionEvent, }; -use litellm_inference_messages::{MessagesCallResponse, messages_body}; +use litellm_inference_messages::{ + MessagesBody, MessagesCallResponse, UnencodableHeader, messages_body, +}; use litellm_inference_testing::{ RecordingSecrets, http_config, no_secrets, provider_http, resources, }; @@ -48,6 +50,9 @@ async fn calls_defer_execution_until_polled( let MessagesCallResponse::Complete(response) = future.await.unwrap() else { panic!("expected a completed message"); }; + let MessagesBody::Message(response) = response else { + panic!("expected a message"); + }; assert_eq!( response.content, message_body()["content"].as_array().unwrap().as_slice() @@ -150,6 +155,37 @@ async fn the_message_passes_through_losslessly(call: MessagesCall) { assert_eq!(serde_json::to_value(&message).unwrap(), upstream_body); } +#[rstest] +#[case::without_stop_reason(json!({"stop_sequence": null}))] +#[case::without_stop_fields(json!({}))] +#[tokio::test] +async fn a_message_without_stop_reason_keeps_it_absent(call: MessagesCall, #[case] stop: Value) { + let mut upstream_body = json!({ + "id": "msg_3", "type": "message", "role": "assistant", "model": MODEL, + "content": [{"type": "text", "text": "hi"}], + "usage": {"input_tokens": 1, "output_tokens": 1} + }); + upstream_body + .as_object_mut() + .unwrap() + .extend(stop.as_object().unwrap().clone()); + let upstream = upstream([json_response(upstream_body.clone())]).await; + + let output = run(MessagesCall { + api_key: Some("sk".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await + .expect("messages call succeeds"); + + let MessagesOutput::Complete(MessagesBody::Unchecked(body)) = output else { + panic!("a message without stop_reason is returned as sent"); + }; + upstream_body["stop_sequence"] = Value::Null; + assert_eq!(Value::Object(body), upstream_body); +} + #[rstest] #[tokio::test] async fn a_json_error_envelope_is_kept_verbatim(call: MessagesCall) { @@ -240,10 +276,42 @@ async fn an_unparseable_success_body_is_reported_as_the_upstream_answer(call: Me assert_eq!(error.upstream_response(), Some((200, "not json"))); } +#[rstest] +#[case::error_object( + json!({"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}), + json!({"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}, + "stop_sequence": null}), +)] +#[case::message_missing_id( + json!({"type": "message", "content": [], "stop_sequence": "###"}), + json!({"type": "message", "content": [], "stop_sequence": "###"}), +)] +#[tokio::test] +async fn a_success_body_that_is_not_a_message_passes_through( + call: MessagesCall, + #[case] body: Value, + #[case] expected: Value, +) { + let upstream = upstream([json_response(body)]).await; + + let response = run(MessagesCall { + api_key: Some("sk".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await + .expect("any JSON object is returned"); + + let MessagesOutput::Complete(response) = response else { + panic!("expected a completed response"); + }; + assert_eq!(serde_json::to_value(response).unwrap(), expected); +} + #[rstest] #[tokio::test] async fn an_unreadable_success_body_is_an_invalid_response(call: MessagesCall) { - let upstream = upstream([json_response(json!({"unexpected": true}))]).await; + let upstream = upstream([json_response(json!(["unexpected"]))]).await; let error = run(MessagesCall { api_key: Some("sk".into()), @@ -275,6 +343,88 @@ async fn a_provider_slower_than_the_timeout_fails_the_call(call: MessagesCall) { assert_eq!(error.upstream_response(), None); } +fn with_settings(call: MessagesCall, settings: MessagesSettings) -> MessagesCall { + MessagesCall { + shaping: MessagesShaping { + settings, + ..call.shaping + }, + ..call + } +} + +#[rstest] +#[case::unary(false)] +#[case::stream(true)] +#[tokio::test] +async fn a_non_ascii_response_header_fails_the_call_when_python_reads_heads_as_ascii( + call: MessagesCall, + #[case] stream: bool, +) { + let upstream = upstream([message_response().insert_header("x-fz-unicode", "café")]).await; + let call = with_fields( + MessagesCall { + api_key: Some("sk".into()), + api_base: Some(upstream.uri()), + ..call + }, + json!({"stream": stream}), + ); + let settings = MessagesSettings { + ascii_response_headers: true, + ..MessagesSettings::default() + }; + + let error = run(with_settings(call, settings)) + .await + .expect_err("the head cannot be encoded"); + + assert_eq!( + litellm_error::find::(&error), + Some(&UnencodableHeader::Received("café".as_bytes().to_vec())) + ); +} + +#[rstest] +#[tokio::test] +async fn a_non_ascii_response_header_is_served_when_python_keeps_raw_heads(call: MessagesCall) { + let upstream = upstream([message_response().insert_header("x-fz-unicode", "café")]).await; + + let message = run_message(MessagesCall { + api_key: Some("sk".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await; + + assert_eq!(message.id, "msg_1"); +} + +#[rstest] +#[tokio::test] +async fn an_unencodable_request_header_fails_before_anything_is_sent(call: MessagesCall) { + let upstream = upstream([message_response()]).await; + let call = MessagesCall { + api_key: Some("sk".into()), + api_base: Some(upstream.uri()), + ..call + }; + let settings = MessagesSettings { + unencodable_request_header: Some(b"h\xed\xa0\x80".to_vec()), + ..MessagesSettings::default() + }; + + let error = run(with_settings(call, settings)) + .await + .expect_err("the header cannot be encoded"); + + assert_eq!( + litellm_error::find::(&error), + Some(&UnencodableHeader::Sent(b"h\xed\xa0\x80".to_vec())) + ); + assert!(received(&upstream).await.is_empty()); +} + #[rstest] #[tokio::test] async fn a_transport_error_does_not_echo_the_provider_url(call: MessagesCall) { @@ -320,6 +470,9 @@ async fn the_facade_sends_through_the_injected_http_pool_configuration(call: Mes let MessagesCallResponse::Complete(message) = response else { panic!("a non-streaming request returns a message"); }; + let MessagesBody::Message(message) = message else { + panic!("expected a message"); + }; assert_eq!(message.id, "msg_1"); let sent = only_request(&upstream).await; assert_eq!(sent.header("x-api-key"), Some("sk-ant")); @@ -327,7 +480,7 @@ async fn the_facade_sends_through_the_injected_http_pool_configuration(call: Mes } #[rstest] -#[case::mistyped_param(json!({"model": MODEL, "messages": [], "max_tokens": "16"}))] +#[case::mistyped_param(json!({"model": MODEL, "messages": "hello", "max_tokens": 16}))] #[case::missing_messages(json!({"model": MODEL, "max_tokens": 16}))] fn a_body_that_does_not_parse_is_an_invalid_request(#[case] raw: Value) { let error = messages_body(object(raw)).expect_err("the body is rejected"); @@ -412,6 +565,9 @@ async fn route_uses_injected_dependencies_and_optional_cache( else { panic!("expected a completed message"); }; + let MessagesBody::Message(response) = response else { + panic!("expected a message"); + }; assert_eq!( response.content, message_body()["content"].as_array().unwrap().as_slice() @@ -484,6 +640,9 @@ async fn cache_overrides_preserve_the_routes_isolated_scope(call: MessagesCall) else { panic!("expected a completed message"); }; + let MessagesBody::Message(response) = response else { + panic!("expected a message"); + }; assert_eq!(response.id, expected["id"].as_str().unwrap()); } assert_eq!(received(&upstream).await.len(), 2); diff --git a/litellm-rust/crates/inference-messages/tests/messages/retry.rs b/litellm-rust/crates/inference-messages/tests/messages/retry.rs new file mode 100644 index 00000000000..e48482dc8b0 --- /dev/null +++ b/litellm-rust/crates/inference-messages/tests/messages/retry.rs @@ -0,0 +1,364 @@ +use futures_util::TryStreamExt; +use litellm_host::{ + interceptors::{RetriedRequest, RetryReason}, + lifecycle::{CallEvent, ExecutionEvent}, +}; +use litellm_inference_messages::MessagesCallResponse; +use litellm_inference_testing::RecordingSecrets; +use litellm_llms::base_llm::messages::context::MessagesModelCapabilities; +use rstest::rstest; + +use super::*; + +const INVALID_SIGNATURE: &str = "messages.1.content.0: Invalid `signature` in `thinking` block"; +const BOUND_ELSEWHERE: &str = "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation."; + +fn rejection(status: u16, message: &str) -> ResponseTemplate { + status_response( + status, + json!({"type": "error", "error": {"type": "invalid_request_error", "message": message}}), + ) +} + +fn sse_response() -> ResponseTemplate { + ResponseTemplate::new(200).set_body_raw( + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + "text/event-stream", + ) +} + +fn replay(call: MessagesCall, api_base: String, stream: bool, thinking: Value) -> MessagesCall { + MessagesCall { + api_key: Some("sk-ant".into()), + api_base: Some(api_base), + ..with_fields( + call, + json!({ + "max_tokens": 4096, + "stream": stream, + "thinking": thinking, + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "plan", "signature": "sig"}, + {"type": "text", "text": "hello"} + ]}, + {"role": "assistant", "content": [{"type": "redacted_thinking", "data": "opaque"}]}, + {"role": "user", "content": "again"} + ] + }), + ) + } +} + +fn enabled() -> Value { + json!({"type": "enabled", "budget_tokens": 2048}) +} + +async fn execute( + host: &RecordingCall, +) -> Result<(MessagesCallResponse, Vec), Error> { + let route = messages_route(Arc::new(RecordingSecrets::empty())); + let response = route + .execute(host.request()?, host, Some(host.events.0.sender.clone())) + .await?; + match response { + MessagesCallResponse::Stream { head, chunks } => Ok(( + MessagesCallResponse::Stream { + head, + chunks: Box::pin(futures_util::stream::empty()), + }, + chunks.try_collect().await?, + )), + complete => Ok((complete, Vec::new())), + } +} + +fn observed_retries(host: &RecordingCall) -> Vec { + host.events + .0 + .lock() + .unwrap() + .iter() + .filter_map(|event| match event { + CallEvent::Execution(ExecutionEvent::ProviderRequestRetried { retry }) => { + Some(retry.clone()) + } + _ => None, + }) + .collect() +} + +#[rstest] +#[case::invalid_signature(INVALID_SIGNATURE)] +#[case::missing_signature( + "messages.1.content.0.thinking.signature.str: Input should be a valid string" +)] +#[case::empty_thinking("messages.1.content.0.thinking: each thinking block must contain thinking")] +#[case::unpinned_binding_mismatch(BOUND_ELSEWHERE)] +#[tokio::test] +async fn a_rejected_thinking_replay_is_sent_once_more_without_thinking( + call: MessagesCall, + #[case] message: &str, + #[values(false, true)] stream: bool, + #[values("anthropic", "azure_ai", "deepseek", "minimax")] provider: &str, +) { + let success = if stream { + sse_response() + } else { + message_response() + }; + let upstream = upstream([rejection(400, message), success]).await; + let host = RecordingCall::::new(replay( + MessagesCall { + custom_llm_provider: Some(provider.into()), + ..call + }, + upstream.uri(), + stream, + enabled(), + )); + + let (response, chunks) = execute(&host).await.expect("the retry succeeds"); + + let [first, second] = <[_; 2]>::try_from(received(&upstream).await).unwrap(); + assert_eq!(first.json()["thinking"], enabled()); + assert_eq!( + second.json()["messages"], + json!([ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [{"type": "text", "text": "hello"}]}, + {"role": "user", "content": "again"} + ]) + ); + let mut expected = first.json(); + expected.as_object_mut().unwrap().shift_remove("thinking"); + expected["messages"] = second.json()["messages"].clone(); + assert_eq!(second.json(), expected); + assert_eq!(first.header("x-hook"), second.header("x-hook")); + let expected_retry = RetriedRequest { + reasons: vec![RetryReason::ThinkingSignature], + body: second.json(), + }; + assert_eq!(*host.retried.lock().unwrap(), vec![expected_retry.clone()]); + assert_eq!(observed_retries(&host), vec![expected_retry]); + assert_eq!( + matches!(response, MessagesCallResponse::Stream { .. }), + stream + ); + assert_eq!(chunks.is_empty(), !stream); +} + +#[rstest] +#[tokio::test] +async fn a_second_rejection_is_the_error_the_caller_sees( + call: MessagesCall, + #[values(false, true)] stream: bool, +) { + let upstream = upstream([ + rejection(400, INVALID_SIGNATURE), + rejection( + 400, + "messages.0.content.0: Invalid `signature` in `thinking` block, again", + ), + ]) + .await; + let host = RecordingCall::::new(replay(call, upstream.uri(), stream, enabled())); + + let error = execute(&host).await.err().expect("the retry fails"); + + assert_eq!(received(&upstream).await.len(), 2); + let (status, body) = error.upstream_response().unwrap(); + assert_eq!(status, 400); + assert!(body.contains("again"), "{body}"); + assert_eq!(host.retried.lock().unwrap().len(), 1); +} + +#[rstest] +#[case::not_a_400(429, INVALID_SIGNATURE, json!({"type": "adaptive"}))] +#[case::unrelated_400(400, "max_tokens: 4096 > 64000", json!({"type": "adaptive"}))] +#[case::signature_without_thinking(400, "Invalid `signature` in `tool_use` block", json!({"type": "adaptive"}))] +#[case::caller_pinned_binding( + 400, + BOUND_ELSEWHERE, + json!({"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}) +)] +#[tokio::test] +async fn other_failures_reach_the_caller_without_a_retry( + call: MessagesCall, + #[case] status: u16, + #[case] message: &str, + #[case] thinking: Value, +) { + let upstream = upstream([rejection(status, message), message_response()]).await; + let adaptive = MessagesShaping { + capabilities: MessagesModelCapabilities { + supports_adaptive_thinking: true, + ..MessagesModelCapabilities::default() + }, + ..MessagesShaping::default() + }; + let host = RecordingCall::::new(replay( + MessagesCall { + shaping: adaptive, + ..call + }, + upstream.uri(), + false, + thinking.clone(), + )); + + let error = execute(&host).await.err().expect("the call fails"); + + let sent = only_request(&upstream).await; + assert_eq!(sent.json()["thinking"], thinking); + assert_eq!(error.upstream_response().unwrap().0, status); + assert!(host.retried.lock().unwrap().is_empty()); + assert!(observed_retries(&host).is_empty()); +} + +const STRICT_REJECTION: &str = "Too many strict tools (21). The maximum is 20."; + +fn strict_tools( + call: MessagesCall, + api_base: String, + stream: bool, + drop_params: bool, +) -> MessagesCall { + MessagesCall { + api_key: Some("sk-ant".into()), + api_base: Some(api_base), + shaping: MessagesShaping { + settings: MessagesSettings { + drop_params, + drops_unsupported_params: drop_params, + ..MessagesSettings::default() + }, + ..MessagesShaping::default() + }, + ..with_fields( + call, + json!({ + "max_tokens": 64, + "stream": stream, + "messages": [{"role": "user", "content": "hi"}], + "tools": [ + {"name": "a", "input_schema": {"type": "object"}, "strict": true}, + {"name": "b", "input_schema": {"type": "object"}, "strict": false}, + {"name": "c", "input_schema": {"type": "object"}} + ] + }), + ) + } +} + +#[rstest] +#[tokio::test] +async fn a_rejected_strict_tool_is_sent_once_more_without_strict_under_drop_params( + call: MessagesCall, + #[values(false, true)] stream: bool, +) { + let success = if stream { + sse_response() + } else { + message_response() + }; + let upstream = upstream([rejection(400, STRICT_REJECTION), success]).await; + let host = RecordingCall::::new(strict_tools(call, upstream.uri(), stream, true)); + + let (response, _) = execute(&host).await.expect("the retry succeeds"); + + let [first, second] = <[_; 2]>::try_from(received(&upstream).await).unwrap(); + assert_eq!(first.json()["tools"][0]["strict"], json!(true)); + let mut expected = first.json(); + expected["tools"] = json!([ + {"name": "a", "input_schema": {"type": "object"}}, + {"name": "b", "input_schema": {"type": "object"}}, + {"name": "c", "input_schema": {"type": "object"}} + ]); + assert_eq!(second.json(), expected); + let expected_retry = RetriedRequest { + reasons: vec![RetryReason::StrictTools { dropped: 1 }], + body: second.json(), + }; + assert_eq!(*host.retried.lock().unwrap(), vec![expected_retry.clone()]); + assert_eq!(observed_retries(&host), vec![expected_retry]); + assert_eq!( + matches!(response, MessagesCallResponse::Stream { .. }), + stream + ); +} + +#[rstest] +#[tokio::test] +async fn a_rejected_strict_tool_reaches_the_caller_without_drop_params(call: MessagesCall) { + let upstream = upstream([rejection(400, STRICT_REJECTION), message_response()]).await; + let host = RecordingCall::::new(strict_tools(call, upstream.uri(), false, false)); + + let error = execute(&host).await.err().expect("the call fails"); + + assert_eq!( + only_request(&upstream).await.json()["tools"][0]["strict"], + json!(true) + ); + assert_eq!(error.upstream_response().unwrap().0, 400); + assert!(observed_retries(&host).is_empty()); +} + +#[rstest] +#[tokio::test] +async fn strict_tools_and_a_thinking_replay_share_the_one_retry(call: MessagesCall) { + let upstream = upstream([ + rejection(400, STRICT_REJECTION), + rejection(400, INVALID_SIGNATURE), + message_response(), + ]) + .await; + let host = RecordingCall::::new(replay( + strict_tools(call, upstream.uri(), false, true), + upstream.uri(), + false, + enabled(), + )); + + let error = execute(&host) + .await + .err() + .expect("the second rejection is raised"); + + let [first, second] = <[_; 2]>::try_from(received(&upstream).await).unwrap(); + assert_eq!(first.json()["thinking"], enabled()); + assert_eq!(second.json()["thinking"], enabled()); + assert_eq!( + second.json()["tools"][0], + json!({"name": "a", "input_schema": {"type": "object"}}) + ); + assert_eq!(received(&upstream).await.len(), 2); + assert!(error.upstream_response().unwrap().1.contains("signature")); +} + +#[rstest] +#[tokio::test] +async fn a_truthy_global_drop_params_that_is_not_true_does_not_drop_strict(call: MessagesCall) { + let upstream = upstream([rejection(400, STRICT_REJECTION), message_response()]).await; + let strict = strict_tools(call, upstream.uri(), false, true); + let host = RecordingCall::::new(MessagesCall { + shaping: MessagesShaping { + settings: MessagesSettings { + drops_unsupported_params: false, + ..strict.shaping.settings.clone() + }, + ..strict.shaping.clone() + }, + ..strict + }); + + let error = execute(&host).await.err().expect("the call fails"); + + assert_eq!( + only_request(&upstream).await.json()["tools"][0]["strict"], + json!(true) + ); + assert_eq!(error.upstream_response().unwrap().0, 400); + assert!(observed_retries(&host).is_empty()); +} diff --git a/litellm-rust/crates/inference-messages/tests/messages/stream.rs b/litellm-rust/crates/inference-messages/tests/messages/stream.rs index 99ba08495aa..c9738fa2a0d 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/stream.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/stream.rs @@ -451,7 +451,7 @@ async fn stalling_error_upstream() -> (String, JoinHandle<()>) { #[rstest] #[tokio::test] -async fn the_timeout_covers_a_stalled_stream_error_body(call: MessagesCall) { +async fn a_stalled_stream_error_body_reads_as_empty(call: MessagesCall) { let (base, connection) = stalling_error_upstream().await; let host = RecordingStreamHost::new( MessagesCall { @@ -467,7 +467,7 @@ async fn the_timeout_covers_a_stalled_stream_error_body(call: MessagesCall) { .expect_err("a stalled error body fails the call"); assert!(error.dispatched(), "{error:?}"); - assert_eq!(error.upstream_response(), None); + assert_eq!(error.upstream_response(), Some((429, ""))); assert!(host.seen.into_inner().unwrap().is_empty()); tokio::time::timeout(Duration::from_secs(5), connection) .await diff --git a/litellm-rust/crates/inference-messages/tests/messages/transport.rs b/litellm-rust/crates/inference-messages/tests/messages/transport.rs new file mode 100644 index 00000000000..f561b09e55f --- /dev/null +++ b/litellm-rust/crates/inference-messages/tests/messages/transport.rs @@ -0,0 +1,256 @@ +use std::collections::VecDeque; + +use bytes::Bytes; +use futures_util::{StreamExt, TryStreamExt, future::BoxFuture}; +use litellm_inference_messages::{ + Delivered, MessagesCallResponse, Transport, TransportBody, TransportRequest, TransportResponse, +}; +use litellm_inference_testing::RecordingSecrets; +use rstest::rstest; + +use super::*; + +const INVALID_SIGNATURE: &str = "messages.1.content.0: Invalid `signature` in `thinking` block"; +const UNREACHABLE_BASE: &str = "http://127.0.0.1:9"; + +#[derive(Debug, thiserror::Error)] +#[error("raised by the host client")] +struct Raised; + +enum Scripted { + Complete(Value), + Stream(&'static [&'static str]), + Rejected(u16, &'static str), + Failed, +} + +struct Sent { + url: String, + headers: Vec<(String, String)>, + body: Value, + stream: bool, +} + +#[derive(Default)] +struct ScriptedTransport { + script: Mutex>, + sent: Mutex>, +} + +impl ScriptedTransport { + fn new(script: impl IntoIterator) -> Arc { + Arc::new(Self { + script: Mutex::new(script.into_iter().collect()), + sent: Mutex::default(), + }) + } + + fn sent(&self) -> std::sync::MutexGuard<'_, Vec> { + self.sent.lock().unwrap() + } +} + +impl Transport for ScriptedTransport { + fn send( + &self, + request: TransportRequest, + ) -> BoxFuture<'static, Result> { + let TransportBody::Json(body) = request.body else { + panic!("an unsigned request hands the host its JSON body"); + }; + self.sent.lock().unwrap().push(Sent { + url: request.url, + headers: request.headers, + body, + stream: request.stream, + }); + let next = self + .script + .lock() + .unwrap() + .pop_front() + .expect("a scripted answer"); + Box::pin(async move { + let (status, delivered) = match next { + Scripted::Complete(body) => (200, Delivered::Complete(body.to_string())), + Scripted::Stream(frames) => ( + 200, + Delivered::Stream( + futures_util::stream::iter( + frames.iter().map(|frame| Ok(Bytes::from_static(frame.as_bytes()))), + ) + .boxed(), + ), + ), + Scripted::Rejected(status, message) => ( + status, + Delivered::Rejected { + text: json!({"type": "error", "error": {"type": "invalid_request_error", "message": message}}) + .to_string(), + error: Error::transport(Raised), + }, + ), + Scripted::Failed => return Err(Error::transport(Raised)), + }; + Ok(TransportResponse { + status, + headers: vec![("request-id".into(), "req_1".into())], + delivered, + }) + }) + } +} + +fn hosted_call(call: MessagesCall, fields: Value) -> MessagesCall { + MessagesCall { + api_key: Some("sk-ant".into()), + api_base: Some(UNREACHABLE_BASE.into()), + ..with_fields(call, fields) + } +} + +async fn execute( + transport: Arc, + call: MessagesCall, +) -> Result<(MessagesCallResponse, Vec), Error> { + let host = RecordingCall::::new(call); + let route = messages_route(Arc::new(RecordingSecrets::empty())).with_transport(transport); + match route + .execute(host.request()?, &host, Some(host.events.0.sender.clone())) + .await? + { + MessagesCallResponse::Stream { head, chunks } => Ok(( + MessagesCallResponse::Stream { + head, + chunks: Box::pin(futures_util::stream::empty()), + }, + chunks.try_collect().await?, + )), + complete => Ok((complete, Vec::new())), + } +} + +#[rstest] +#[tokio::test] +async fn the_host_transport_sends_the_wire_request_instead_of_the_native_client( + call: MessagesCall, +) { + let transport = ScriptedTransport::new([Scripted::Complete(message_body())]); + + let (response, _) = execute(transport.clone(), hosted_call(call, json!({}))) + .await + .expect("the host transport answers"); + + let MessagesCallResponse::Complete(MessagesBody::Message(message)) = response else { + panic!("a complete message"); + }; + assert_eq!(message.id, "msg_1"); + let sent = transport.sent(); + let [request] = sent.as_slice() else { + panic!("one request"); + }; + assert_eq!(request.url, format!("{UNREACHABLE_BASE}/v1/messages")); + assert!(!request.stream); + assert_eq!(request.body["model"], MODEL); + assert!( + request + .headers + .iter() + .any(|(name, value)| name == "x-api-key" && value == "sk-ant"), + "{:?}", + request.headers + ); +} + +#[rstest] +#[tokio::test] +async fn a_host_stream_reaches_the_caller_chunk_by_chunk(call: MessagesCall) { + let frames: &'static [&'static str] = &[ + "event: message_start\ndata: {\"type\":\"message_start\"}\n\n", + "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + ]; + let transport = ScriptedTransport::new([Scripted::Stream(frames)]); + + let (response, chunks) = execute( + transport.clone(), + hosted_call(call, json!({"stream": true})), + ) + .await + .expect("the host stream opens"); + + let MessagesCallResponse::Stream { head, .. } = response else { + panic!("a stream"); + }; + assert_eq!( + head.headers, + vec![("request-id".to_string(), "req_1".to_string())] + ); + assert_eq!( + chunks, + frames + .iter() + .map(|frame| Bytes::from_static(frame.as_bytes())) + .collect::>() + ); + assert!(transport.sent()[0].stream); +} + +fn replayed_thinking() -> Value { + json!({ + "max_tokens": 4096, + "thinking": {"type": "enabled", "budget_tokens": 2048}, + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "plan", "signature": "sig"}, + {"type": "text", "text": "hello"} + ]}, + {"role": "user", "content": "again"} + ] + }) +} + +#[rstest] +#[tokio::test] +async fn a_recoverable_rejection_is_retried_through_the_host_transport(call: MessagesCall) { + let transport = ScriptedTransport::new([ + Scripted::Rejected(400, INVALID_SIGNATURE), + Scripted::Complete(message_body()), + ]); + + execute(transport.clone(), hosted_call(call, replayed_thinking())) + .await + .expect("the retry succeeds"); + + let sent = transport.sent(); + let [first, second] = sent.as_slice() else { + panic!("two requests"); + }; + assert!(first.body.get("thinking").is_some()); + assert!(second.body.get("thinking").is_none()); +} + +#[rstest] +#[case::final_rejection(vec![Scripted::Rejected(429, "slow down")])] +#[case::spent_retry(vec![ + Scripted::Rejected(400, INVALID_SIGNATURE), + Scripted::Rejected(400, INVALID_SIGNATURE), +])] +#[case::client_failure(vec![Scripted::Failed])] +#[tokio::test] +async fn the_caller_gets_the_error_the_host_transport_raised( + call: MessagesCall, + #[case] script: Vec, +) { + let attempts = script.len(); + let transport = ScriptedTransport::new(script); + + let error = execute(transport.clone(), hosted_call(call, replayed_thinking())) + .await + .err() + .expect("the call fails"); + + assert!(litellm_error::find::(&error).is_some(), "{error:?}"); + assert_eq!(error.upstream_response(), None); + assert_eq!(transport.sent().len(), attempts); +} diff --git a/litellm-rust/crates/inference-messages/tests/support/mod.rs b/litellm-rust/crates/inference-messages/tests/support/mod.rs index 3cecc865070..9115b6e89bb 100644 --- a/litellm-rust/crates/inference-messages/tests/support/mod.rs +++ b/litellm-rust/crates/inference-messages/tests/support/mod.rs @@ -117,6 +117,7 @@ pub struct RecordingCall { pub events: Arc, pub chunks: Mutex>, pub head: Mutex>, + pub retried: Mutex>, } #[derive(Default)] @@ -167,6 +168,7 @@ impl RecordingCall

{ events: Arc::new(CallEvents::default()), chunks: Mutex::new(Vec::new()), head: Mutex::new(None), + retried: Mutex::new(Vec::new()), } } } @@ -195,6 +197,14 @@ impl litellm_host::interceptors::Intercepto ) -> Result<(), P::Error> { Ok(()) } + + async fn provider_request_retried( + &self, + retry: litellm_host::interceptors::RetriedRequest, + ) -> Result<(), P::Error> { + self.retried.lock().unwrap().push(retry); + Ok(()) + } } impl

RecordingCall

diff --git a/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs b/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs index 9f59bb9a2a2..509704340fe 100644 --- a/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs +++ b/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs @@ -22,6 +22,7 @@ pub(crate) fn event_name(event: &CallEvent) -> &'static str { CallEvent::Execution(ExecutionEvent::ResultReady { .. }) => "result_ready", CallEvent::Started { .. } => "started", CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { .. }) => "response", + CallEvent::Execution(ExecutionEvent::ProviderRequestRetried { .. }) => "retried", CallEvent::Succeeded { .. } => "success", CallEvent::Failed { .. } => "failure", CallEvent::Cancelled { .. } => "cancelled", diff --git a/litellm-rust/crates/inference-ocr/tests/ocr/machine.rs b/litellm-rust/crates/inference-ocr/tests/ocr/machine.rs index c7a5d58ad3d..9da62e6ccf4 100644 --- a/litellm-rust/crates/inference-ocr/tests/ocr/machine.rs +++ b/litellm-rust/crates/inference-ocr/tests/ocr/machine.rs @@ -66,6 +66,11 @@ async fn drive_until( reply.send(host.after_provider_response(raw).await); Ok(()) } + HostRequest::Intercept(InterceptRequest::ProviderRequestRetried { reply, .. }) => { + ops.push("retried"); + reply.send(Ok(())); + Ok(()) + } }; if let Err(failure) = answer { break machine.interrupt(failure).await.map(completed); @@ -86,6 +91,7 @@ async fn drive_until_notified(machine: &mut OcrMachine, host: &LocalOcrHost, sto MachineStep::Suspended(HostRequest::HostCall(op)) => host.handle_host_call(op).await.unwrap(), MachineStep::Suspended(HostRequest::Intercept(InterceptRequest::BeforeProviderRequest { wire, reply, .. })) => reply.send(*wire), MachineStep::Suspended(HostRequest::Intercept(InterceptRequest::AfterProviderResponse { reply, .. })) => reply.send(Ok(())), + MachineStep::Suspended(HostRequest::Intercept(InterceptRequest::ProviderRequestRetried { reply, .. })) => reply.send(Ok(())), MachineStep::Suspended(HostRequest::Stream(stream)) => match stream { litellm_host::protocol::StreamDelivery::Open(head, _) => match head {}, litellm_host::protocol::StreamDelivery::Chunk(chunk, _) => match chunk {}, diff --git a/litellm-rust/crates/inference-ocr/tests/resources.rs b/litellm-rust/crates/inference-ocr/tests/resources.rs index d618647e89b..3132a837eaf 100644 --- a/litellm-rust/crates/inference-ocr/tests/resources.rs +++ b/litellm-rust/crates/inference-ocr/tests/resources.rs @@ -7,7 +7,7 @@ use std::sync::{ use litellm_auth::AuthServices; use litellm_auth_gcp::{ - CredentialSource, VertexAuth, VertexAuthFuture, VertexProviderLoader, VertexTokenSource, + CredentialSource, GcpAuthFuture, GcpProviderLoader, GcpTokenSource, VertexAuth, }; use litellm_http::{HttpSettings, Resolution}; use litellm_inference::resources::CoreResources; @@ -20,28 +20,28 @@ use support::{ReceivedRequest, json_response, upstream}; struct TokenSource(String); -impl VertexTokenSource for TokenSource { - fn project_id(&self) -> VertexAuthFuture<'_, String> { +impl GcpTokenSource for TokenSource { + fn project_id(&self) -> GcpAuthFuture<'_, String> { Box::pin(async { Ok(self.0.clone()) }) } - fn token(&self) -> VertexAuthFuture<'_, String> { - Box::pin(async { Ok(self.0.clone()) }) + fn token(&self) -> GcpAuthFuture<'_, litellm_secret_value::SecretValue> { + Box::pin(async { Ok(litellm_secret_value::SecretValue::new(self.0.clone())) }) } } #[derive(Default)] struct Loader(AtomicUsize); -impl VertexProviderLoader for Loader { - fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc> { +impl GcpProviderLoader for Loader { + fn load(&self, source: CredentialSource) -> GcpAuthFuture<'_, Arc> { Box::pin(async move { self.0.fetch_add(1, Ordering::SeqCst); let identity = match source { CredentialSource::Trusted(secret) => secret.expose().to_string(), other => panic!("unexpected credential source: {other:?}"), }; - Ok(Arc::new(TokenSource(identity)) as Arc) + Ok(Arc::new(TokenSource(identity)) as Arc) }) } } diff --git a/litellm-rust/crates/inference-responses/src/ids.rs b/litellm-rust/crates/inference-responses/src/ids.rs index b51aea1559e..fbba8f949a0 100644 --- a/litellm-rust/crates/inference-responses/src/ids.rs +++ b/litellm-rust/crates/inference-responses/src/ids.rs @@ -11,7 +11,7 @@ //! | Id | Applied to | Decoded plaintext | //! |---|---|---| //! | `resp_` | the response `id` | `litellm:custom_llm_provider:{provider};model_id:{model_id};response_id:{id}` | -//! | `encitem_` | the `id` of an output item with `encrypted_content` | `litellm:model_id:{model_id};item_id:{id}` | +//! | `encitem_` | the `id` of a reasoning or compaction output item, or of any item with `encrypted_content` | `litellm:model_id:{model_id};item_id:{id}` | //! | `litellm_enc:;{content}` | that item's `encrypted_content` | `` is `model_id:{model_id}`, and the original content follows the `;` unencoded | //! | `cntr_` | `container_id` on output items, `code_interpreter_call`, and message annotations | `litellm:custom_llm_provider:{provider};model_id:{model_id};container_id:{id}` | //! @@ -25,8 +25,9 @@ //! - `encitem_` and `litellm_enc:` carry encrypted-content affinity: encrypted reasoning only //! decrypts under the API key that produced it, so the router pins replays to that deployment. //! They are stamped only when the router sets `encrypted_content_affinity_enabled` and a -//! deployment `model_id` is known (see [`IdScope`]). Items without an `id`, such as Codex's, -//! carry the deployment in the wrapped content alone. +//! deployment `model_id` is known (see [`IdScope`]). Reasoning and compaction items get an +//! `encitem_` id by type, so it matches the one Python streams before their content arrives. +//! Items without an `id`, such as Codex's, carry the deployment in the wrapped content alone. //! - `cntr_` lets container calls find the provider and deployment that own a container. A //! container id that already names a provider is left as it is. //! - On the way in, [`restore_input`] turns `encitem_` ids and `litellm_enc:` content in the @@ -51,6 +52,7 @@ const RESPONSE_PREFIX: &str = "resp_"; const ITEM_PREFIX: &str = "encitem_"; const CONTAINER_PREFIX: &str = "cntr_"; const WRAPPED_CONTENT_PREFIX: &str = "litellm_enc:"; +const ENCRYPTED_ITEM_TYPES: [&str; 2] = ["reasoning", "compaction"]; /// Python's non-strict `base64.b64decode`: characters outside the alphabet are skipped and /// non-zero trailing bits are accepted @@ -211,15 +213,23 @@ pub fn decode_container_id(id: &str) -> DecodedContainerId { } fn encode_encrypted_item(item: &mut Map, model_id: &str) { - let Some(Value::String(content)) = item.get_mut("encrypted_content") else { - return; + let wrapped = match item.get_mut("encrypted_content") { + Some(Value::String(content)) if !content.is_empty() => { + *content = wrap_encrypted_content(content, model_id); + true + } + _ => false, }; - if content.is_empty() { + let encrypted_type = item + .get("type") + .and_then(Value::as_str) + .is_some_and(|item_type| ENCRYPTED_ITEM_TYPES.contains(&item_type)); + if !wrapped && !encrypted_type { return; } - *content = wrap_encrypted_content(content, model_id); if let Some(Value::String(id)) = item.get_mut("id") && !id.is_empty() + && decode_encrypted_item_id(id).is_none() { *id = encrypted_item_id(model_id, id); } @@ -456,6 +466,33 @@ mod tests { ); } + #[rstest] + #[case::reasoning_without_ciphertext( + json!({"type": "reasoning", "id": "rs_0a1b", "summary": []}), + json!({"type": "reasoning", "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", "summary": []}) + )] + #[case::compaction( + json!({"type": "compaction", "id": "cmp_0a1b", "encrypted_content": "gAAAA"}), + json!({ + "type": "compaction", + "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOmNtcF8wYTFi", + "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA" + }) + )] + #[case::already_encoded( + json!({"type": "reasoning", "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", "summary": []}), + json!({"type": "reasoning", "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", "summary": []}) + )] + #[case::message( + json!({"type": "message", "id": "msg_1", "content": []}), + json!({"type": "message", "id": "msg_1", "content": []}) + )] + fn encodes_item_ids_by_type_under_affinity(#[case] item: Value, #[case] expected: Value) { + let mut encoded = response("resp_1", json!([item])); + encode_response(&mut encoded, &scope(Some("openai"), Some("deploy-1"), true)); + assert_eq!(Value::Array(encoded.output), json!([expected])); + } + #[rstest] #[case::affinity_disabled(scope(Some("openai"), Some("deploy-1"), false))] #[case::no_model_id(scope(Some("openai"), None, true))] diff --git a/litellm-rust/crates/inference/AGENTS.md b/litellm-rust/crates/inference/AGENTS.md index bcbc98a209e..bc9be4825ea 100644 --- a/litellm-rust/crates/inference/AGENTS.md +++ b/litellm-rust/crates/inference/AGENTS.md @@ -81,3 +81,4 @@ - Cache lookup follows provider preparation, credential resolution, and the request interceptor - keys describe the effective provider URL, authenticated headers, and rewritten body - signed requests bypass caching until the signing identity has a stable cache representation +- A service that keys entries by the caller's arguments (`keyed_by_caller`, the Python host's legacy cache) is looked up before provider preparation, as Python's `@client` wrapper does, and a hit skips request building and the request interceptor diff --git a/litellm-rust/crates/inference/Cargo.toml b/litellm-rust/crates/inference/Cargo.toml index 807a3ea85d3..168b9a569ea 100644 --- a/litellm-rust/crates/inference/Cargo.toml +++ b/litellm-rust/crates/inference/Cargo.toml @@ -12,6 +12,7 @@ litellm-framer.workspace = true tokio-util = { version = "0.7", features = ["codec"] } litellm-secrets.workspace = true litellm-core-utils.workspace = true +litellm-error.workspace = true litellm-host.workspace = true bytes.workspace = true futures-util.workspace = true @@ -29,7 +30,6 @@ time.workspace = true url.workspace = true [dev-dependencies] -litellm-error.workspace = true litellm-cache-memory.workspace = true litellm-http = { workspace = true, features = ["test-support"] } litellm-llms = { workspace = true, features = ["test-support"] } diff --git a/litellm-rust/crates/inference/src/caching.rs b/litellm-rust/crates/inference/src/caching.rs index d9b5b891015..fa9ceb6bd73 100644 --- a/litellm-rust/crates/inference/src/caching.rs +++ b/litellm-rust/crates/inference/src/caching.rs @@ -61,6 +61,10 @@ pub trait StreamCachable: Cachable { fn replay(data: Bytes) -> Option>; fn bytes(chunk: &Self::Chunk) -> &[u8]; + + fn storable_stream(text: &str) -> bool { + successful_stream(text, Self::TERMINAL_EVENT) + } } #[derive(Serialize, Deserialize)] @@ -238,12 +242,19 @@ impl CallCache

{ identity: &ProviderIdentity, wire: &WireRequest, ) -> Self { + Self::new( + cache, + policy, + CacheRequest::from_wire(identity.clone(), Some(wire)), + ) + } + + pub fn new(cache: Option<&ScopedCache>, policy: CachePolicy, request: CacheRequest) -> Self { let session = cache.and_then(|cache| { if !policy.enabled() { return None; } let options = cache.options(Some(policy)); - let request = CacheRequest::from_wire(identity.clone(), Some(wire)); Some(CacheSession { request: options.request( &cache.service.config().namespace, @@ -281,7 +292,7 @@ impl CallCache

{ P::Response: Serialize, { let Some(session) = self.session.filter(|session| { - *source == ResultSource::Provider && session.request.controls.writes() + !matches!(source, ResultSource::Cache { .. }) && session.request.controls.writes() }) else { return output; }; @@ -322,7 +333,7 @@ fn capture_stream( None => { if let Some(data) = captured && let Ok(text) = String::from_utf8(data) - && successful_stream(&text, P::TERMINAL_EVENT) + && P::storable_stream(&text) && let Ok(entry) = serde_json::to_value(ResponseEnvelope::new( P::SURFACE, CachedOutput::::Stream(text), diff --git a/litellm-rust/crates/inference/src/context.rs b/litellm-rust/crates/inference/src/context.rs index ef2635d24a1..39b259a4c97 100644 --- a/litellm-rust/crates/inference/src/context.rs +++ b/litellm-rust/crates/inference/src/context.rs @@ -1,6 +1,6 @@ use litellm_cache_response::CachePolicy; use litellm_host::{ - interceptors::{ExecutionFacts, Interceptors, RawResponse, ResultSource}, + interceptors::{ExecutionFacts, Interceptors, RawResponse, ResultSource, RetriedRequest}, lifecycle::{CallEvent, ExecutionEvent}, observation::ObservationSender, }; @@ -57,4 +57,15 @@ impl<'a, I: Interceptors> CallContext<'a, I> { .await .map_err(Error::post_call) } + + pub async fn provider_request_retried(&self, retry: RetriedRequest) -> Result<(), Error> { + if let Some(observers) = &self.observers { + observers.emit(CallEvent::Execution( + ExecutionEvent::ProviderRequestRetried { + retry: retry.clone(), + }, + )); + } + self.interceptors.provider_request_retried(retry).await + } } diff --git a/litellm-rust/crates/inference/src/error.rs b/litellm-rust/crates/inference/src/error.rs index 841e0c53799..5f6e1b3f70d 100644 --- a/litellm-rust/crates/inference/src/error.rs +++ b/litellm-rust/crates/inference/src/error.rs @@ -24,6 +24,8 @@ pub enum Error { Secret(#[from] SecretError), #[error("the host driver failed")] Host(#[source] MachineFault), + #[error("the host transport failed")] + Transport(#[source] litellm_error::Source), #[error("post-call hook failed")] PostCallHook(#[source] Arc), } @@ -47,6 +49,10 @@ impl Error { Self::PostCallHook(Arc::new(error)) } + pub fn transport(cause: impl std::error::Error + Send + Sync + 'static) -> Self { + Self::Transport(litellm_error::Source::new(cause)) + } + /// The caller's request is what is wrong, as opposed to the environment, the wire, or /// the provider's answer. pub fn is_request(&self) -> bool { @@ -55,7 +61,11 @@ impl Error { Self::Llm(error) => error.is_request(), Self::Auth(error) => error.is_request(), Self::Http(error) => error.is_request(), - Self::Declined(_) | Self::Secret(_) | Self::Host(_) | Self::PostCallHook(_) => false, + Self::Declined(_) + | Self::Secret(_) + | Self::Host(_) + | Self::Transport(_) + | Self::PostCallHook(_) => false, } } @@ -70,6 +80,7 @@ impl Error { | Self::Headers(_) | Self::Secret(_) | Self::Host(_) + | Self::Transport(_) | Self::PostCallHook(_) => false, } } @@ -85,6 +96,7 @@ impl Error { | Self::Http(_) | Self::Secret(_) | Self::Host(_) + | Self::Transport(_) | Self::PostCallHook(_) => false, } } @@ -100,6 +112,7 @@ impl Error { | Self::Http(_) | Self::Secret(_) | Self::Host(_) + | Self::Transport(_) | Self::PostCallHook(_) => None, } } @@ -121,7 +134,7 @@ impl Error { match self { Self::Llm(error) => error.dispatched(), Self::Http(error) => error.dispatched(), - Self::Host(_) | Self::PostCallHook(_) => true, + Self::Host(_) | Self::Transport(_) | Self::PostCallHook(_) => true, Self::InvalidProvider(_) | Self::UnresolvedProvider { .. } | Self::Declined(_) @@ -142,6 +155,7 @@ impl Error { | Self::Http(_) | Self::Secret(_) | Self::Host(_) + | Self::Transport(_) | Self::PostCallHook(_) => None, } } @@ -164,6 +178,14 @@ impl Error { } } + pub fn body_cut(&self) -> Option { + match self { + Self::Llm(error) => error.body_cut(), + Self::Http(error) => error.body_cut(), + _ => None, + } + } + /// `Some(connect)` when the call timed out, where `connect` says the connection was never /// established; `None` for every other error. pub fn timeout(&self) -> Option { @@ -185,6 +207,7 @@ impl Error { | Self::Headers(_) | Self::Secret(_) | Self::Host(_) + | Self::Transport(_) | Self::PostCallHook(_) => None, } } diff --git a/litellm-rust/crates/inference/src/outbound.rs b/litellm-rust/crates/inference/src/outbound.rs index b0cc71d425d..87fae0bbe9e 100644 --- a/litellm-rust/crates/inference/src/outbound.rs +++ b/litellm-rust/crates/inference/src/outbound.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use litellm_http::outbound::{json, signed}; +use litellm_http::outbound::{json, json_bytes, signed}; use litellm_llms::base_llm::auth::Authenticated; use serde_json::Value; @@ -34,3 +34,18 @@ pub fn outbound_request( Some(signer) => signed(request, &signer), } } + +pub fn raw_leaf_outbound_request( + authenticated: Authenticated, + url: &str, + body: &Value, + timeout: Option, +) -> Result { + let Authenticated { headers, signer } = authenticated; + let bytes = litellm_host::raw_json::to_vec(body).map_err(litellm_http::Error::request_body)?; + let request = json_bytes(url, &headers, bytes, timeout)?; + match signer { + None => Ok(request), + Some(signer) => signed(request, &signer), + } +} diff --git a/litellm-rust/crates/llms-types/src/formats/messages/request.rs b/litellm-rust/crates/llms-types/src/formats/messages/request.rs index d30d21e74e7..2e53b3adc5f 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/request.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/request.rs @@ -320,7 +320,7 @@ impl ThinkingConfig { #[macro_rules_attribute::apply(crate::wire_type)] pub struct MessagesRequest { pub model: String, - pub messages: Vec, + pub messages: Vec>, #[serde(flatten)] pub params: MessagesOptionalParams, } @@ -329,7 +329,7 @@ pub struct MessagesRequest { #[derive(Default)] pub struct MessagesOptionalParams { #[serde(skip_serializing_if = "Option::is_none")] - pub max_tokens: Option, + pub max_tokens: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub system: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -337,13 +337,21 @@ pub struct MessagesOptionalParams { #[serde(skip_serializing_if = "Option::is_none")] pub stop_sequences: Option>, #[serde(skip_serializing_if = "Option::is_none")] - pub stream: Option, + pub stream: Option>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::recognized::exact_float" + )] + pub temperature: Option>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::recognized::exact_float" + )] + pub top_p: Option>, #[serde(skip_serializing_if = "Option::is_none")] - pub temperature: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub top_p: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub top_k: Option, + pub top_k: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub tools: Option>>, #[serde(skip_serializing_if = "Option::is_none")] @@ -374,6 +382,12 @@ pub struct MessagesOptionalParams { pub extra: Map, } +impl MessagesOptionalParams { + pub fn streams(&self) -> bool { + self.stream.as_ref().is_some_and(Recognized::is_truthy) + } +} + impl Message { pub fn blocks(&self) -> &[ContentBlock] { match &self.content { @@ -447,8 +461,8 @@ mod tests { assert_eq!( ( - request.params.max_tokens, - request.params.stream, + request.params.max_tokens.clone(), + request.params.stream.clone(), request .params .extra @@ -456,11 +470,39 @@ mod tests { .map(String::as_str) .collect::>(), ), - (Some(16_u64), Some(true), vec!["safeguards"]) + ( + Some(Recognized::Known(16_u64)), + Some(Recognized::Known(true)), + vec!["safeguards"] + ) ); assert_eq!(serde_json::to_value(request).unwrap(), body); } + #[rstest] + #[case::numeric_string(json!({"max_tokens": "1024", "temperature": "0.5", "top_p": "x", "top_k": "5"}))] + #[case::string_stream(json!({"stream": "true"}))] + #[case::integer_stream(json!({"stream": 0}))] + #[case::floats(json!({"max_tokens": 1024.0, "top_k": 1.5}))] + #[case::booleans(json!({"max_tokens": true, "temperature": false}))] + #[case::wide_integers(json!({"max_tokens": 18_446_744_073_709_551_615_u64, "top_k": 9_223_372_036_854_775_808_u64}))] + #[case::nested(json!({"temperature": [1], "top_p": {"p": 1}}))] + fn scalar_params_of_any_type_round_trip_unchanged(#[case] params: Value) { + let body = Value::Object( + [ + ("model".to_string(), json!("m")), + ( + "messages".to_string(), + json!([{"role": "user", "content": "hi"}]), + ), + ] + .into_iter() + .chain(params.as_object().unwrap().clone()) + .collect(), + ); + assert_eq!(round_trip::(&body), body); + } + #[test] fn text_constructor_serializes_as_a_text_block() { assert_eq!( diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index 46c14a21c59..e3ad783c791 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -320,7 +320,6 @@ impl AnthropicBeta { Self::AfkMode20260131 | Self::CePlugins20260901 | Self::CeUserManagement20260713 - | Self::Compact20260904 | Self::Dreaming20260421 | Self::ExperimentalClaudeTag20261021 | Self::ExtendedCacheTtl20250411 @@ -403,7 +402,8 @@ impl AnthropicBeta { | Self::Output128k20250219 | Self::TokenEfficientTools20250219 | Self::WebSearch20250305 => &[BetaProvider::BedrockConverse, BetaProvider::Bedrock], - Self::MidConversationOutputConfig20260701 + Self::Compact20260904 + | Self::MidConversationOutputConfig20260701 | Self::MidConversationToolChanges20260701 => &[ BetaProvider::BedrockConverse, BetaProvider::BedrockMantle, diff --git a/litellm-rust/crates/llms-types/src/recognized.rs b/litellm-rust/crates/llms-types/src/recognized.rs index 0545389afbc..2c389103aef 100644 --- a/litellm-rust/crates/llms-types/src/recognized.rs +++ b/litellm-rust/crates/llms-types/src/recognized.rs @@ -17,6 +17,25 @@ where as serde::Deserialize<'de>>::deserialize(deserializer).map(Some) } +/// `deserialize_with` for an optional float Python forwards as the caller sent it: an integer +/// stays `Unrecognized` with its exact digits, as Python sends, logs and prints it, and `null` +/// stays absent as before. +pub fn exact_float<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value = ::deserialize(deserializer)?; + if value.is_null() { + return Ok(None); + } + if matches!(&value, Value::Number(number) if !number.is_f64()) { + return Ok(Some(Recognized::Unrecognized(value))); + } + serde_json::from_value(value) + .map(Some) + .map_err(serde::de::Error::custom) +} + impl Recognized { pub fn known(&self) -> Option<&T> { match self { @@ -24,6 +43,60 @@ impl Recognized { Self::Unrecognized(_) => None, } } + + pub fn map_known(self, f: impl FnOnce(T) -> T) -> Self { + match self { + Self::Known(value) => Self::Known(f(value)), + unrecognized => unrecognized, + } + } +} + +impl Recognized { + pub fn is_truthy(&self) -> bool { + match self { + Self::Known(flag) => *flag, + Self::Unrecognized(other) => python_truthy(other), + } + } +} + +pub fn python_truthy(value: &Value) -> bool { + match value { + Value::Null => false, + Value::Bool(flag) => *flag, + Value::Number(number) => number.as_f64().is_none_or(|number| number != 0.0), + Value::String(text) => !text.is_empty(), + Value::Array(items) => !items.is_empty(), + Value::Object(fields) => !fields.is_empty(), + } +} + +pub fn map_known_runs( + items: Vec>, + mut f: impl FnMut(Vec) -> Vec, +) -> Vec> { + let mut mapped = Vec::with_capacity(items.len()); + let mut run = Vec::new(); + for item in items { + match item { + Recognized::Known(value) => run.push(value), + unrecognized => { + if !run.is_empty() { + mapped.extend( + f(std::mem::take(&mut run)) + .into_iter() + .map(Recognized::Known), + ); + } + mapped.push(unrecognized); + } + } + } + if !run.is_empty() { + mapped.extend(f(run).into_iter().map(Recognized::Known)); + } + mapped } #[cfg(test)] @@ -32,6 +105,82 @@ mod tests { use rstest::rstest; use serde_json::json; + #[rstest] + #[case::known_true(Recognized::Known(true), true)] + #[case::known_false(Recognized::Known(false), false)] + #[case::string_true(Recognized::Unrecognized(json!("true")), true)] + #[case::string_false(Recognized::Unrecognized(json!("false")), true)] + #[case::empty_string(Recognized::Unrecognized(json!("")), false)] + #[case::zero(Recognized::Unrecognized(json!(0)), false)] + #[case::negative_zero(Recognized::Unrecognized(json!(-0.0)), false)] + #[case::one(Recognized::Unrecognized(json!(1)), true)] + #[case::null(Recognized::Unrecognized(Value::Null), false)] + #[case::empty_list(Recognized::Unrecognized(json!([])), false)] + #[case::list(Recognized::Unrecognized(json!([0])), true)] + #[case::empty_object(Recognized::Unrecognized(json!({})), false)] + #[case::object(Recognized::Unrecognized(json!({"a": 0})), true)] + fn flags_are_read_by_python_truthiness(#[case] flag: Recognized, #[case] expected: bool) { + assert_eq!(flag.is_truthy(), expected); + } + + #[test] + fn runs_of_known_items_are_mapped_around_unrecognized_ones() { + let items = vec![ + Recognized::Known(1), + Recognized::Known(2), + Recognized::Unrecognized(json!("x")), + Recognized::Known(3), + ]; + let mut runs = Vec::new(); + let mapped = map_known_runs(items, |run: Vec| { + runs.push(run.clone()); + run.into_iter() + .filter(|item| *item != 2) + .map(|item| item * 10) + .collect() + }); + assert_eq!(runs, vec![vec![1, 2], vec![3]]); + assert_eq!( + mapped, + vec![ + Recognized::Known(10), + Recognized::Unrecognized(json!("x")), + Recognized::Known(30), + ] + ); + assert_eq!( + map_known_runs(Vec::>::new(), |run| run), + vec![] + ); + } + + #[derive(serde::Deserialize)] + struct Sampling { + #[serde(default, deserialize_with = "exact_float")] + temperature: Option>, + } + + #[rstest] + #[case::float(json!({"temperature": 0.5}), Some(Recognized::Known(0.5)))] + #[case::small_int(json!({"temperature": 2}), Some(Recognized::Unrecognized(json!(2))))] + #[case::zero(json!({"temperature": 0}), Some(Recognized::Unrecognized(json!(0))))] + #[case::integral_float(json!({"temperature": 1.0}), Some(Recognized::Known(1.0)))] + #[case::largest_exact_int(json!({"temperature": 9_007_199_254_740_992_i64}), Some(Recognized::Unrecognized(json!(9_007_199_254_740_992_i64))))] + #[case::inexact_int(json!({"temperature": 1_152_921_504_606_846_977_i64}), Some(Recognized::Unrecognized(json!(1_152_921_504_606_846_977_i64))))] + #[case::inexact_negative_int(json!({"temperature": -9_223_372_036_854_775_807_i64}), Some(Recognized::Unrecognized(json!(-9_223_372_036_854_775_807_i64))))] + #[case::inexact_unsigned_int(json!({"temperature": 18_446_744_073_709_551_615_u64}), Some(Recognized::Unrecognized(json!(18_446_744_073_709_551_615_u64))))] + #[case::string(json!({"temperature": "0.5"}), Some(Recognized::Unrecognized(json!("0.5"))))] + #[case::null(json!({"temperature": null}), None)] + #[case::absent(json!({}), None)] + fn integers_keep_their_digits(#[case] value: Value, #[case] expected: Option>) { + assert_eq!( + serde_json::from_value::(value) + .unwrap() + .temperature, + expected + ); + } + #[rstest] #[case::known(json!(7), Recognized::Known(7))] #[case::wrong_type(json!("7"), Recognized::Unrecognized(json!("7")))] diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index 79ee17e8b97..d59cd1be1d2 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -23,9 +23,11 @@ litellm-http.workspace = true litellm-secrets.workspace = true litellm-python-compat.workspace = true litellm-router-types.workspace = true +litellm-tracing.workspace = true base64.workspace = true bytes.workspace = true data-url = "0.3.2" +fancy-regex.workspace = true futures-util.workspace = true reqwest.workspace = true serde.workspace = true diff --git a/litellm-rust/crates/llms/src/anthropic/beta_headers.rs b/litellm-rust/crates/llms/src/anthropic/beta_headers.rs index 7842b478dce..4a3575c9bbc 100644 --- a/litellm-rust/crates/llms/src/anthropic/beta_headers.rs +++ b/litellm-rust/crates/llms/src/anthropic/beta_headers.rs @@ -1,5 +1,5 @@ use litellm_http::request::{with_header, without_headers}; -use litellm_llms_types::providers::anthropic::{BETA_HEADER, BetaProvider, BetaSet}; +use litellm_llms_types::providers::anthropic::{AnthropicBeta, BETA_HEADER, BetaProvider, BetaSet}; use crate::{anthropic::common_utils::existing_betas, base_llm::auth::Headers}; @@ -33,6 +33,16 @@ impl BetaPolicy { } } +pub fn accepted_or_unlisted(betas: BetaSet, provider: BetaProvider) -> BetaSet { + betas + .into_iter() + .filter_map(|beta| match beta { + AnthropicBeta::Other(_) => Some(beta), + known => known.on(provider), + }) + .collect() +} + #[cfg(test)] mod tests { use crate::anthropic::beta_headers::BetaPolicy; diff --git a/litellm-rust/crates/llms/src/anthropic/common_utils.rs b/litellm-rust/crates/llms/src/anthropic/common_utils.rs index 13653f438f7..cadc5711474 100644 --- a/litellm-rust/crates/llms/src/anthropic/common_utils.rs +++ b/litellm-rust/crates/llms/src/anthropic/common_utils.rs @@ -1,4 +1,7 @@ +use std::sync::LazyLock; + use crate::base_llm::messages::context::MessagesModelCapabilities; +use fancy_regex::Regex; use litellm_auth::CredentialPlacement; use litellm_secret_value::SecretValue; @@ -22,7 +25,7 @@ use serde_json::Value; use crate::{ anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX, - base_llm::auth::{AuthScheme, Headers}, + base_llm::auth::{AuthScheme, Headers, ValidatedEnvironment}, }; pub const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY"; @@ -80,10 +83,21 @@ impl<'a> OauthToken<'a> { } pub fn into_auth(self) -> AuthScheme { - AuthScheme::Credential { - placement: CredentialPlacement::Bearer, - secret: SecretValue::new(self.0), - } + oauth_bearer(SecretValue::new(self.0)) + } +} + +fn oauth_bearer(secret: SecretValue) -> AuthScheme { + AuthScheme::Credential { + placement: CredentialPlacement::Bearer, + secret, + } +} + +pub fn minted_oauth_environment(headers: Headers, token: SecretValue) -> ValidatedEnvironment { + ValidatedEnvironment { + headers: with_oauth_companions(headers, &[], BrowserAccess::Omitted), + auth: oauth_bearer(token), } } @@ -92,11 +106,23 @@ pub fn get_api_key( api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, ) -> Option { - resolve_non_empty(api_key, env_lookup, &[ANTHROPIC_API_KEY_ENV]) + nonempty_secret(api_key.map(str::to_owned)) + .or_else(|| nonempty_secret(env_lookup(ANTHROPIC_API_KEY_ENV))) } pub fn get_auth_token(env_lookup: &dyn Fn(&str) -> Option) -> Option { - resolve_non_empty(None, env_lookup, &[ANTHROPIC_AUTH_TOKEN_ENV]) + nonempty_secret(env_lookup(ANTHROPIC_AUTH_TOKEN_ENV)) +} + +/// Python's `normalize_nonempty_secret_str`: `str.strip()` also strips the ASCII separators. +fn nonempty_secret(value: Option) -> Option { + value + .map(|value| { + value + .trim_matches(|c: char| c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)) + .to_owned() + }) + .filter(|value| !value.is_empty()) } /// Python's `AnthropicModelInfo.get_auth_header`, naming the credential instead of building @@ -232,14 +258,24 @@ pub fn forwarded_oauth_bearer(headers: &[(String, String)]) -> Option Headers { - merge_beta_headers( +#[derive(Clone, Copy, PartialEq, Eq)] +enum BrowserAccess { + Granted, + Omitted, +} + +fn with_oauth_companions(headers: Headers, dropped: &[&str], browser: BrowserAccess) -> Headers { + let headers = merge_beta_headers( without_headers(headers, dropped), BetaSet::from_iter([AnthropicBeta::Oauth20250420]), - ) - .into_iter() - .chain([(DIRECT_BROWSER_ACCESS_HEADER.to_string(), "true".to_string())]) - .collect() + ); + match browser { + BrowserAccess::Granted => headers + .into_iter() + .chain([(DIRECT_BROWSER_ACCESS_HEADER.to_string(), "true".to_string())]) + .collect(), + BrowserAccess::Omitted => headers, + } } pub fn optionally_handle_anthropic_oauth(headers: Headers, api_key: Option<&str>) -> OauthHandling { @@ -247,13 +283,17 @@ pub fn optionally_handle_anthropic_oauth(headers: Headers, api_key: Option<&str> forwarded_oauth_bearer(&headers).map(|token| SecretValue::new(token.as_str())) { return OauthHandling::Bearer { - headers: with_oauth_companions(headers, &[API_KEY_HEADER, AUTHORIZATION]), + headers: with_oauth_companions( + headers, + &[API_KEY_HEADER, AUTHORIZATION], + BrowserAccess::Granted, + ), token, }; } if let Some(token) = api_key.and_then(OauthToken::parse) { return OauthHandling::Bearer { - headers: with_oauth_companions(headers, &[API_KEY_HEADER]), + headers: with_oauth_companions(headers, &[API_KEY_HEADER], BrowserAccess::Granted), token: SecretValue::new(token.as_str()), }; } @@ -278,12 +318,19 @@ pub fn has_advisor_tool(tools: Option<&[Recognized]>) -> bool { .any(|tool| matches!(tool, Recognized::Known(MessagesTool::Advisor { .. }))) } -pub fn requires_native_compaction_beta(compaction: Option<&Value>, messages: &[Message]) -> bool { +pub fn requires_native_compaction_beta( + compaction: Option<&Value>, + messages: &[Recognized], +) -> bool { compaction.is_some() - || messages.iter().flat_map(Message::blocks).any(|block| { - block.is_type(ContentBlockType::Compaction) - && block.signature.as_deref().is_some_and(|s| !s.is_empty()) - }) + || messages + .iter() + .filter_map(Recognized::known) + .flat_map(Message::blocks) + .any(|block| { + block.is_type(ContentBlockType::Compaction) + && block.signature.as_deref().is_some_and(|s| !s.is_empty()) + }) } fn is_blank(text: Option<&str>) -> bool { @@ -650,6 +697,113 @@ pub fn flatten_unencrypted_web_search_results(messages: Vec) -> Vec bool { + let lower = error_text.to_lowercase(); + if !lower.contains("thinking") { + return false; + } + (lower.contains("signature") && (lower.contains("invalid") || lower.contains("valid string"))) + || lower.contains("must contain thinking") +} + +pub fn is_anthropic_thinking_binding_mismatch_error(error_text: &str) -> bool { + error_text + .to_lowercase() + .contains("bound to a different conversation") +} + +pub fn anthropic_request_errors_on_thinking_binding_mismatch(request_body: &Value) -> bool { + request_body + .pointer("/thinking/block_binding/prefix_mismatch_behavior") + .and_then(Value::as_str) + == Some("error") +} + +fn is_thinking_value_block(block: &Value) -> bool { + matches!( + block.get("type").and_then(Value::as_str), + Some("thinking" | "redacted_thinking") + ) +} + +fn without_thinking_value_blocks(message: Value) -> Option { + let Value::Object(mut message) = message else { + return Some(message); + }; + if let Some(Value::Array(content)) = message.get_mut("content") { + content.retain(|block| !is_thinking_value_block(block)); + if content.is_empty() { + return None; + } + } + Some(Value::Object(message)) +} + +pub fn strip_thinking_blocks_from_anthropic_messages_request(request_body: Value) -> Value { + let Value::Object(mut body) = request_body else { + return request_body; + }; + if let Some(Value::Array(messages)) = body.get_mut("messages") { + *messages = std::mem::take(messages) + .into_iter() + .filter_map(without_thinking_value_blocks) + .collect(); + } + body.shift_remove("thinking"); + Value::Object(body) +} + +static CLAUDE_STRICT_TOOLS_REJECTION: LazyLock = LazyLock::new(|| { + Regex::new(concat!( + r"Too many strict tools \(\d+\)", + r"|Schemas contains too many (?:optional parameters|parameters with union types)", + r"|[Cc]ompiled grammar (?:is too large|size)", + r"|tools\.\d+\.custom(?:\.strict)?: ", + r"|Grammar compilation timed out", + )) + .expect("valid regex") +}); + +pub fn is_claude_strict_tools_rejection(status: u16, error_text: &str) -> bool { + status == 400 + && CLAUDE_STRICT_TOOLS_REJECTION + .is_match(error_text) + .unwrap_or(false) +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ToolsWithoutStrict { + pub tools: Vec, + pub dropped_count: usize, +} + +pub fn tools_without_strict(tools: Option<&Value>) -> Option { + let tools = tools?.as_array()?; + if !tools.iter().all(Value::is_object) { + return None; + } + let dropped_count = tools + .iter() + .filter(|tool| tool.get("strict") == Some(&Value::Bool(true))) + .count(); + if dropped_count == 0 { + return None; + } + Some(ToolsWithoutStrict { + tools: tools + .iter() + .cloned() + .map(|mut tool| { + if let Value::Object(fields) = &mut tool { + fields.shift_remove("strict"); + } + tool + }) + .collect(), + dropped_count, + }) +} + #[cfg(test)] mod tests { use crate::base_llm::messages::context::SupportedEffortTiers; @@ -668,7 +822,7 @@ mod tests { serde_json::from_value(value).unwrap() } - fn history(messages: Value) -> Vec { + fn history(messages: Value) -> Vec> { serde_json::from_value(messages).unwrap() } @@ -1636,6 +1790,11 @@ mod tests { &[("ANTHROPIC_AUTH_TOKEN", "env-token")], Some(("Authorization", "env-token")), )] + #[case::auth_token_stripped_of_whitespace( + None, + &[("ANTHROPIC_AUTH_TOKEN", " env-token \t\n\u{1f}")], + Some(("Authorization", "env-token")), + )] #[case::auth_token_when_the_env_key_is_blank( None, &[("ANTHROPIC_API_KEY", " \t"), ("ANTHROPIC_AUTH_TOKEN", "env-token")], @@ -1647,7 +1806,19 @@ mod tests { &[("ANTHROPIC_API_KEY", OAUTH_BEARER)], Some(("Authorization", OAUTH_TOKEN)), )] + #[case::env_key_stripped_of_whitespace( + None, + &[("ANTHROPIC_API_KEY", " sk-env \t\n\u{1f}")], + Some(("x-api-key", "sk-env")), + )] + #[case::api_key_param_stripped_of_separators(Some("\u{1c}sk-param "), &[], Some(("x-api-key", "sk-param")))] + #[case::separator_only_env_key_falls_to_the_auth_token( + None, + &[("ANTHROPIC_API_KEY", "\u{1c}\u{1f}"), ("ANTHROPIC_AUTH_TOKEN", "env-token")], + Some(("Authorization", "env-token")), + )] #[case::no_credentials(None, &[], None)] + #[case::separator_only_auth_token(None, &[("ANTHROPIC_AUTH_TOKEN", "\u{1c}\u{1f}")], None)] #[case::blank_everything(Some(""), &[("ANTHROPIC_API_KEY", " "), ("ANTHROPIC_AUTH_TOKEN", " \t")], None)] fn auth_header_prefers_the_key_then_the_auth_token( #[case] api_key: Option<&str>, @@ -1913,6 +2084,7 @@ mod tests { supports_reasoning: false, supports_adaptive_thinking: false, thinking_always_on: false, + thinking_on_by_default: false, supports_legacy_thinking: false, supports_output_config: false, supports_sampling_params: true, @@ -1973,4 +2145,166 @@ mod tests { fn the_messages_url_follows_python_ensure_api_route(#[case] api_base: &str, #[case] url: &str) { assert_eq!(complete_anthropic_url(Some(api_base), &|_| None), url); } + + fn shared_cases(table: &str) -> Vec<(String, String, bool)> { + let fixture: Value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/anthropic_messages_http_error_retry.json" + ))) + .unwrap(); + fixture[table] + .as_array() + .unwrap() + .iter() + .map(|case| { + ( + case["name"].as_str().unwrap().to_string(), + case["error_text"].as_str().unwrap().to_string(), + case["expected"].as_bool().unwrap(), + ) + }) + .collect() + } + + #[rstest] + #[case::invalid_thinking_block( + "invalid_thinking_block_error", + is_anthropic_invalid_thinking_block_error + )] + #[case::thinking_binding_mismatch( + "thinking_binding_mismatch_error", + is_anthropic_thinking_binding_mismatch_error + )] + fn error_texts_match_the_cases_shared_with_python( + #[case] table: &str, + #[case] recognizes: fn(&str) -> bool, + ) { + let cases = shared_cases(table); + assert!(!cases.is_empty()); + let mismatched: Vec<&String> = cases + .iter() + .filter(|(_, error_text, expected)| recognizes(error_text) != *expected) + .map(|(name, _, _)| name) + .collect(); + assert_eq!(mismatched, Vec::<&String>::new()); + } + + #[rstest] + #[case::pinned(json!({"thinking": {"block_binding": {"prefix_mismatch_behavior": "error"}}}), true)] + #[case::other_behavior(json!({"thinking": {"block_binding": {"prefix_mismatch_behavior": "drop"}}}), false)] + #[case::string_binding(json!({"thinking": {"block_binding": "strict"}}), false)] + #[case::no_thinking(json!({"messages": []}), false)] + #[case::not_an_object(json!(["thinking"]), false)] + fn a_pinned_binding_is_the_caller_asking_for_the_error( + #[case] body: Value, + #[case] expected: bool, + ) { + assert_eq!( + anthropic_request_errors_on_thinking_binding_mismatch(&body), + expected + ); + } + + #[rstest] + #[case::mixed_turn( + json!({"model": "m", "thinking": {"type": "enabled"}, "messages": [ + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "t", "signature": "s"}, + {"type": "text", "text": "a"}, + {"type": "redacted_thinking", "data": "d"} + ]} + ]}), + json!({"model": "m", "messages": [{"role": "assistant", "content": [{"type": "text", "text": "a"}]}]}) + )] + #[case::thinking_only_and_empty_turns_are_dropped( + json!({"messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [{"type": "redacted_thinking", "data": "d"}]}, + {"role": "assistant", "content": []}, + {"role": "user", "content": "again"} + ]}), + json!({"messages": [{"role": "user", "content": "hi"}, {"role": "user", "content": "again"}]}) + )] + #[case::non_object_entries_are_kept( + json!({"messages": ["raw", {"role": "user", "content": ["raw", {"type": "thinking"}]}]}), + json!({"messages": ["raw", {"role": "user", "content": ["raw"]}]}) + )] + #[case::messages_not_a_list( + json!({"messages": "hi", "thinking": {"type": "adaptive"}}), + json!({"messages": "hi"}) + )] + fn retrying_strips_thinking_from_the_replay_and_the_request( + #[case] body: Value, + #[case] expected: Value, + ) { + assert_eq!( + strip_thinking_blocks_from_anthropic_messages_request(body), + expected + ); + } + + #[test] + fn stripping_keeps_the_order_of_the_remaining_fields() { + let stripped = strip_thinking_blocks_from_anthropic_messages_request( + json!({"model": "m", "thinking": {}, "messages": [], "max_tokens": 1}), + ); + let keys: Vec<&str> = stripped + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + assert_eq!(keys, ["model", "messages", "max_tokens"]); + } + + fn shared_table(table: &str) -> Vec { + let fixture: Value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/anthropic_messages_http_error_retry.json" + ))) + .unwrap(); + fixture[table].as_array().unwrap().clone() + } + + #[test] + fn strict_tools_rejections_match_the_cases_shared_with_python() { + let cases = shared_table("claude_strict_tools_rejection"); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let status = u16::try_from(case["status"].as_u64().unwrap()).unwrap(); + is_claude_strict_tools_rejection(status, case["error_text"].as_str().unwrap()) + != case["expected"].as_bool().unwrap() + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); + } + + #[test] + fn tools_without_strict_matches_the_cases_shared_with_python() { + let cases = shared_table("tools_without_strict"); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let expected = case["expected"] + .as_object() + .map(|expected| ToolsWithoutStrict { + tools: expected["tools"].as_array().unwrap().clone(), + dropped_count: usize::try_from(expected["dropped_count"].as_u64().unwrap()) + .unwrap(), + }); + tools_without_strict(Some(&case["tools"])) != expected + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); + } + + #[test] + fn absent_tools_have_no_strict_to_drop() { + assert_eq!(tools_without_strict(None), None); + } } diff --git a/litellm-rust/crates/llms/src/anthropic/messages/handler.rs b/litellm-rust/crates/llms/src/anthropic/messages/handler.rs index 54f924ddec6..46a16e7d546 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/handler.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/handler.rs @@ -3,7 +3,7 @@ use litellm_llms_types::{ AdaptiveThinking, EnabledThinking, Message, MessagesOptionalParams, MessagesRequest, ThinkingConfig, ThinkingDisplay, }, - recognized::Recognized, + recognized::{Recognized, map_known_runs}, }; use serde_json::{Value, json}; @@ -13,14 +13,19 @@ use crate::{ flatten_unencrypted_web_search_results, sanitize_tool_use_ids, strip_empty_content_blocks, strip_provider_specific_fields, }, + base_llm::messages::context::MessagesShapeOptions, }; pub fn shape_anthropic_messages_request( request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { Ok(MessagesRequest { - messages: sanitize_anthropic_messages(request.messages), + messages: if options.presanitized { + map_known_runs(request.messages, strip_provider_specific_fields) + } else { + map_known_runs(request.messages, sanitize_anthropic_messages) + }, params: MessagesOptionalParams { metadata: request .params @@ -28,7 +33,10 @@ pub fn shape_anthropic_messages_request( .as_ref() .map(validate_anthropic_api_metadata) .transpose()?, - thinking: with_reasoning_auto_summary(request.params.thinking, reasoning_auto_summary), + thinking: with_reasoning_auto_summary( + request.params.thinking, + options.reasoning_auto_summary, + ), ..request.params }, ..request @@ -299,7 +307,10 @@ mod tests { "thinking": {"type": "enabled", "budget_tokens": 1024}, "safeguards": [{"type": "dangerous_tool_use"}] })), - true, + MessagesShapeOptions { + reasoning_auto_summary: true, + presanitized: false, + }, ) .unwrap(); assert_eq!( @@ -315,4 +326,27 @@ mod tests { }) ); } + + #[test] + fn presanitized_shaping_keeps_blocks_and_strips_only_provider_fields() { + let messages = json!([{"role": "user", "content": [ + {"type": "text", "text": " ", "cache_control": {"type": "ephemeral"}}, + {"type": "tool_result", "tool_use_id": "functions.Bash:0", "content": "ok", "provider_specific_fields": {"x": 1}} + ]}]); + let sanitized = shape_anthropic_messages_request( + request(json!({"model": "m", "messages": messages})), + MessagesShapeOptions { + reasoning_auto_summary: false, + presanitized: true, + }, + ) + .unwrap(); + assert_eq!( + serde_json::to_value(sanitized).unwrap(), + json!({"model": "m", "messages": [{"role": "user", "content": [ + {"type": "text", "text": " ", "cache_control": {"type": "ephemeral"}}, + {"type": "tool_result", "tool_use_id": "functions.Bash:0", "content": "ok"} + ]}]}) + ); + } } diff --git a/litellm-rust/crates/llms/src/anthropic/messages/mod.rs b/litellm-rust/crates/llms/src/anthropic/messages/mod.rs index 597a1d819d9..4fa24a3f0f5 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/mod.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/mod.rs @@ -1,3 +1,4 @@ pub mod handler; +mod scalar; pub mod thinking; pub mod transformation; diff --git a/litellm-rust/crates/llms/src/anthropic/messages/scalar.rs b/litellm-rust/crates/llms/src/anthropic/messages/scalar.rs new file mode 100644 index 00000000000..db9aa01d177 --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/messages/scalar.rs @@ -0,0 +1,113 @@ +use litellm_host::raw_json; +use litellm_llms_types::recognized::Recognized; +use litellm_python_compat::{ + Value as PythonValue, + json::{from_json, loads}, + repr::to_str, +}; +use serde_json::Value; + +pub(crate) fn python_value(value: &Value) -> PythonValue { + match value { + Value::String(text) => raw_json::decode(text) + .and_then(|raw| loads(raw).ok()) + .unwrap_or_else(|| PythonValue::Str(text.clone())), + Value::Array(items) => PythonValue::List(items.iter().map(python_value).collect()), + Value::Object(entries) => PythonValue::Dict( + entries + .iter() + .map(|(key, item)| (PythonValue::Str(key.clone()), python_value(item))) + .collect(), + ), + Value::Null | Value::Bool(_) | Value::Number(_) => from_json(value.clone()), + } +} + +pub(crate) fn python_number(value: &Value) -> Option { + match value { + Value::Bool(flag) => Some(f64::from(u8::from(*flag))), + Value::Number(number) => number.as_f64(), + Value::String(text) => match raw_json::decode(text)? { + "NaN" => Some(f64::NAN), + "Infinity" => Some(f64::INFINITY), + "-Infinity" => Some(f64::NEG_INFINITY), + raw => raw.parse().ok(), + }, + _ => None, + } +} + +pub(crate) fn is_one(value: &Recognized) -> bool { + match value { + Recognized::Known(number) => *number == 1.0, + Recognized::Unrecognized(other) => python_number(other) == Some(1.0), + } +} + +pub(crate) fn python_str(value: &Recognized) -> String { + match value { + Recognized::Known(known) => serde_json::to_value(known) + .map(|known| to_str(&from_json(known))) + .unwrap_or_default(), + Recognized::Unrecognized(other) => unrecognized_str(other), + } +} + +fn unrecognized_str(value: &Value) -> String { + match value { + Value::Null => "None".into(), + Value::Bool(true) => "True".into(), + Value::Bool(false) => "False".into(), + Value::String(text) => match raw_json::decode(text) { + Some("NaN") => "nan".into(), + Some("Infinity") => "inf".into(), + Some("-Infinity") => "-inf".into(), + Some(raw) => raw.trim_matches('"').into(), + None => text.clone(), + }, + other => other.to_string(), + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::int(json!(3), Some(3.0))] + #[case::float(json!(1.5), Some(1.5))] + #[case::bool(json!(true), Some(1.0))] + #[case::wide(json!(raw_json::encode("18446744073709551616")), Some(18_446_744_073_709_551_616.0))] + #[case::negative_infinity(json!(raw_json::encode("-Infinity")), Some(f64::NEG_INFINITY))] + #[case::string(json!("1"), None)] + #[case::marked_string(json!(raw_json::encode("\"1\"")), None)] + #[case::null(Value::Null, None)] + fn numbers_compare_as_python_compares_them( + #[case] value: Value, + #[case] expected: Option, + ) { + assert_eq!(python_number(&value), expected); + } + + #[test] + fn nan_is_a_number_unequal_to_everything() { + let nan = json!(raw_json::encode("NaN")); + assert!(python_number(&nan).is_some_and(f64::is_nan)); + assert!(!is_one(&Recognized::Unrecognized(nan))); + } + + #[rstest] + #[case::known(Recognized::Known(0.5), "0.5")] + #[case::known_wide(Recognized::Known(1e16), "1e+16")] + #[case::int(Recognized::Unrecognized(json!(0)), "0")] + #[case::bool(Recognized::Unrecognized(json!(false)), "False")] + #[case::string(Recognized::Unrecognized(json!("0.5")), "0.5")] + #[case::nan(Recognized::Unrecognized(json!(raw_json::encode("NaN"))), "nan")] + #[case::wide(Recognized::Unrecognized(json!(raw_json::encode("-1180591620717411303424"))), "-1180591620717411303424")] + fn values_print_as_python_prints_them(#[case] value: Recognized, #[case] expected: &str) { + assert_eq!(python_str(&value), expected); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs b/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs index ae3488a21f9..c6f7c1679ef 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs @@ -1,17 +1,20 @@ +use litellm_host::raw_json; use litellm_llms_types::{ formats::{ chat_completions::ReasoningEffort, messages::{ - AdaptiveThinking, EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig, - ThinkingConfig, ThinkingDisplay, + AdaptiveThinking, DisabledThinking, EffortLevel, EnabledThinking, + MessagesOptionalParams, MessagesRequest, OutputConfig, ThinkingConfig, ThinkingDisplay, }, }, recognized::Recognized, }; -use litellm_python_compat::{json::from_json, repr::repr, truthy::truthy}; +use litellm_python_compat::{ + Value as PythonValue, json::from_json, repr::repr, reprlib::reprlib, truthy::truthy, +}; use serde_json::Value; -use strum::VariantArray; +use super::scalar::{is_one, python_number, python_str, python_value}; use crate::base_llm::messages::context::{ MessagesModelCapabilities, ThinkingBudgets, ThinkingContext, }; @@ -56,13 +59,61 @@ fn unmapped_effort(effort: &Value) -> Error { Error::InvalidRequest(crate::ErrorDetail::InvalidChoice { field: "reasoning effort", actual: repr(&from_json(effort.clone())), - choices: ReasoningEffort::VARIANTS - .iter() - .map(|effort| <&'static str>::from(*effort)) - .collect(), + choices: vec!["minimal", "low", "medium", "high", "xhigh", "max", "none"], }) } +fn invalid_adaptive_effort(effort: &Value) -> Error { + Error::InvalidRequest(crate::ErrorDetail::InvalidReasoningEffort { + actual: repr(&from_json(effort.clone())), + }) +} + +const NON_STRING_EFFORT_REASON: &str = "it is not a string. Send reasoning_effort as a level name such as 'high' to set a reasoning level."; +const NON_NUMERIC_BUDGET_REASON: &str = + "it is not a number. Send budget_tokens as an integer to set a reasoning level."; + +fn warn_dropped_reasoning( + param: &str, + value: &PythonValue, + model: &str, + reason: &str, + drop_params_gated: bool, +) { + let gate = if drop_params_gated { + " (drop_params=True)" + } else { + "" + }; + litellm_tracing::warn!( + "Dropping {param}={} for model={}{gate} because {reason}", + reprlib(value), + repr(&PythonValue::Str(model.to_owned())) + ); +} + +fn warn_non_string_reasoning_effort(effort: &Value, model: &str) { + let value = python_value(effort); + let requested = match &value { + PythonValue::Str(_) => return, + PythonValue::Dict(entries) => entries + .iter() + .find(|(key, _)| *key == PythonValue::Str("effort".to_owned())) + .map(|(_, level)| level), + other => Some(other), + }; + if matches!(requested, None | Some(PythonValue::None)) { + return; + } + warn_dropped_reasoning( + "reasoning_effort", + &value, + model, + NON_STRING_EFFORT_REASON, + false, + ); +} + fn unsupported_effort(level: EffortLevel, model: &str) -> Error { Error::InvalidRequest(crate::ErrorDetail::UnsupportedValue { field: "effort", @@ -82,11 +133,56 @@ fn output_effort(effort: ReasoningEffort) -> Option { } } -fn fit_budget_to_max_tokens(budget_tokens: u64, max_tokens: Option) -> Option { - let Some(max_tokens) = max_tokens else { - return Some(budget_tokens); +/// `cap_thinking_budget_to_max_tokens`, whose first comparison (`max_tokens <= 1024`) raises +/// `TypeError` for a `max_tokens` Python cannot order against an int, such as a string. +fn fit_budget_to_max_tokens( + budget_tokens: u64, + max_tokens: Option<&Recognized>, +) -> Result>, Error> { + let max_tokens = match max_tokens { + None => return Ok(Some(Recognized::Known(budget_tokens))), + Some(Recognized::Known(max_tokens)) => *max_tokens, + Some(Recognized::Unrecognized(other)) => { + return fit_budget_to_python_number(budget_tokens, other); + } }; - (max_tokens > ANTHROPIC_MIN_THINKING_BUDGET_TOKENS).then(|| budget_tokens.min(max_tokens - 1)) + Ok((max_tokens > ANTHROPIC_MIN_THINKING_BUDGET_TOKENS) + .then(|| Recognized::Known(budget_tokens.min(max_tokens - 1)))) +} + +fn enabled_thinking(budget_tokens: Recognized) -> ThinkingConfig { + ThinkingConfig::Enabled(EnabledThinking { + budget_tokens: Some(budget_tokens), + ..EnabledThinking::default() + }) +} + +fn fit_budget_to_python_number( + budget_tokens: u64, + max_tokens: &Value, +) -> Result>, Error> { + let Some(max_tokens) = python_number(max_tokens) else { + if max_tokens.is_null() { + return Ok(Some(Recognized::Known(budget_tokens))); + } + return Err(Error::InvalidRequest( + crate::ErrorDetail::UnorderableTypes { + op: "<=", + left: python_type_name(max_tokens), + right: "int", + }, + )); + }; + if max_tokens <= ANTHROPIC_MIN_THINKING_BUDGET_TOKENS as f64 { + return Ok(None); + } + if (budget_tokens as f64) < max_tokens { + return Ok(Some(Recognized::Known(budget_tokens))); + } + Ok(Some(Recognized::Unrecognized( + serde_json::Number::from_f64(max_tokens - 1.0) + .map_or_else(|| Value::String(raw_json::encode("NaN")), Value::Number), + ))) } fn known_thinking(request: &MessagesRequest) -> Option<&ThinkingConfig> { @@ -161,10 +257,17 @@ fn translate_reasoning_effort( }; let effort = match reasoning_effort { Recognized::Known(effort) => effort, - Recognized::Unrecognized(value @ Value::String(_)) => { - return Err(unmapped_effort(&value)); + Recognized::Unrecognized(value) if matches!(python_value(&value), PythonValue::Str(_)) => { + return Err(if context.capabilities.supports_adaptive_thinking { + invalid_adaptive_effort(&value) + } else { + unmapped_effort(&value) + }); + } + Recognized::Unrecognized(value) => { + warn_non_string_reasoning_effort(&value, &request.model); + return Ok(request); } - Recognized::Unrecognized(_) => return Ok(request), }; let (Some(level), Some(budget)) = ( output_effort(effort), @@ -172,7 +275,11 @@ fn translate_reasoning_effort( ) else { return Ok(MessagesRequest { params: MessagesOptionalParams { - thinking: None, + thinking: (context.capabilities.thinking_on_by_default + && !context.capabilities.thinking_always_on) + .then(|| { + Recognized::Known(ThinkingConfig::Disabled(DisabledThinking::default())) + }), output_config: None, ..request.params }, @@ -199,10 +306,20 @@ fn translate_reasoning_effort( ..request }); } - let Some(budget) = fit_budget_to_max_tokens(budget, request.params.max_tokens) else { + let Some(budget) = fit_budget_to_max_tokens(budget, request.params.max_tokens.as_ref())? else { + litellm_tracing::warn!( + "Dropping `thinking` mapped from reasoning_effort={} for model={}: max_tokens={} is too small to fit the minimum thinking budget.", + <&'static str>::from(effort), + request.model, + request + .params + .max_tokens + .as_ref() + .map_or_else(|| "None".to_owned(), python_str) + ); return Ok(request); }; - let enabled = ThinkingConfig::enabled(budget); + let enabled = enabled_thinking(budget); Ok(MessagesRequest { params: MessagesOptionalParams { thinking: Some( @@ -336,6 +453,15 @@ fn translate_legacy_thinking_for_adaptive_model( crate::ErrorDetail::NonNumericThinkingBudget { actual }, )); } + if let (None, Some(Recognized::Unrecognized(dropped))) = (budget, &enabled.budget_tokens) { + warn_dropped_reasoning( + "thinking.budget_tokens", + &python_value(dropped), + &request.model, + NON_NUMERIC_BUDGET_REASON, + true, + ); + } let thinking = ThinkingConfig::Adaptive(AdaptiveThinking { display: kept_display(enabled.display.as_ref(), context), extra: carried_thinking_keys(&enabled.extra, context), @@ -392,11 +518,20 @@ fn translate_adaptive_effort_for_non_adaptive_model( } else { None }; + let thinking = match budget { + Some(budget) => fit_budget_to_max_tokens(budget, request.params.max_tokens.as_ref())? + .map(|budget| Recognized::Known(enabled_thinking(budget))), + None => None, + }; + if thinking.is_none() { + litellm_tracing::warn!( + "Dropping adaptive `thinking`/`output_config.effort` for model={}: the model does not support extended thinking, or max_tokens is too small to fit the minimum thinking budget.", + request.model + ); + } Ok(MessagesRequest { params: MessagesOptionalParams { - thinking: budget - .and_then(|budget| fit_budget_to_max_tokens(budget, request.params.max_tokens)) - .map(|budget| Recognized::Known(ThinkingConfig::enabled(budget))), + thinking, output_config: without_effort(request.params.output_config), ..request.params }, @@ -414,7 +549,8 @@ fn drop_incompatible_temperature_for_thinking( let pinned = request .params .temperature - .is_some_and(|temperature| temperature != 1.0); + .as_ref() + .is_some_and(|temperature| !is_one(temperature)); let thinking_enabled = matches!(known_thinking(&request), Some(ThinkingConfig::Enabled(_))); let effort_enabled = known_effort(&request).is_some(); if !pinned || !(thinking_enabled || effort_enabled) { @@ -447,7 +583,7 @@ mod tests { use super::*; use crate::base_llm::messages::context::SupportedEffortTiers; - const EFFORT_CHOICES: &str = "'none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max'"; + const EFFORT_CHOICES: &str = "'minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'none'"; fn request(fields: Value) -> MessagesRequest { let mut body = serde_json::json!({"model": "claude", "messages": [{"role": "user", "content": "Hello"}]}); @@ -748,6 +884,33 @@ mod tests { ); } + #[rstest] + #[case::default_thinking(true, false, Some(serde_json::json!({"type": "disabled"}))) ] + #[case::legacy(false, false, None)] + #[case::always_on(true, true, None)] + fn none_disables_only_optional_default_thinking( + #[case] thinking_on_by_default: bool, + #[case] thinking_always_on: bool, + #[case] thinking: Option, + ) { + let capabilities = MessagesModelCapabilities { + thinking_on_by_default, + thinking_always_on, + ..opus_4_7() + }; + let expected = match thinking { + Some(thinking) => serde_json::json!({"max_tokens": 1024, "thinking": thinking}), + None => serde_json::json!({"max_tokens": 1024}), + }; + assert_eq!( + translate( + capabilities, + serde_json::json!({"max_tokens": 1024, "reasoning_effort": "none"}) + ), + Ok(request(expected)) + ); + } + #[rstest] #[case::adaptive_model(opus_4_7())] #[case::effort_capable_model(opus_4_5())] @@ -788,17 +951,17 @@ mod tests { #[case::invalid_on_adaptive_model( opus_4_7(), serde_json::json!({"max_tokens": 1024, "reasoning_effort": "invalid"}), - format!("Unmapped reasoning effort: 'invalid'. Must be one of: {EFFORT_CHOICES}.") + format!("Invalid reasoning_effort: 'invalid'. Must be one of: {EFFORT_CHOICES}") )] #[case::disabled_on_adaptive_model( opus_4_7(), serde_json::json!({"max_tokens": 1024, "reasoning_effort": "disabled"}), - format!("Unmapped reasoning effort: 'disabled'. Must be one of: {EFFORT_CHOICES}.") + format!("Invalid reasoning_effort: 'disabled'. Must be one of: {EFFORT_CHOICES}") )] #[case::empty_on_adaptive_model( opus_4_7(), serde_json::json!({"max_tokens": 1024, "reasoning_effort": ""}), - format!("Unmapped reasoning effort: ''. Must be one of: {EFFORT_CHOICES}.") + format!("Invalid reasoning_effort: ''. Must be one of: {EFFORT_CHOICES}") )] #[case::xhigh_without_xhigh_tier_on_4_6( sonnet_4_6(), @@ -1403,6 +1566,34 @@ mod tests { ); } + #[rstest] + #[case::reasoning_effort(json!({"max_tokens": "8192", "reasoning_effort": "high"}), "str")] + #[case::caller_thinking( + json!({"max_tokens": "8192", "reasoning_effort": "low", "thinking": {"type": "enabled", "budget_tokens": 2048}}), + "str" + )] + #[case::adaptive_interface( + json!({"max_tokens": "8192", "thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}}), + "str" + )] + #[case::list(json!({"max_tokens": [8192], "reasoning_effort": "high"}), "list")] + fn an_unorderable_max_tokens_raises_python_type_error_while_capping_the_budget( + haiku_4_5: MessagesModelCapabilities, + #[case] fields: Value, + #[case] left: &'static str, + ) { + assert_eq!( + translate(haiku_4_5, fields), + Err(Error::InvalidRequest( + crate::ErrorDetail::UnorderableTypes { + op: "<=", + left, + right: "int", + } + )) + ); + } + #[rstest] #[case::summarized("summarized", false, Some("summarized"))] #[case::updates_kept("updates", true, Some("updates"))] diff --git a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs index 18789e5b2be..bbb9b4e4c17 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs @@ -1,29 +1,45 @@ use litellm_auth::CredentialPlacement; +use litellm_host::interceptors::{RetriedRequest, RetryReason}; +use litellm_http::request::{with_header, without_headers}; use litellm_llms_types::{ formats::messages::{ - ContentBlock, ContentBlockType, ContextEdit, ContextManagement, Message, + ContentBlock, ContentBlockType, ContextEdit, ContextManagement, Message, MessageContent, MessagesOptionalParams, MessagesRequest, Speed, ThinkingConfig, ThinkingDisplay, }, - providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS}, - recognized::Recognized, + providers::anthropic::{AnthropicBeta, BETA_HEADER, BetaProvider, BetaSet, DEFAULT_HEADERS}, + recognized::{Recognized, map_known_runs}, }; use litellm_router_types::LitellmParams; use serde_json::{Map, Value, json}; -use super::{handler::shape_anthropic_messages_request, thinking::translate_thinking}; -use crate::base_llm::messages::context::MessagesTransformContext; +use super::{ + handler::shape_anthropic_messages_request, + scalar::{is_one, python_str, python_value}, + thinking::translate_thinking, +}; +use crate::anthropic::beta_headers::accepted_or_unlisted; +use crate::base_llm::messages::{ + context::{MessagesShapeOptions, MessagesTransformContext}, + tools::ToolCustomField, +}; use crate::{ Error, anthropic::common_utils::{ ANTHROPIC_API_BASE_ENV, ANTHROPIC_API_KEY_ENV, ANTHROPIC_AUTH_TOKEN_ENV, - ANTHROPIC_BASE_URL_ENV, OauthHandling, complete_anthropic_url, get_auth_header, - has_advisor_tool, has_anthropic_credential, is_tool_search_used, merge_beta_headers, - optionally_handle_anthropic_oauth, requires_native_compaction_beta, strip_advisor_blocks, - strip_encrypted_reasoning_blocks, + ANTHROPIC_BASE_URL_ENV, OauthHandling, + anthropic_request_errors_on_thinking_binding_mismatch, complete_anthropic_url, + existing_betas, get_auth_header, has_advisor_tool, has_anthropic_credential, + is_anthropic_invalid_thinking_block_error, is_anthropic_thinking_binding_mismatch_error, + is_claude_strict_tools_rejection, is_tool_search_used, merge_beta_headers, + minted_oauth_environment, optionally_handle_anthropic_oauth, + requires_native_compaction_beta, strip_advisor_blocks, strip_encrypted_reasoning_blocks, + strip_thinking_blocks_from_anthropic_messages_request, tools_without_strict, }, base_llm::{ auth::AuthScheme, - messages::transformation::{BaseMessagesConfig, Headers, ValidatedEnvironment}, + messages::transformation::{ + BaseMessagesConfig, FederatedEnvironment, Headers, ValidatedEnvironment, + }, }, }; @@ -35,9 +51,9 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn get_complete_url( @@ -59,6 +75,12 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { transform_messages_request(request, &context.keeping_display_updates()) } + fn tool_custom_field(&self, model: &str) -> ToolCustomField { + ToolCustomField::Rewrite { + hoist_defer_loading: is_claude_model(model), + } + } + fn secret_names(&self) -> &'static [&'static str] { &[ ANTHROPIC_API_KEY_ENV, @@ -79,31 +101,22 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { _litellm_params: &LitellmParams, env_lookup: &dyn Fn(&str) -> Option, ) -> Result { - let headers = match optionally_handle_anthropic_oauth(headers, api_key) { - OauthHandling::Bearer { headers, token } => { - return Ok(ValidatedEnvironment { - headers, - auth: AuthScheme::Credential { - placement: CredentialPlacement::Bearer, - secret: token, - }, - }); - } - OauthHandling::Untouched(headers) => headers, - }; - if has_anthropic_credential(&headers) { - return Ok(ValidatedEnvironment { - headers, - auth: AuthScheme::Forwarded, - }); + match anthropic_environment(headers, api_key, env_lookup) { + FederatedEnvironment::Validated(validated) => Ok(validated), + FederatedEnvironment::Unauthenticated { missing, .. } => Err(missing), } - let auth = get_auth_header(api_key, env_lookup).ok_or(Error::Auth( - litellm_auth::Error::MissingApiKey { - provider: "Anthropic", - environment_variable: ANTHROPIC_API_KEY_ENV, - }, - ))?; - Ok(ValidatedEnvironment { headers, auth }) + } + + fn validate_federated_environment( + &self, + headers: Headers, + api_key: Option<&str>, + _model: &str, + _optional_params: &MessagesOptionalParams, + _litellm_params: &LitellmParams, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + Ok(anthropic_environment(headers, api_key, env_lookup)) } fn default_headers(&self) -> &'static [(&'static str, &'static str)] { @@ -116,6 +129,157 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { provider_feature_betas(request, BetaProvider::Anthropic), ) } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + + fn headers_for_url(&self, headers: Headers, url: &str) -> Headers { + if !is_foundry_url(url) { + return headers; + } + let betas = existing_betas(&headers); + if betas.is_empty() { + return headers; + } + let kept = accepted_or_unlisted(betas, BetaProvider::AzureAi); + if kept.is_empty() { + return without_headers(headers, &[BETA_HEADER]); + } + with_header(headers, BETA_HEADER, kept.to_string()) + } +} + +const FOUNDRY_HOST_SUFFIXES: [&str; 3] = [ + "services.ai.azure.com", + "openai.azure.com", + "cognitiveservices.azure.com", +]; + +fn is_foundry_url(url: &str) -> bool { + let absolute = if url.contains("://") { + url.to_string() + } else { + format!("https://{url}") + }; + url::Url::parse(&absolute) + .ok() + .and_then(|parsed| parsed.host_str().map(str::to_ascii_lowercase)) + .is_some_and(|host| { + FOUNDRY_HOST_SUFFIXES + .iter() + .any(|suffix| host == *suffix || host.ends_with(&format!(".{suffix}"))) + }) +} + +pub(crate) fn is_claude_model(model: &str) -> bool { + model.to_lowercase().contains("claude") +} + +pub fn should_retry_anthropic_messages_on_http_error( + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, +) -> bool { + if status != 400 { + return false; + } + let drops_rejected_strict_tools = context.drops_unsupported_params + && is_claude_strict_tools_rejection(status, error_text) + && tools_without_strict(request_body.get("tools")).is_some(); + if drops_rejected_strict_tools { + return true; + } + let caller_asked_for_this_error = + anthropic_request_errors_on_thinking_binding_mismatch(request_body) + && is_anthropic_thinking_binding_mismatch_error(error_text); + !caller_asked_for_this_error && is_anthropic_invalid_thinking_block_error(error_text) +} + +pub fn transform_anthropic_messages_request_on_http_error( + status: u16, + error_text: &str, + request_body: Value, +) -> RetriedRequest { + let mut reasons = Vec::new(); + let mut body = if status == 400 && is_anthropic_invalid_thinking_block_error(error_text) { + reasons.push(RetryReason::ThinkingSignature); + strip_thinking_blocks_from_anthropic_messages_request(request_body) + } else { + request_body + }; + let without_strict = if is_claude_strict_tools_rejection(status, error_text) { + tools_without_strict(body.get("tools")) + } else { + None + }; + if let (Some(without_strict), Value::Object(fields)) = (without_strict, &mut body) { + reasons.push(RetryReason::StrictTools { + dropped: without_strict.dropped_count, + }); + fields.insert("tools".to_owned(), Value::Array(without_strict.tools)); + } + RetriedRequest { reasons, body } +} + +pub fn retry_anthropic_messages_after_http_error( + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, +) -> Option { + should_retry_anthropic_messages_on_http_error(status, error_text, request_body, context).then( + || { + transform_anthropic_messages_request_on_http_error( + status, + error_text, + request_body.clone(), + ) + }, + ) +} + +fn anthropic_environment( + headers: Headers, + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> FederatedEnvironment { + let headers = match optionally_handle_anthropic_oauth(headers, api_key) { + OauthHandling::Bearer { headers, token } => { + return FederatedEnvironment::Validated(ValidatedEnvironment { + headers, + auth: AuthScheme::Credential { + placement: CredentialPlacement::Bearer, + secret: token, + }, + }); + } + OauthHandling::Untouched(headers) => headers, + }; + if has_anthropic_credential(&headers) { + return FederatedEnvironment::Validated(ValidatedEnvironment { + headers, + auth: AuthScheme::Forwarded, + }); + } + match get_auth_header(api_key, env_lookup) { + Some(auth) => FederatedEnvironment::Validated(ValidatedEnvironment { headers, auth }), + None => FederatedEnvironment::Unauthenticated { + headers, + missing: Error::Auth(litellm_auth::Error::MissingApiKey { + provider: "Anthropic", + environment_variable: ANTHROPIC_API_KEY_ENV, + }), + federate: minted_oauth_environment, + }, + } } pub(crate) fn transform_messages_request( @@ -125,6 +289,7 @@ pub(crate) fn transform_messages_request( if request.params.max_tokens.is_none() { return Err(Error::MissingField("max_tokens")); } + let request = without_unstructured_output_format(request, context); let request = drop_unsupported_params(request, context)?; let request = translate_thinking(request, &context.thinking)?; let context_management = request @@ -132,13 +297,20 @@ pub(crate) fn transform_messages_request( .context_management .clone() .map(map_openai_context_management_to_anthropic); - let messages = if has_advisor_tool(request.params.tools.as_deref()) { + let messages = if has_advisor_tool(request.params.tools.as_deref()) + || continues_message_thread(&request.params) + { request.messages } else { - strip_advisor_blocks(request.messages) + map_known_runs(request.messages, strip_advisor_blocks) + }; + let messages = if context.drops_unsupported_params { + drop_non_string_tool_result_texts(messages) + } else { + messages }; Ok(MessagesRequest { - messages: strip_encrypted_reasoning_blocks(messages), + messages: map_known_runs(messages, strip_encrypted_reasoning_blocks), params: MessagesOptionalParams { context_management, extra: without_null_safeguards(request.params.extra), @@ -148,6 +320,84 @@ pub(crate) fn transform_messages_request( }) } +fn without_unstructured_output_format( + request: MessagesRequest, + context: &MessagesTransformContext, +) -> MessagesRequest { + if !context.drops_unsupported_params + || request + .params + .output_format + .as_ref() + .is_none_or(Value::is_object) + { + return request; + } + MessagesRequest { + params: MessagesOptionalParams { + output_format: None, + ..request.params + }, + ..request + } +} + +fn is_python_str(value: &Value) -> bool { + match value { + Value::String(text) => { + litellm_host::raw_json::decode(text).is_none_or(|raw| raw.starts_with('"')) + } + _ => false, + } +} + +fn is_non_string_text(part: &Value) -> bool { + part.get("type").and_then(Value::as_str) == Some("text") + && part.get("text").is_some_and(|text| !is_python_str(text)) +} + +fn without_non_string_texts(content: &mut Value) { + if let Value::Array(parts) = content { + parts.retain(|part| !is_non_string_text(part)); + } +} + +fn drop_non_string_tool_result_texts( + messages: Vec>, +) -> Vec> { + messages + .into_iter() + .map(|message| match message { + Recognized::Known(mut message) => { + if message.role == "user" + && let MessageContent::Blocks(blocks) = &mut message.content + { + blocks + .iter_mut() + .filter(|block| block.is_type(ContentBlockType::ToolResult)) + .filter_map(|block| block.content.as_mut()) + .for_each(without_non_string_texts); + } + Recognized::Known(message) + } + Recognized::Unrecognized(mut message) => { + if message.get("role").and_then(Value::as_str) == Some("user") + && let Some(Value::Array(blocks)) = message.get_mut("content") + { + blocks + .iter_mut() + .filter(|block| { + block.get("type").and_then(Value::as_str) == Some("tool_result") + }) + .filter_map(|block| block.get_mut("content")) + .for_each(without_non_string_texts); + } + Recognized::Unrecognized(message) + } + }) + .collect() +} + fn without_null_safeguards(extra: Map) -> Map { extra .into_iter() @@ -193,22 +443,45 @@ fn mapped_host_betas(params: &MessagesOptionalParams) -> BetaSet { .collect() } +fn continues_message_thread(params: &MessagesOptionalParams) -> bool { + let Some(Value::Object(thread)) = params.extra.get("thread") else { + return false; + }; + thread.get("type").and_then(Value::as_str) == Some("continue") + && thread + .get("previous_message_id") + .is_none_or(Value::is_string) +} + +fn is_threaded(params: &MessagesOptionalParams) -> bool { + params + .extra + .get("thread") + .is_some_and(|thread| !thread.is_null()) +} + fn feature_betas(request: &MessagesRequest) -> BetaSet { let params = &request.params; let tools = params.tools.as_deref(); + let scanned_messages: &[Recognized] = if continues_message_thread(params) { + &[] + } else { + &request.messages + }; [ - requires_native_compaction_beta(params.compaction.as_ref(), &request.messages) + requires_native_compaction_beta(params.compaction.as_ref(), scanned_messages) .then_some(AnthropicBeta::Compact20260904), uses_structured_output(params).then_some(AnthropicBeta::StructuredOutputs20251113), (params.speed == Some(Recognized::Known(Speed::Fast))) .then_some(AnthropicBeta::FastMode20260201), - messages_carry_output_config(&request.messages) + messages_carry_output_config(scanned_messages) .then_some(AnthropicBeta::PerTurnControl20260701), + is_threaded(params).then_some(AnthropicBeta::MessageThreads20260812), uses_thinking_display_updates(params) .then_some(AnthropicBeta::ThinkingDisplayUpdates20260818), uses_thinking_block_binding(params) .then_some(AnthropicBeta::ThinkingBindingControls20260801), - messages_carry_tool_changes(&request.messages) + messages_carry_tool_changes(scanned_messages) .then_some(AnthropicBeta::MidConversationToolChanges20260701), has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301), is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120), @@ -254,9 +527,10 @@ fn carries_safeguards(params: &MessagesOptionalParams) -> bool { .is_some_and(|value| !value.is_null()) } -fn messages_carry_output_config(messages: &[Message]) -> bool { +fn messages_carry_output_config(messages: &[Recognized]) -> bool { messages .iter() + .filter_map(Recognized::known) .any(|message| message.extra.contains_key("output_config")) } @@ -301,9 +575,10 @@ fn is_tool_reference_change(block: &ContentBlock) -> bool { == Some("tool_reference") } -fn messages_carry_tool_changes(messages: &[Message]) -> bool { +fn messages_carry_tool_changes(messages: &[Recognized]) -> bool { messages .iter() + .filter_map(Recognized::known) .filter(|message| message.role == "system") .flat_map(Message::blocks) .any(is_tool_reference_change) @@ -334,7 +609,7 @@ pub fn drop_unsupported_params( let params = request.params; let speed = match ¶ms.speed { Some(speed) if !capabilities.supports_speed => { - reject("speed", format!("'{}'", speed_text(speed)), "")?; + reject("speed", speed_repr(speed), "")?; None } _ => params.speed.clone(), @@ -346,21 +621,21 @@ pub fn drop_unsupported_params( }); } let temperature = match params.temperature { - Some(temperature) if temperature != 1.0 => { + Some(temperature) if !is_one(&temperature) => { reject( "temperature", - json!(temperature).to_string(), + python_str(&temperature), "Only temperature=1 is supported. ", )?; None } temperature => temperature, }; - if let Some(top_p) = params.top_p { - reject("top_p", json!(top_p).to_string(), "")?; + if let Some(top_p) = ¶ms.top_p { + reject("top_p", python_str(top_p), "")?; } - if let Some(top_k) = params.top_k { - reject("top_k", json!(top_k).to_string(), "")?; + if let Some(top_k) = ¶ms.top_k { + reject("top_k", python_str(top_k), "")?; } Ok(MessagesRequest { params: MessagesOptionalParams { @@ -374,11 +649,10 @@ pub fn drop_unsupported_params( }) } -fn speed_text(speed: &Recognized) -> String { +fn speed_repr(speed: &Recognized) -> String { match speed { - Recognized::Known(speed) => <&'static str>::from(*speed).to_string(), - Recognized::Unrecognized(Value::String(text)) => text.clone(), - Recognized::Unrecognized(other) => other.to_string(), + Recognized::Known(speed) => format!("'{}'", <&'static str>::from(*speed)), + Recognized::Unrecognized(other) => litellm_python_compat::repr::repr(&python_value(other)), } } @@ -437,6 +711,7 @@ mod tests { use super::*; use crate::anthropic::common_utils::ENCRYPTED_REASONING_SIGNATURE_PREFIX; + use litellm_http::request::header_values; type Env = &'static [(&'static str, &'static str)]; @@ -679,6 +954,21 @@ mod tests { json!({"speed": "fast"}), "claude does not support speed='fast'. To drop unsupported params, set `litellm.drop_params = True`." )] + #[case::integer_temperature_keeps_no_decimal( + sampling_removed(), + json!({"temperature": 0}), + "claude does not support temperature=0. Only temperature=1 is supported. To drop unsupported params, set `litellm.drop_params = True`." + )] + #[case::integer_top_p( + sampling_removed(), + json!({"top_p": 1}), + "claude does not support top_p=1. To drop unsupported params, set `litellm.drop_params = True`." + )] + #[case::non_string_speed_is_reprd( + unmapped(), + json!({"speed": 2}), + "claude does not support speed=2. To drop unsupported params, set `litellm.drop_params = True`." + )] #[case::speed_ahead_of_sampling_params( sampling_removed(), json!({"speed": "fast", "temperature": 0.5}), @@ -696,6 +986,28 @@ mod tests { assert_eq!(detail.to_string(), message); } + #[rstest] + #[case::string_dropped(json!("json"), true, None)] + #[case::string_kept_without_drop_params(json!("json"), false, Some(json!("json")))] + #[case::object_kept(json!({"type": "json_schema"}), true, Some(json!({"type": "json_schema"})))] + fn a_non_object_output_format_is_dropped_under_drop_params( + #[case] output_format: Value, + #[case] drops_unsupported_params: bool, + #[case] expected: Option, + ) { + let context = MessagesTransformContext { + drops_unsupported_params, + ..MessagesTransformContext::with_lookup(unmapped(), false, &no_env) + }; + let transformed = ANTHROPIC_MESSAGES_CONFIG + .transform_anthropic_messages_request( + request(json!({"output_format": output_format})), + &context, + ) + .unwrap(); + assert_eq!(transformed.params.output_format, expected); + } + #[rstest] #[case::compaction_threshold( json!([{"type": "compaction", "compact_threshold": 200000}]), @@ -802,6 +1114,40 @@ mod tests { ); } + #[rstest] + #[case::continue_turn(json!({"type": "continue", "previous_message_id": "msg_1"}), false)] + #[case::continue_without_a_previous_message_id(json!({"type": "continue"}), false)] + #[case::continue_with_extra_keys(json!({"type": "continue", "previous_message_id": "msg_1", "note": 1}), false)] + #[case::create_turn(json!({"type": "create"}), true)] + #[case::continue_with_a_null_previous_message_id(json!({"type": "continue", "previous_message_id": null}), true)] + #[case::continue_with_a_numeric_previous_message_id(json!({"type": "continue", "previous_message_id": 1}), true)] + #[case::not_an_object(json!("continue"), true)] + fn advisor_history_is_kept_on_a_thread_continue_turn( + #[case] thread: Value, + #[case] stripped: bool, + unmapped: MessagesModelCapabilities, + ) { + let messages = if stripped { + json!([ + {"role": "user", "content": "Build a worker pool."}, + {"role": "assistant", "content": [ + {"type": "text", "text": "Let me consult the advisor."}, + {"type": "text", "text": "Here is the implementation."} + ]} + ]) + } else { + advisor_history() + }; + assert_eq!( + transform( + json!({"messages": advisor_history(), "thread": thread.clone()}), + unmapped, + false + ), + Ok(body(json!({"messages": messages, "thread": thread}))) + ); + } + #[rstest] fn bridge_minted_reasoning_is_stripped_from_the_wire(unmapped: MessagesModelCapabilities) { let messages = json!([ @@ -864,6 +1210,7 @@ mod tests { keeps_display_updates: false, }, drop_params: true, + drops_unsupported_params: false, } ); return; @@ -1165,6 +1512,55 @@ mod tests { ]}), &[] )] + #[case::thread_create(json!({"thread": {"type": "create"}}), &["message-threads-2026-08-12"])] + #[case::thread_continue( + json!({"thread": {"type": "continue", "previous_message_id": "msg_1"}}), + &["message-threads-2026-08-12"] + )] + #[case::thread_of_any_shape(json!({"thread": "opaque"}), &["message-threads-2026-08-12"])] + #[case::null_thread(json!({"thread": null}), &[])] + #[case::thread_create_still_scans_history( + json!({ + "thread": {"type": "create"}, + "messages": [ + {"role": "assistant", "content": [{"type": "compaction", "content": "summary", "signature": "sig"}]}, + {"role": "user", "content": "hi", "output_config": {"effort": "low"}}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ], + }), + &[ + "compact-2026-09-04", + "message-threads-2026-08-12", + "mid-conversation-tool-changes-2026-07-01", + "per-turn-control-2026-07-01", + ] + )] + #[case::thread_continue_skips_history( + json!({ + "thread": {"type": "continue", "previous_message_id": "msg_1"}, + "messages": [ + {"role": "assistant", "content": [{"type": "compaction", "content": "summary", "signature": "sig"}]}, + {"role": "user", "content": "hi", "output_config": {"effort": "low"}}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ], + }), + &["message-threads-2026-08-12"] + )] + #[case::thread_continue_keeps_param_betas( + json!({ + "thread": {"type": "continue", "previous_message_id": "msg_1"}, + "compaction": {}, + "speed": "fast", + }), + &["compact-2026-09-04", "fast-mode-2026-02-01", "message-threads-2026-08-12"] + )] + #[case::malformed_continue_scans_history( + json!({ + "thread": {"type": "continue", "previous_message_id": 7}, + "messages": [{"role": "user", "content": "hi", "output_config": {"effort": "low"}}], + }), + &["message-threads-2026-08-12", "per-turn-control-2026-07-01"] + )] fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) { assert_eq!(feature_betas(&request(fields)), betas(expected)); } @@ -1352,4 +1748,158 @@ mod tests { .collect(); assert_eq!(undeclared, Vec::<&String>::new()); } + + const FOUNDRY: &str = "https://team.services.ai.azure.com/anthropic/v1/messages"; + + #[rstest] + #[case::drops_a_beta_foundry_lacks( + FOUNDRY, + "advisor-tool-2026-03-01,context-1m-2025-08-07", + Some("context-1m-2025-08-07") + )] + #[case::forwards_unknown_betas_sorted( + FOUNDRY, + "zzz,files-api-2025-04-14,aaa", + Some("aaa,files-api-2025-04-14,zzz") + )] + #[case::removes_the_header_when_nothing_is_left(FOUNDRY, "advisor-tool-2026-03-01", None)] + #[case::cognitive_services_host( + "team.cognitiveservices.azure.com/anthropic", + "advisor-tool-2026-03-01", + None + )] + #[case::anthropic_host_is_untouched( + "https://api.anthropic.com/v1/messages", + "advisor-tool-2026-03-01", + Some("advisor-tool-2026-03-01") + )] + #[case::lookalike_host_is_untouched( + "https://notservices.ai.azure.com.example/v1/messages", + "advisor-tool-2026-03-01", + Some("advisor-tool-2026-03-01") + )] + fn foundry_urls_filter_betas_through_the_azure_ai_row( + #[case] url: &str, + #[case] betas: &str, + #[case] expected: Option<&str>, + ) { + let filtered = ANTHROPIC_MESSAGES_CONFIG + .headers_for_url(headers(&[("x-a", "1"), (BETA_HEADER, betas)]), url); + assert_eq!( + header_values(&filtered, BETA_HEADER).collect::>(), + expected.into_iter().collect::>() + ); + assert_eq!(header_values(&filtered, "x-a").collect::>(), ["1"]); + } + + #[rstest] + fn foundry_url_without_betas_keeps_the_headers() { + let sent = headers(&[("x-a", "1")]); + assert_eq!( + ANTHROPIC_MESSAGES_CONFIG.headers_for_url(sent.clone(), FOUNDRY), + sent + ); + } + + #[test] + fn the_http_error_retry_matches_the_cases_shared_with_python() { + let fixture: Value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/anthropic_messages_http_error_retry.json" + ))) + .unwrap(); + let table = &fixture["http_error_retry"]; + let cases = table["cases"].as_array().unwrap(); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let mut body = table["request"].clone(); + body["thinking"] = case["thinking"].clone(); + let status = u16::try_from(case["status"].as_u64().unwrap()).unwrap(); + let error_text = case["error_text"].as_str().unwrap(); + let retried = should_retry_anthropic_messages_on_http_error( + status, + error_text, + &body, + &MessagesTransformContext::default(), + ); + let stripped = case["stripped"].as_bool().unwrap(); + let transformed = transform_anthropic_messages_request_on_http_error( + status, + error_text, + body.clone(), + ); + let expected = if stripped { + RetriedRequest { + reasons: vec![RetryReason::ThinkingSignature], + body: table["stripped_request"].clone(), + } + } else { + RetriedRequest { + reasons: Vec::new(), + body, + } + }; + retried != case["retried"].as_bool().unwrap() || transformed != expected + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); + } + + #[test] + fn the_strict_tools_retry_matches_the_cases_shared_with_python() { + let fixture: Value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/anthropic_messages_http_error_retry.json" + ))) + .unwrap(); + let replay = &fixture["http_error_retry"]; + let table = &fixture["strict_tools_retry"]; + let cases = table["cases"].as_array().unwrap(); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let mut body = replay["request"].clone(); + body["thinking"] = case["thinking"].clone(); + body["tools"] = table["tools"].clone(); + let status = u16::try_from(case["status"].as_u64().unwrap()).unwrap(); + let error_text = case["error_text"].as_str().unwrap(); + let context = MessagesTransformContext { + drops_unsupported_params: case["drops_unsupported_params"].as_bool().unwrap(), + ..MessagesTransformContext::default() + }; + let mut reasons = Vec::new(); + let mut expected = if case["stripped"].as_bool().unwrap() { + reasons.push(RetryReason::ThinkingSignature); + replay["stripped_request"].clone() + } else { + body.clone() + }; + expected["tools"] = if case["strict_dropped"].as_bool().unwrap() { + reasons.push(RetryReason::StrictTools { + dropped: usize::try_from(table["dropped"].as_u64().unwrap()).unwrap(), + }); + table["tools_without_strict"].clone() + } else { + table["tools"].clone() + }; + let retried = should_retry_anthropic_messages_on_http_error( + status, error_text, &body, &context, + ); + let transformed = + transform_anthropic_messages_request_on_http_error(status, error_text, body); + retried != case["retried"].as_bool().unwrap() + || transformed + != RetriedRequest { + reasons, + body: expected, + } + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); + } } diff --git a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs index d13cfa81a94..fb50d229bc7 100644 --- a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -1,3 +1,5 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; +use litellm_host::interceptors::RetriedRequest; use litellm_http::request::{has_bearer_auth, has_header}; use litellm_llms_types::{ formats::messages::{MessagesOptionalParams, MessagesRequest}, @@ -5,6 +7,7 @@ use litellm_llms_types::{ }; use litellm_router_types::LitellmParams; use litellm_secret_value::SecretValue; +use serde_json::Value; use crate::{ Error, @@ -21,7 +24,7 @@ use crate::{ base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, messages::{ - context::MessagesTransformContext, + context::{MessagesShapeOptions, MessagesTransformContext}, normalization::{normalize_system_role_messages, strip_cache_control_scope}, transformation::BaseMessagesConfig, }, @@ -36,12 +39,28 @@ pub const AZURE_ANTHROPIC_MESSAGES_CONFIG: AzureAnthropicMessagesConfig = AzureAnthropicMessagesConfig; impl BaseMessagesConfig for AzureAnthropicMessagesConfig { + fn tool_custom_field(&self, model: &str) -> crate::base_llm::messages::tools::ToolCustomField { + crate::base_llm::messages::tools::ToolCustomField::Rewrite { + hoist_defer_loading: crate::anthropic::messages::transformation::is_claude_model(model), + } + } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn get_complete_url( diff --git a/litellm-rust/crates/llms/src/base_llm/messages/context.rs b/litellm-rust/crates/llms/src/base_llm/messages/context.rs index b514e975aa4..e5593d3b216 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/context.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/context.rs @@ -23,6 +23,12 @@ impl SupportedEffortTiers { } } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MessagesShapeOptions { + pub reasoning_auto_summary: bool, + pub presanitized: bool, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct MessagesModelCapabilities { #[serde(default)] @@ -32,6 +38,8 @@ pub struct MessagesModelCapabilities { #[serde(default)] pub thinking_always_on: bool, #[serde(default)] + pub thinking_on_by_default: bool, + #[serde(default)] pub supports_legacy_thinking: bool, #[serde(default)] pub supports_output_config: bool, @@ -55,6 +63,7 @@ impl Default for MessagesModelCapabilities { supports_reasoning: false, supports_adaptive_thinking: false, thinking_always_on: false, + thinking_on_by_default: false, supports_legacy_thinking: false, supports_output_config: false, supports_sampling_params: true, @@ -120,6 +129,7 @@ pub struct ThinkingContext { pub struct MessagesTransformContext { pub thinking: ThinkingContext, pub drop_params: bool, + pub drops_unsupported_params: bool, } impl MessagesTransformContext { pub fn new(capabilities: MessagesModelCapabilities, drop_params: bool) -> Self { @@ -139,6 +149,7 @@ impl MessagesTransformContext { keeps_display_updates: false, }, drop_params, + drops_unsupported_params: false, } } diff --git a/litellm-rust/crates/llms/src/base_llm/messages/mod.rs b/litellm-rust/crates/llms/src/base_llm/messages/mod.rs index e9591daa9da..5c1563c7a23 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/mod.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/mod.rs @@ -2,4 +2,5 @@ pub mod context; pub mod mid_conversation_system; pub mod normalization; pub mod streaming; +pub mod tools; pub mod transformation; diff --git a/litellm-rust/crates/llms/src/base_llm/messages/normalization.rs b/litellm-rust/crates/llms/src/base_llm/messages/normalization.rs index 7e24da2b5eb..ee2f4dcaeef 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/normalization.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/normalization.rs @@ -1,6 +1,9 @@ -use litellm_llms_types::formats::messages::{ - CacheControl, ContentBlock, Message, MessageContent, MessagesOptionalParams, MessagesRequest, - SystemPrompt, +use litellm_llms_types::{ + formats::messages::{ + CacheControl, ContentBlock, Message, MessageContent, MessagesOptionalParams, + MessagesRequest, SystemPrompt, + }, + recognized::{Recognized, map_known_runs}, }; use crate::{ @@ -22,15 +25,22 @@ pub fn normalize_system_role_messages( let leading_count = request .messages .iter() - .take_while(|message| is_system_role_message(message)) + .take_while(|message| message.known().is_some_and(is_system_role_message)) .count(); let mut messages = request.messages.into_iter(); - let hoisted: Vec = messages.by_ref().take(leading_count).collect(); - let remaining: Vec = messages.collect(); + let hoisted: Vec = messages + .by_ref() + .take(leading_count) + .filter_map(|message| match message { + Recognized::Known(message) => Some(message), + Recognized::Unrecognized(_) => None, + }) + .collect(); + let remaining: Vec> = messages.collect(); let remaining = if supports_mid_conversation_system { remaining } else { - convert_mid_conversation_system_turns(remaining) + map_known_runs(remaining, convert_mid_conversation_system_turns) }; let system = if hoisted.is_empty() { request.params.system @@ -94,7 +104,7 @@ pub fn strip_cache_control_scope(request: MessagesRequest) -> MessagesRequest { messages: request .messages .into_iter() - .map(strip_scope_from_message) + .map(|message| message.map_known(strip_scope_from_message)) .collect(), params: MessagesOptionalParams { system: request.params.system.map(strip_scope_from_system), diff --git a/litellm-rust/crates/llms/src/base_llm/messages/tools.rs b/litellm-rust/crates/llms/src/base_llm/messages/tools.rs new file mode 100644 index 00000000000..49113872c56 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/messages/tools.rs @@ -0,0 +1,194 @@ +use litellm_llms_types::{ + formats::messages::{MessagesOptionalParams, MessagesRequest, MessagesTool}, + recognized::Recognized, +}; +use serde_json::{Map, Value}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ToolCustomField { + Keep, + Rewrite { hoist_defer_loading: bool }, +} + +impl ToolCustomField { + pub fn apply(self, request: MessagesRequest, drop_params: bool) -> MessagesRequest { + match self { + Self::Keep => request, + Self::Rewrite { + hoist_defer_loading, + } => tools_without_custom_field(request, hoist_defer_loading, drop_params), + } + } +} + +fn tools_without_custom_field( + request: MessagesRequest, + hoist: bool, + drop_custom: bool, +) -> MessagesRequest { + let tools = request.params.tools.map(|tools| { + tools + .into_iter() + .map(|tool| tool_without_custom_field(tool, hoist, drop_custom)) + .collect() + }); + MessagesRequest { + params: MessagesOptionalParams { + tools, + ..request.params + }, + ..request + } +} + +fn tool_without_custom_field( + tool: Recognized, + hoist: bool, + drop_custom: bool, +) -> Recognized { + let Ok(Value::Object(fields)) = serde_json::to_value(&tool) else { + return tool; + }; + let Some(custom) = fields.get("custom").cloned() else { + return tool; + }; + let defer_loading = hoist + .then(|| hoistable_defer_loading(&fields, &custom)) + .flatten(); + let without_custom = fields + .iter() + .filter(|(key, _)| *key != "custom") + .map(|(key, value)| (key.clone(), value.clone())); + let rebuilt: Map = match defer_loading { + None if drop_custom => without_custom.collect(), + None => return tool, + Some(defer_loading) => { + let unhoisted: Map = custom + .as_object() + .into_iter() + .flatten() + .filter(|(key, _)| *key != "defer_loading") + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + let kept_custom = (!unhoisted.is_empty() && !drop_custom) + .then(|| ("custom".to_string(), Value::Object(unhoisted))); + without_custom + .chain([("defer_loading".to_string(), Value::Bool(defer_loading))]) + .chain(kept_custom) + .collect() + } + }; + serde_json::from_value(Value::Object(rebuilt)).unwrap_or(tool) +} + +fn hoistable_defer_loading(tool: &Map, custom: &Value) -> Option { + if tool.contains_key("defer_loading") { + return None; + } + custom.as_object()?.get("defer_loading")?.as_bool() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + const WEATHER: &str = "get_weather"; + + fn tool(fields: Value) -> Value { + merged( + json!({"name": WEATHER, "input_schema": {"type": "object"}}), + fields, + ) + } + + fn tools_after(policy: ToolCustomField, tools: Value, drop_custom: bool) -> Value { + let request: MessagesRequest = serde_json::from_value(json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}], + "tools": [tools] + })) + .unwrap(); + serde_json::to_value(policy.apply(request, drop_custom)).unwrap()["tools"][0].clone() + } + + fn merged(base: Value, fields: Value) -> Value { + let mut merged = base.as_object().unwrap().clone(); + merged.extend(fields.as_object().unwrap().clone()); + Value::Object(merged) + } + + const HOIST: ToolCustomField = ToolCustomField::Rewrite { + hoist_defer_loading: true, + }; + const NO_HOIST: ToolCustomField = ToolCustomField::Rewrite { + hoist_defer_loading: false, + }; + + #[rstest] + #[case::hoists_defer_loading( + HOIST, + json!({"custom": {"defer_loading": true}}), + false, + json!({"defer_loading": true}) + )] + #[case::keeps_other_custom_keys( + HOIST, + json!({"custom": {"defer_loading": false, "note": "kept"}}), + false, + json!({"defer_loading": false, "custom": {"note": "kept"}}) + )] + #[case::drops_other_custom_keys_with_drop_params( + HOIST, + json!({"custom": {"defer_loading": true, "note": "dropped"}}), + true, + json!({"defer_loading": true}) + )] + #[case::keeps_an_opaque_custom( + HOIST, + json!({"custom": "opaque"}), + false, + json!({"custom": "opaque"}) + )] + #[case::drops_an_opaque_custom_with_drop_params( + HOIST, + json!({"custom": "opaque"}), + true, + json!({}) + )] + #[case::own_defer_loading_wins( + HOIST, + json!({"defer_loading": false, "custom": {"defer_loading": true}}), + false, + json!({"defer_loading": false, "custom": {"defer_loading": true}}) + )] + #[case::not_hoisted_off_claude_hosts( + NO_HOIST, + json!({"custom": {"defer_loading": true}}), + false, + json!({"custom": {"defer_loading": true}}) + )] + fn tool_custom_field_follows_the_python_handler( + #[case] policy: ToolCustomField, + #[case] fields: Value, + #[case] drop_custom: bool, + #[case] expected: Value, + ) { + assert_eq!( + tools_after(policy, tool(fields), drop_custom), + tool(expected) + ); + } + + #[rstest] + fn keep_leaves_the_custom_field() { + let custom = tool(json!({"custom": {"defer_loading": true}})); + assert_eq!( + tools_after(ToolCustomField::Keep, custom.clone(), true), + custom + ); + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs b/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs index 744ce89afdb..13c021c2aec 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/transformation.rs @@ -1,19 +1,33 @@ +use litellm_host::interceptors::RetriedRequest; use litellm_llms_types::formats::messages::{ MessagesOptionalParams, MessagesRequest, MessagesResponse, }; use litellm_router_types::LitellmParams; +use litellm_secret_value::SecretValue; use serde_json::Value; -use super::context::MessagesTransformContext; +use super::{ + context::{MessagesShapeOptions, MessagesTransformContext}, + tools::ToolCustomField, +}; pub use crate::base_llm::auth::{Headers, ValidatedEnvironment}; use crate::{Error, base_llm::messages::streaming::StreamDecoder}; +pub enum FederatedEnvironment { + Validated(ValidatedEnvironment), + Unauthenticated { + headers: Headers, + missing: Error, + federate: fn(Headers, SecretValue) -> ValidatedEnvironment, + }, +} + pub trait BaseMessagesConfig: Sync { fn shape_request( &self, request: MessagesRequest, - _reasoning_auto_summary: bool, + _options: MessagesShapeOptions, ) -> Result { Ok(request) } @@ -35,6 +49,10 @@ pub trait BaseMessagesConfig: Sync { Ok(request) } + fn tool_custom_field(&self, _model: &str) -> ToolCustomField { + ToolCustomField::Keep + } + fn transform_anthropic_messages_response( &self, _model: &str, @@ -62,6 +80,26 @@ pub trait BaseMessagesConfig: Sync { env_lookup: &dyn Fn(&str) -> Option, ) -> Result; + fn validate_federated_environment( + &self, + headers: Headers, + api_key: Option<&str>, + model: &str, + optional_params: &MessagesOptionalParams, + litellm_params: &LitellmParams, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + self.validate_environment( + headers, + api_key, + model, + optional_params, + litellm_params, + env_lookup, + ) + .map(FederatedEnvironment::Validated) + } + /// `None` relays the upstream bytes untouched, which is right for every host that already /// speaks Anthropic SSE. A host on another wire returns the decoder that lifts its frames /// into Anthropic stream events, and the route re-encodes those as Anthropic SSE. @@ -77,6 +115,10 @@ pub trait BaseMessagesConfig: Sync { headers } + fn headers_for_url(&self, headers: Headers, _url: &str) -> Headers { + headers + } + fn request_body( &self, request: MessagesRequest, @@ -91,6 +133,16 @@ pub trait BaseMessagesConfig: Sync { fn wire_body(&self, body: Value) -> Value { body } + + fn retry_after_http_error( + &self, + _status: u16, + _error_text: &str, + _request_body: &Value, + _context: &MessagesTransformContext, + ) -> Option { + None + } } #[cfg(test)] @@ -149,9 +201,10 @@ mod tests { } #[rstest] - #[case::disabled(false)] - #[case::enabled(true)] - fn default_shaping_preserves_provider_policy_inputs(#[case] reasoning_auto_summary: bool) { + fn default_shaping_preserves_provider_policy_inputs( + #[values(false, true)] reasoning_auto_summary: bool, + #[values(false, true)] presanitized: bool, + ) { let request: MessagesRequest = serde_json::from_value(serde_json::json!({ "model": "test-model", "metadata": {"user_id": 7, "extra": "keep"}, @@ -160,7 +213,13 @@ mod tests { })) .unwrap(); assert_eq!( - DefaultsConfig.shape_request(request.clone(), reasoning_auto_summary), + DefaultsConfig.shape_request( + request.clone(), + MessagesShapeOptions { + reasoning_auto_summary, + presanitized + } + ), Ok(request) ); assert_eq!( diff --git a/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs b/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs index 5b89345a1c5..4bf30b56f00 100644 --- a/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs @@ -1,3 +1,5 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; +use litellm_host::interceptors::RetriedRequest; use std::sync::LazyLock; use litellm_auth::{AwsParams, CredentialPlacement, ParamKind, ParamSpec}; @@ -21,7 +23,10 @@ use crate::{ }, base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, - messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig}, + messages::{ + context::{MessagesShapeOptions, MessagesTransformContext}, + transformation::BaseMessagesConfig, + }, }, bedrock::common_utils::validated_region, }; @@ -50,12 +55,28 @@ pub const BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG: BedrockClaudePlatformMessages BedrockClaudePlatformMessagesConfig; impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig { + fn tool_custom_field(&self, model: &str) -> crate::base_llm::messages::tools::ToolCustomField { + crate::base_llm::messages::tools::ToolCustomField::Rewrite { + hoist_defer_loading: crate::anthropic::messages::transformation::is_claude_model(model), + } + } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn validate_environment( diff --git a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs index 1d380c917f6..fa037974342 100644 --- a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs @@ -1,9 +1,11 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; use litellm_core_utils::decline::Decline; +use litellm_host::interceptors::RetriedRequest; use std::convert::Infallible; use crate::{ anthropic::messages::handler::shape_anthropic_messages_request, - base_llm::messages::context::MessagesTransformContext, + base_llm::messages::context::{MessagesShapeOptions, MessagesTransformContext}, }; use futures_util::StreamExt; use litellm_auth::AwsParams; @@ -103,12 +105,22 @@ fn invoke_url( } impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn get_complete_url( diff --git a/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs b/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs index f139b93e259..0e67ea1a139 100644 --- a/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs @@ -1,3 +1,5 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; +use litellm_host::interceptors::RetriedRequest; use std::sync::LazyLock; use litellm_auth::{AwsParams, CredentialPlacement}; @@ -17,6 +19,7 @@ use serde_json::Value; use crate::{ Error, anthropic::{ + beta_headers::accepted_or_unlisted, common_utils::{existing_betas, merge_beta_headers}, messages::{ handler::shape_anthropic_messages_request, @@ -26,7 +29,7 @@ use crate::{ base_llm::{ auth::AuthScheme, messages::{ - context::MessagesTransformContext, + context::{MessagesShapeOptions, MessagesTransformContext}, transformation::{BaseMessagesConfig, Headers, ValidatedEnvironment}, }, }, @@ -56,6 +59,16 @@ pub const BEDROCK_MANTLE_MESSAGES_CONFIG: BedrockMantleAnthropicMessagesConfig = BedrockMantleAnthropicMessagesConfig; impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig { + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn validate_environment( &self, headers: Headers, @@ -129,9 +142,9 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig { fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn wire_body(&self, body: Value) -> Value { @@ -191,14 +204,10 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig { .filter_map(Value::as_str) .filter_map(|beta| beta.parse::().ok()) .collect(); - let caller: BetaSet = existing_betas(&headers) - .union(body_betas) - .into_iter() - .filter_map(|beta| match beta { - AnthropicBeta::Other(_) => Some(beta), - _ => beta.on(BetaProvider::BedrockMantle), - }) - .collect(); + let caller = accepted_or_unlisted( + existing_betas(&headers).union(body_betas), + BetaProvider::BedrockMantle, + ); let safeguards: BetaSet = request .params .extra diff --git a/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs b/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs index 0dbcea01439..ee2aec9efb9 100644 --- a/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs @@ -1,4 +1,6 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; use litellm_auth::CredentialPlacement; +use litellm_host::interceptors::RetriedRequest; use litellm_secret_value::SecretValue; use litellm_core_utils::settings::resolve_non_empty; @@ -21,7 +23,10 @@ use crate::{ }, base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, - messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig}, + messages::{ + context::{MessagesShapeOptions, MessagesTransformContext}, + transformation::BaseMessagesConfig, + }, }, }; @@ -42,12 +47,28 @@ pub const DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG: DeepSeekAnthropicMessagesConfig = DeepSeekAnthropicMessagesConfig; impl BaseMessagesConfig for DeepSeekAnthropicMessagesConfig { + fn tool_custom_field(&self, _model: &str) -> crate::base_llm::messages::tools::ToolCustomField { + crate::base_llm::messages::tools::ToolCustomField::Rewrite { + hoist_defer_loading: false, + } + } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn get_complete_url( diff --git a/litellm-rust/crates/llms/src/edenai/messages/transformation.rs b/litellm-rust/crates/llms/src/edenai/messages/transformation.rs index d5944484522..d4f577e09a9 100644 --- a/litellm-rust/crates/llms/src/edenai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/edenai/messages/transformation.rs @@ -1,4 +1,6 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; use litellm_auth::CredentialPlacement; +use litellm_host::interceptors::RetriedRequest; use litellm_secret_value::SecretValue; use litellm_core_utils::settings::resolve_non_empty; @@ -33,6 +35,22 @@ pub struct EdenAIAnthropicMessagesConfig; pub const EDENAI_MESSAGES_CONFIG: EdenAIAnthropicMessagesConfig = EdenAIAnthropicMessagesConfig; impl BaseMessagesConfig for EdenAIAnthropicMessagesConfig { + fn tool_custom_field(&self, _model: &str) -> crate::base_llm::messages::tools::ToolCustomField { + crate::base_llm::messages::tools::ToolCustomField::Rewrite { + hoist_defer_loading: false, + } + } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn validate_environment( &self, headers: Headers, diff --git a/litellm-rust/crates/llms/src/error.rs b/litellm-rust/crates/llms/src/error.rs index 2844fbf3e01..77ee1f1036f 100644 --- a/litellm-rust/crates/llms/src/error.rs +++ b/litellm-rust/crates/llms/src/error.rs @@ -103,6 +103,13 @@ impl Error { matches!(self, Self::Http(error) if error.is_body_read_timeout()) } + pub fn body_cut(&self) -> Option { + match self { + Self::Http(error) => error.body_cut(), + _ => None, + } + } + pub fn is_timeout(&self) -> Option { match self { Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()), @@ -161,6 +168,10 @@ pub enum ErrorDetail { actual: String, choices: Vec<&'static str>, }, + #[error( + "Invalid reasoning_effort: {actual}. Must be one of: 'minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'none'" + )] + InvalidReasoningEffort { actual: String }, #[error("{field}='{value}' is not supported by this model. Got model: {model}")] UnsupportedValue { field: &'static str, @@ -180,6 +191,14 @@ pub enum ErrorDetail { "thinking.budget_tokens must be a number, got {actual}. To drop it, set `litellm.drop_params=True`, or `drop_params: true` under `litellm_settings` on the proxy" )] NonNumericThinkingBudget { actual: &'static str }, + /// Python's comparison of two operands it cannot order, which raises `TypeError` with this + /// text before any request is sent. + #[error("'{op}' not supported between instances of '{left}' and '{right}'")] + UnorderableTypes { + op: &'static str, + left: &'static str, + right: &'static str, + }, #[error( "Missing API Base - A call is being made to LLM Provider but no api base is set either in the environment variables ({{LLM_PROVIDER}}_API_KEY) or via params" )] diff --git a/litellm-rust/crates/llms/src/minimax/messages/transformation.rs b/litellm-rust/crates/llms/src/minimax/messages/transformation.rs index dafe5fbd16b..2eaf182e657 100644 --- a/litellm-rust/crates/llms/src/minimax/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/minimax/messages/transformation.rs @@ -1,5 +1,8 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; use litellm_auth::CredentialPlacement; +use litellm_host::interceptors::RetriedRequest; use litellm_secret_value::SecretValue; +use serde_json::Value; use litellm_core_utils::settings::resolve_non_empty; use litellm_llms_types::{ @@ -19,7 +22,10 @@ use crate::{ }, base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, - messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig}, + messages::{ + context::{MessagesShapeOptions, MessagesTransformContext}, + transformation::BaseMessagesConfig, + }, }, }; @@ -31,12 +37,28 @@ pub struct MinimaxMessagesConfig; pub const MINIMAX_MESSAGES_CONFIG: MinimaxMessagesConfig = MinimaxMessagesConfig; impl BaseMessagesConfig for MinimaxMessagesConfig { + fn tool_custom_field(&self, _model: &str) -> crate::base_llm::messages::tools::ToolCustomField { + crate::base_llm::messages::tools::ToolCustomField::Rewrite { + hoist_defer_loading: false, + } + } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn validate_environment( diff --git a/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs b/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs index eeb3d1cb7f8..dc09f4e4149 100644 --- a/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs @@ -1,5 +1,8 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; use litellm_auth::CredentialPlacement; +use litellm_host::interceptors::RetriedRequest; use litellm_secret_value::SecretValue; +use serde_json::Value; use litellm_core_utils::settings::resolve_non_empty; use litellm_http::request::{has_bearer_auth, without_headers}; @@ -31,6 +34,16 @@ pub const OPENROUTER_MESSAGES_CONFIG: OpenRouterAnthropicMessagesConfig = OpenRouterAnthropicMessagesConfig; impl BaseMessagesConfig for OpenRouterAnthropicMessagesConfig { + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn validate_environment( &self, headers: Headers, diff --git a/litellm-rust/crates/llms/src/tencent/messages/transformation.rs b/litellm-rust/crates/llms/src/tencent/messages/transformation.rs index 621a09f4fb0..b3ee28b7474 100644 --- a/litellm-rust/crates/llms/src/tencent/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/tencent/messages/transformation.rs @@ -1,5 +1,8 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; use litellm_auth::CredentialPlacement; +use litellm_host::interceptors::RetriedRequest; use litellm_secret_value::SecretValue; +use serde_json::Value; use litellm_core_utils::settings::resolve_non_empty; use litellm_llms_types::{ @@ -19,7 +22,10 @@ use crate::{ }, base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, - messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig}, + messages::{ + context::{MessagesShapeOptions, MessagesTransformContext}, + transformation::BaseMessagesConfig, + }, }, }; @@ -32,12 +38,28 @@ pub struct TencentAnthropicMessagesConfig; pub const TENCENT_MESSAGES_CONFIG: TencentAnthropicMessagesConfig = TencentAnthropicMessagesConfig; impl BaseMessagesConfig for TencentAnthropicMessagesConfig { + fn tool_custom_field(&self, _model: &str) -> crate::base_llm::messages::tools::ToolCustomField { + crate::base_llm::messages::tools::ToolCustomField::Rewrite { + hoist_defer_loading: false, + } + } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn validate_environment( diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index 038617c2e10..674b9951d3e 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -1,8 +1,10 @@ +use crate::anthropic::messages::transformation::retry_anthropic_messages_after_http_error; use litellm_auth::VertexParams; use litellm_auth_gcp::{ VertexConfig, get_vertex_ai_location, get_vertex_ai_project, get_vertex_ai_project_from_credentials, secret_names, }; +use litellm_host::interceptors::RetriedRequest; use litellm_http::request::{has_header, header_value, without_headers}; use litellm_llms_types::{ formats::messages::{MessagesOptionalParams, MessagesRequest, MessagesTool, OutputConfig}, @@ -24,7 +26,7 @@ use crate::{ base_llm::{ auth::{AuthScheme, Headers, ValidatedEnvironment}, messages::{ - context::MessagesTransformContext, + context::{MessagesShapeOptions, MessagesTransformContext}, normalization::{normalize_system_role_messages, strip_cache_control_scope}, transformation::BaseMessagesConfig, }, @@ -47,12 +49,28 @@ pub const VERTEX_ANTHROPIC_MESSAGES_CONFIG: VertexAiPartnerModelsAnthropicMessag VertexAiPartnerModelsAnthropicMessagesConfig; impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig { + fn tool_custom_field(&self, model: &str) -> crate::base_llm::messages::tools::ToolCustomField { + crate::base_llm::messages::tools::ToolCustomField::Rewrite { + hoist_defer_loading: crate::anthropic::messages::transformation::is_claude_model(model), + } + } + + fn retry_after_http_error( + &self, + status: u16, + error_text: &str, + request_body: &Value, + context: &MessagesTransformContext, + ) -> Option { + retry_anthropic_messages_after_http_error(status, error_text, request_body, context) + } + fn shape_request( &self, request: MessagesRequest, - reasoning_auto_summary: bool, + options: MessagesShapeOptions, ) -> Result { - shape_anthropic_messages_request(request, reasoning_auto_summary) + shape_anthropic_messages_request(request, options) } fn get_complete_url( @@ -700,6 +718,7 @@ mod tests { ..ThinkingContext::default() }, drop_params: false, + drops_unsupported_params: false, }, ); assert_eq!(body["system"], json!("be terse")); @@ -717,6 +736,7 @@ mod tests { ..ThinkingContext::default() }, drop_params: false, + drops_unsupported_params: false, } } diff --git a/litellm-rust/crates/llms/tests/fixtures/anthropic_messages_http_error_retry.json b/litellm-rust/crates/llms/tests/fixtures/anthropic_messages_http_error_retry.json new file mode 100644 index 00000000000..843a1689056 --- /dev/null +++ b/litellm-rust/crates/llms/tests/fixtures/anthropic_messages_http_error_retry.json @@ -0,0 +1,623 @@ +{ + "invalid_thinking_block_error": [ + { + "name": "invalid_signature", + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "expected": true + }, + { + "name": "invalid_signature_in_an_error_envelope", + "error_text": "{\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"messages.3.content.3: Invalid `signature` in `thinking` block\"},\"request_id\":\"req_011Ca2EtQDxp7x6RGUY2jVn9\"}", + "expected": true + }, + { + "name": "signature_not_a_string", + "error_text": "{\"message\":\"messages.2.content.0.thinking.signature.str: Input should be a valid string\"}", + "expected": true + }, + { + "name": "signature_not_a_string_bare", + "error_text": "messages.4.content.1.thinking.signature.str: Input should be a valid string", + "expected": true + }, + { + "name": "empty_thinking", + "error_text": "messages.1.content.0.thinking: each thinking block must contain thinking", + "expected": true + }, + { + "name": "bound_to_a_different_conversation", + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation.", + "expected": true + }, + { + "name": "uppercase", + "error_text": "MESSAGES.1: INVALID SIGNATURE IN THINKING BLOCK", + "expected": true + }, + { + "name": "signature_without_a_verdict", + "error_text": "thinking signature is malformed", + "expected": false + }, + { + "name": "invalid_without_thinking", + "error_text": "Invalid `signature` in `tool_use` block", + "expected": false + }, + { + "name": "model_not_found", + "error_text": "invalid_request_error: model not found", + "expected": false + }, + { + "name": "rate_limited", + "error_text": "rate limit exceeded", + "expected": false + }, + { + "name": "unrelated", + "error_text": "max_tokens: 4096 > 64000", + "expected": false + }, + { + "name": "empty", + "error_text": "", + "expected": false + } + ], + "thinking_binding_mismatch_error": [ + { + "name": "bound_elsewhere", + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation.", + "expected": true + }, + { + "name": "mixed_case", + "error_text": "The block is Bound To A Different Conversation.", + "expected": true + }, + { + "name": "plain_signature", + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "expected": false + } + ], + "http_error_retry": { + "request": { + "model": "claude-sonnet-4-5", + "max_tokens": 4096, + "messages": [ + { + "role": "user", + "content": "hi" + }, + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "plan", + "signature": "sig" + }, + { + "type": "text", + "text": "hello" + } + ] + }, + { + "role": "assistant", + "content": [ + { + "type": "redacted_thinking", + "data": "opaque" + } + ] + }, + { + "role": "user", + "content": "again" + } + ] + }, + "stripped_request": { + "model": "claude-sonnet-4-5", + "max_tokens": 4096, + "messages": [ + { + "role": "user", + "content": "hi" + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "hello" + } + ] + }, + { + "role": "user", + "content": "again" + } + ] + }, + "cases": [ + { + "name": "invalid_signature", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "retried": true, + "stripped": true + }, + { + "name": "missing_signature", + "status": 400, + "error_text": "messages.1.content.0.thinking.signature.str: Input should be a valid string", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "retried": true, + "stripped": true + }, + { + "name": "empty_thinking", + "status": 400, + "error_text": "messages.1.content.0.thinking: each thinking block must contain thinking", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "retried": true, + "stripped": true + }, + { + "name": "unpinned_binding_mismatch", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation.", + "thinking": { + "type": "adaptive" + }, + "retried": true, + "stripped": true + }, + { + "name": "caller_pinned_binding", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation.", + "thinking": { + "type": "adaptive", + "block_binding": { + "prefix_mismatch_behavior": "error" + } + }, + "retried": false, + "stripped": true + }, + { + "name": "caller_pinned_binding_other_signature_error", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "thinking": { + "type": "adaptive", + "block_binding": { + "prefix_mismatch_behavior": "error" + } + }, + "retried": true, + "stripped": true + }, + { + "name": "not_a_400", + "status": 429, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "retried": false, + "stripped": false + }, + { + "name": "unrelated_400", + "status": 400, + "error_text": "max_tokens: 4096 > 64000", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "retried": false, + "stripped": false + }, + { + "name": "signature_without_thinking", + "status": 400, + "error_text": "Invalid `signature` in `tool_use` block", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "retried": false, + "stripped": false + } + ] + }, + "claude_strict_tools_rejection": [ + { + "name": "too_many_strict_tools", + "status": 400, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "expected": true + }, + { + "name": "too_many_strict_tools_in_an_error_envelope", + "status": 400, + "error_text": "{\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"Too many strict tools (24). The maximum is 20.\"}}", + "expected": true + }, + { + "name": "non_ascii_digits", + "status": 400, + "error_text": "Too many strict tools (٢١)", + "expected": true + }, + { + "name": "optional_parameters", + "status": 400, + "error_text": "Schemas contains too many optional parameters (30)", + "expected": true + }, + { + "name": "union_types", + "status": 400, + "error_text": "Schemas contains too many parameters with union types", + "expected": true + }, + { + "name": "compiled_grammar_too_large", + "status": 400, + "error_text": "The compiled grammar is too large", + "expected": true + }, + { + "name": "compiled_grammar_size", + "status": 400, + "error_text": "Compiled grammar size exceeds the limit", + "expected": true + }, + { + "name": "custom_strict", + "status": 400, + "error_text": "tools.3.custom.strict: Extra inputs are not permitted", + "expected": true + }, + { + "name": "custom", + "status": 400, + "error_text": "tools.0.custom: Extra inputs are not permitted", + "expected": true + }, + { + "name": "grammar_compilation_timed_out", + "status": 400, + "error_text": "Grammar compilation timed out", + "expected": true + }, + { + "name": "no_count", + "status": 400, + "error_text": "tools.0.strict: too many strict tools", + "expected": false + }, + { + "name": "empty_count", + "status": 400, + "error_text": "Too many strict tools ()", + "expected": false + }, + { + "name": "lowercase_grammar_timeout", + "status": 400, + "error_text": "grammar compilation timed out", + "expected": false + }, + { + "name": "custom_without_separator", + "status": 400, + "error_text": "tools.0.custom.strict", + "expected": false + }, + { + "name": "uppercase_schemas", + "status": 400, + "error_text": "SCHEMAS CONTAINS TOO MANY OPTIONAL PARAMETERS", + "expected": false + }, + { + "name": "not_a_400", + "status": 429, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "expected": false + }, + { + "name": "empty", + "status": 400, + "error_text": "", + "expected": false + } + ], + "tools_without_strict": [ + { + "name": "strict_dropped_from_every_tool", + "tools": [ + { + "name": "a", + "strict": true, + "input_schema": {} + }, + { + "name": "b", + "strict": false + }, + { + "name": "c" + } + ], + "expected": { + "tools": [ + { + "name": "a", + "input_schema": {} + }, + { + "name": "b" + }, + { + "name": "c" + } + ], + "dropped_count": 1 + } + }, + { + "name": "only_a_true_strict_counts", + "tools": [ + { + "name": "a", + "strict": 1 + }, + { + "name": "b", + "strict": "true" + } + ], + "expected": null + }, + { + "name": "no_strict", + "tools": [ + { + "name": "a" + } + ], + "expected": null + }, + { + "name": "empty", + "tools": [], + "expected": null + }, + { + "name": "non_object_tool", + "tools": [ + { + "name": "a", + "strict": true + }, + "b" + ], + "expected": null + }, + { + "name": "not_a_list", + "tools": { + "name": "a", + "strict": true + }, + "expected": null + }, + { + "name": "null", + "tools": null, + "expected": null + } + ], + "strict_tools_retry": { + "tools": [ + { + "name": "a", + "input_schema": { + "type": "object" + }, + "strict": true + }, + { + "name": "b", + "input_schema": { + "type": "object" + }, + "strict": false + }, + { + "name": "c", + "input_schema": { + "type": "object" + } + } + ], + "tools_without_strict": [ + { + "name": "a", + "input_schema": { + "type": "object" + } + }, + { + "name": "b", + "input_schema": { + "type": "object" + } + }, + { + "name": "c", + "input_schema": { + "type": "object" + } + } + ], + "dropped": 1, + "cases": [ + { + "name": "strict_rejection_under_drop_params", + "status": 400, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": true, + "stripped": false, + "strict_dropped": true + }, + { + "name": "strict_rejection_without_drop_params", + "status": 400, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": false, + "retried": false, + "stripped": false, + "strict_dropped": true + }, + { + "name": "compiled_grammar_under_drop_params", + "status": 400, + "error_text": "The compiled grammar is too large", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": true, + "stripped": false, + "strict_dropped": true + }, + { + "name": "strict_text_on_a_429", + "status": 429, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": false, + "stripped": false, + "strict_dropped": false + }, + { + "name": "unrelated_400_under_drop_params", + "status": 400, + "error_text": "max_tokens: 4096 > 64000", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": false, + "stripped": false, + "strict_dropped": false + }, + { + "name": "thinking_rejection_keeps_strict", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": true, + "stripped": true, + "strict_dropped": false + }, + { + "name": "thinking_rejection_without_drop_params_keeps_strict", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": false, + "retried": true, + "stripped": true, + "strict_dropped": false + }, + { + "name": "one_error_naming_both_without_drop_params", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": false, + "retried": true, + "stripped": true, + "strict_dropped": true + }, + { + "name": "pinned_binding_mismatch_with_a_strict_rejection_under_drop_params", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation. Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "adaptive", + "block_binding": { + "prefix_mismatch_behavior": "error" + } + }, + "drops_unsupported_params": true, + "retried": true, + "stripped": true, + "strict_dropped": true + }, + { + "name": "pinned_binding_mismatch_with_a_strict_rejection_without_drop_params", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation. Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "adaptive", + "block_binding": { + "prefix_mismatch_behavior": "error" + } + }, + "drops_unsupported_params": false, + "retried": false, + "stripped": true, + "strict_dropped": true + } + ] + } +} diff --git a/litellm-rust/crates/llms/tests/messages_http_error_retry.rs b/litellm-rust/crates/llms/tests/messages_http_error_retry.rs new file mode 100644 index 00000000000..ac8ceac4ac6 --- /dev/null +++ b/litellm-rust/crates/llms/tests/messages_http_error_retry.rs @@ -0,0 +1,138 @@ +use litellm_host::interceptors::{RetriedRequest, RetryReason}; +use litellm_llms::{ + anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG, + azure_ai::messages::transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG, + base_llm::messages::{context::MessagesTransformContext, transformation::BaseMessagesConfig}, + bedrock::{ + claude_platform::messages::transformation::BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG, + messages::invoke_transformations::anthropic_claude_transformation::BEDROCK_ANTHROPIC_MESSAGES_CONFIG, + }, + bedrock_mantle::messages::transformation::BEDROCK_MANTLE_MESSAGES_CONFIG, + deepseek::messages::transformation::DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG, + edenai::messages::transformation::EDENAI_MESSAGES_CONFIG, + minimax::messages::transformation::MINIMAX_MESSAGES_CONFIG, + openrouter::messages::transformation::OPENROUTER_MESSAGES_CONFIG, + tencent::messages::transformation::TENCENT_MESSAGES_CONFIG, + vertex_ai::messages::transformation::VERTEX_ANTHROPIC_MESSAGES_CONFIG, +}; +use rstest::rstest; +use serde_json::Value; + +#[rstest] +#[case::anthropic(&ANTHROPIC_MESSAGES_CONFIG)] +#[case::azure_ai(&AZURE_ANTHROPIC_MESSAGES_CONFIG)] +#[case::bedrock_invoke(&BEDROCK_ANTHROPIC_MESSAGES_CONFIG)] +#[case::bedrock_claude_platform(&BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG)] +#[case::bedrock_mantle(&BEDROCK_MANTLE_MESSAGES_CONFIG)] +#[case::deepseek(&DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG)] +#[case::edenai(&EDENAI_MESSAGES_CONFIG)] +#[case::minimax(&MINIMAX_MESSAGES_CONFIG)] +#[case::openrouter(&OPENROUTER_MESSAGES_CONFIG)] +#[case::tencent(&TENCENT_MESSAGES_CONFIG)] +#[case::vertex_ai(&VERTEX_ANTHROPIC_MESSAGES_CONFIG)] +fn every_messages_provider_retries_like_the_python_base_config( + #[case] config: &dyn BaseMessagesConfig, +) { + let fixture: Value = serde_json::from_str(include_str!( + "fixtures/anthropic_messages_http_error_retry.json" + )) + .unwrap(); + let table = &fixture["http_error_retry"]; + let cases = table["cases"].as_array().unwrap(); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let mut body = table["request"].clone(); + body["thinking"] = case["thinking"].clone(); + let status = u16::try_from(case["status"].as_u64().unwrap()).unwrap(); + let expected = case["retried"].as_bool().unwrap().then(|| { + if case["stripped"].as_bool().unwrap() { + RetriedRequest { + reasons: vec![RetryReason::ThinkingSignature], + body: table["stripped_request"].clone(), + } + } else { + RetriedRequest { + reasons: Vec::new(), + body: body.clone(), + } + } + }); + config.retry_after_http_error( + status, + case["error_text"].as_str().unwrap(), + &body, + &MessagesTransformContext::default(), + ) != expected + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); +} + +#[rstest] +#[case::anthropic(&ANTHROPIC_MESSAGES_CONFIG)] +#[case::azure_ai(&AZURE_ANTHROPIC_MESSAGES_CONFIG)] +#[case::bedrock_invoke(&BEDROCK_ANTHROPIC_MESSAGES_CONFIG)] +#[case::bedrock_claude_platform(&BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG)] +#[case::bedrock_mantle(&BEDROCK_MANTLE_MESSAGES_CONFIG)] +#[case::deepseek(&DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG)] +#[case::edenai(&EDENAI_MESSAGES_CONFIG)] +#[case::minimax(&MINIMAX_MESSAGES_CONFIG)] +#[case::openrouter(&OPENROUTER_MESSAGES_CONFIG)] +#[case::tencent(&TENCENT_MESSAGES_CONFIG)] +#[case::vertex_ai(&VERTEX_ANTHROPIC_MESSAGES_CONFIG)] +fn every_messages_provider_drops_rejected_strict_tools_like_the_python_base_config( + #[case] config: &dyn BaseMessagesConfig, +) { + let fixture: Value = serde_json::from_str(include_str!( + "fixtures/anthropic_messages_http_error_retry.json" + )) + .unwrap(); + let replay = &fixture["http_error_retry"]; + let table = &fixture["strict_tools_retry"]; + let cases = table["cases"].as_array().unwrap(); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let mut body = replay["request"].clone(); + body["thinking"] = case["thinking"].clone(); + body["tools"] = table["tools"].clone(); + let status = u16::try_from(case["status"].as_u64().unwrap()).unwrap(); + let expected = case["retried"].as_bool().unwrap().then(|| { + let mut reasons = Vec::new(); + let mut expected = if case["stripped"].as_bool().unwrap() { + reasons.push(RetryReason::ThinkingSignature); + replay["stripped_request"].clone() + } else { + body.clone() + }; + expected["tools"] = if case["strict_dropped"].as_bool().unwrap() { + reasons.push(RetryReason::StrictTools { + dropped: usize::try_from(table["dropped"].as_u64().unwrap()).unwrap(), + }); + table["tools_without_strict"].clone() + } else { + table["tools"].clone() + }; + RetriedRequest { + reasons, + body: expected, + } + }); + config.retry_after_http_error( + status, + case["error_text"].as_str().unwrap(), + &body, + &MessagesTransformContext { + drops_unsupported_params: case["drops_unsupported_params"].as_bool().unwrap(), + ..MessagesTransformContext::default() + }, + ) != expected + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); +} diff --git a/litellm-rust/crates/model-catalog-refresh/AGENTS.md b/litellm-rust/crates/model-catalog-refresh/AGENTS.md new file mode 100644 index 00000000000..96228716abf --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/AGENTS.md @@ -0,0 +1,9 @@ +# Model catalog refresh + +- `fetch` fetches one document over http(s) or `s3://` and never parses it +- Only 429, 500, 502, 503, 504 and transport errors are retried. S3 errors are final +- `Failed.retryable` means only the attempt limit ended the fetch +- A server's Retry-After overrides the backoff, capped at `max_delay` +- `Start::After(failed)` resumes the backoff schedule where another caller's first attempt left off +- Error reasons use the URL with userinfo, query and fragment stripped, since presigned URLs carry credentials +- Bodies over `MAX_BODY_BYTES` are rejected diff --git a/litellm-rust/crates/model-catalog-refresh/Cargo.toml b/litellm-rust/crates/model-catalog-refresh/Cargo.toml index 27219042c6c..76f4a9fb969 100644 --- a/litellm-rust/crates/model-catalog-refresh/Cargo.toml +++ b/litellm-rust/crates/model-catalog-refresh/Cargo.toml @@ -6,14 +6,19 @@ license.workspace = true repository.workspace = true [dependencies] +backon.workspace = true +bytes.workspace = true +litellm-error.workspace = true litellm-http.workspace = true -litellm-model-catalog.workspace = true +litellm-s3.workspace = true +litellm-tracing.workspace = true +reqwest.workspace = true thiserror.workspace = true +tokio = { workspace = true, features = ["time"] } url.workspace = true [dev-dependencies] -litellm-error.workspace = true +aws-credential-types = "1.3.0" litellm-http = { workspace = true, features = ["test-support"] } -reqwest.workspace = true rstest.workspace = true -tokio.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } diff --git a/litellm-rust/crates/model-catalog-refresh/src/attempt.rs b/litellm-rust/crates/model-catalog-refresh/src/attempt.rs new file mode 100644 index 00000000000..45d80e36a18 --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/src/attempt.rs @@ -0,0 +1,153 @@ +use std::time::Duration; + +use bytes::Bytes; + +use litellm_http::Client; +use litellm_s3::S3Uri; +use litellm_s3::sdk::config::retry::RetryConfig; +use litellm_s3::sdk::config::timeout::TimeoutConfig; +use reqwest::header::{ETAG, HeaderMap, RETRY_AFTER}; +use url::Url; + +use crate::{Error, RETRYABLE_STATUSES, S3Access}; + +pub(crate) const MAX_BODY_BYTES: u64 = 100 * 1024 * 1024; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Fetched { + pub body: Vec, + pub etag: Option, +} + +#[derive(Debug)] +pub(crate) enum Cause { + Status(u16), + Transport(Error), +} + +#[derive(Debug)] +pub(crate) struct Rejected { + pub(crate) cause: Cause, + pub(crate) retryable: bool, + pub(crate) retry_after: Option, +} + +impl Rejected { + pub(crate) fn fatal(error: Error) -> Self { + Self { + cause: Cause::Transport(error), + retryable: false, + retry_after: None, + } + } +} + +/// Any status below 400 is the document +pub(crate) async fn http( + client: &Client, + url: Url, + timeout: Duration, +) -> Result { + let transport = |error: litellm_http::Error| Rejected { + cause: Cause::Transport(Error::from(error)), + retryable: true, + retry_after: None, + }; + let mut response = tokio::time::timeout(timeout, client.get(url).send()) + .await + .map_err(litellm_http::Error::read_timeout) + .and_then(|sent| sent.map_err(litellm_http::Error::from)) + .map_err(transport)?; + let status = response.status().as_u16(); + let etag = etag(response.headers()); + let retry_after = response + .headers() + .get(RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.trim().parse().ok()) + .map(Duration::from_secs); + if status >= 400 { + return Err(Rejected { + cause: Cause::Status(status), + retryable: RETRYABLE_STATUSES.contains(&status), + retry_after, + }); + } + if response + .content_length() + .is_some_and(|length| length > MAX_BODY_BYTES) + { + return Err(Rejected::fatal(Error::body_too_large(MAX_BODY_BYTES))); + } + let mut body = Vec::new(); + while let Some(chunk) = tokio::time::timeout(timeout, response.chunk()) + .await + .map_err(litellm_http::Error::body_read_timeout) + .and_then(|read| read.map_err(litellm_http::Error::from)) + .map_err(transport)? + { + append(&mut body, &chunk).map_err(Rejected::fatal)?; + } + Ok(Fetched { body, etag }) +} + +/// S3 failures are final: the SDK already classifies its own errors and a missing or denied object stays so +pub(crate) async fn s3( + client: &Client, + s3: &S3Access, + object: &S3Uri, + timeout: Duration, +) -> Result { + let fetched = async { + let config = match s3 { + S3Access::Environment { region } => { + litellm_s3::environment_config(client.clone(), region.clone()).await + } + S3Access::Config(config) => config.to_builder(), + } + .retry_config(RetryConfig::disabled()) + .timeout_config( + TimeoutConfig::builder() + .operation_attempt_timeout(timeout) + .build(), + ) + .build(); + let output = litellm_s3::sdk::Client::from_conf(config) + .get_object() + .bucket(&object.bucket) + .key(&object.key) + .send() + .await?; + if output + .content_length() + .is_some_and(|length| length.unsigned_abs() > MAX_BODY_BYTES) + { + return Err(Error::body_too_large(MAX_BODY_BYTES)); + } + let etag = output.e_tag().map(str::to_owned); + let mut stream = output.body; + let mut body = Vec::new(); + while let Some(chunk) = tokio::time::timeout(timeout, stream.try_next()).await?? { + append(&mut body, &chunk)?; + } + Ok::<_, Error>(Fetched { body, etag }) + }; + fetched.await.map_err(Rejected::fatal) +} + +fn append(body: &mut Vec, chunk: &Bytes) -> Result<(), Error> { + if (body.len() + chunk.len()) as u64 > MAX_BODY_BYTES { + return Err(Error::body_too_large(MAX_BODY_BYTES)); + } + body.extend_from_slice(chunk); + Ok(()) +} + +fn etag(headers: &HeaderMap) -> Option { + let values: Vec<&str> = headers + .get_all(ETAG) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect(); + (!values.is_empty()).then(|| values.join(", ")) +} diff --git a/litellm-rust/crates/model-catalog-refresh/src/error.rs b/litellm-rust/crates/model-catalog-refresh/src/error.rs index 656f32370cb..c79daeb4777 100644 --- a/litellm-rust/crates/model-catalog-refresh/src/error.rs +++ b/litellm-rust/crates/model-catalog-refresh/src/error.rs @@ -1,6 +1,9 @@ -use std::io; - +use litellm_s3::sdk::config::http::HttpResponse; +use litellm_s3::sdk::error::SdkError; +use litellm_s3::sdk::operation::get_object::GetObjectError; +use litellm_s3::sdk::primitives::ByteStreamError; use thiserror::Error; +use tokio::time::error::Elapsed; #[derive(Debug, Error)] #[error(transparent)] @@ -10,18 +13,48 @@ pub struct Error(Box); enum Repr { #[error(transparent)] Http(#[from] litellm_http::Error), - #[error("could not read the catalog")] - Read(#[source] io::Error), #[error("invalid catalog URL")] InvalidUrl(#[source] url::ParseError), #[error("unsupported catalog URL scheme {scheme}")] UnsupportedScheme { scheme: String }, + #[error("invalid S3 URL")] + S3Uri(#[source] litellm_s3::Error), + #[error("S3 GetObject failed")] + S3GetObject(#[source] Box>), + #[error("S3 GetObject body read failed")] + S3Body(#[source] ByteStreamError), + #[error("S3 GetObject timed out")] + S3Timeout(#[source] Elapsed), + #[error("response body is over the {limit} byte limit")] + BodyTooLarge { limit: u64 }, } impl Error { pub fn unsupported_scheme(scheme: String) -> Self { Self(Box::new(Repr::UnsupportedScheme { scheme })) } + + pub(crate) fn body_too_large(limit: u64) -> Self { + Self(Box::new(Repr::BodyTooLarge { limit })) + } +} + +impl From> for Error { + fn from(cause: SdkError) -> Self { + Self(Box::new(Repr::S3GetObject(Box::new(cause)))) + } +} + +impl From for Error { + fn from(cause: ByteStreamError) -> Self { + Self(Box::new(Repr::S3Body(cause))) + } +} + +impl From for Error { + fn from(cause: Elapsed) -> Self { + Self(Box::new(Repr::S3Timeout(cause))) + } } impl From for Error { @@ -30,9 +63,9 @@ impl From for Error { } } -impl From for Error { - fn from(cause: io::Error) -> Self { - Self(Box::new(Repr::Read(cause))) +impl From for Error { + fn from(cause: litellm_s3::Error) -> Self { + Self(Box::new(Repr::S3Uri(cause))) } } @@ -44,29 +77,18 @@ impl From for Error { #[cfg(test)] mod tests { + use std::io; + use litellm_error::{Report, find}; use rstest::rstest; use super::*; - use crate::{DoneReason, Reply, Step}; #[rstest] - fn exhausted_transport_keeps_its_typed_cause() { - let reply = Reply::Transport { - error: io::Error::from(io::ErrorKind::ConnectionReset).into(), - }; - let Reply::Transport { error } = reply else { - panic!("transport failure") - }; - let step = Step::Done { - reason: DoneReason::Exhausted(error), - }; - let Step::Done { - reason: DoneReason::Exhausted(error), - } = step - else { - panic!("exhausted attempts") - }; + fn an_s3_failure_keeps_its_typed_cause_once() { + let error = Error::from(SdkError::::timeout_error( + io::Error::from(io::ErrorKind::ConnectionReset), + )); let cause = find::(&error).expect("typed I/O cause"); assert_eq!(cause.kind(), io::ErrorKind::ConnectionReset); assert_eq!( diff --git a/litellm-rust/crates/model-catalog-refresh/src/fetch.rs b/litellm-rust/crates/model-catalog-refresh/src/fetch.rs new file mode 100644 index 00000000000..bd58827b546 --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/src/fetch.rs @@ -0,0 +1,122 @@ +use std::borrow::Cow; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::Duration; + +use backon::Retryable; +use litellm_error::Report; +use litellm_http::Client; +use url::Url; + +use crate::attempt::{self, Cause, Fetched, Rejected}; +use crate::{RetryPolicy, Source}; + +/// How an `s3://` fetch reaches S3 +#[derive(Clone, Debug)] +pub enum S3Access { + /// What an AWS SDK finds in the environment and shared config files, loaded on each `s3://` attempt, with + /// `region` winning over the region it finds + Environment { region: Option }, + /// A config the caller built with its region, endpoint and credentials + Config(Box), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Failed { + pub reason: String, + /// Another attempt could succeed. Only the attempt limit ended the fetch + pub retryable: bool, + pub retry_after: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Start { + Fresh, + /// Continues a fetch whose first attempt failed retryably elsewhere, waiting before its next attempt + After(Failed), +} + +pub async fn fetch( + client: &Client, + s3: &S3Access, + url: &str, + policy: RetryPolicy, + start: Start, +) -> Result { + let shown = redacted(url); + let source = + Source::parse(url).map_err(|error| Failed::of(&shown, &Rejected::fatal(error), 1))?; + let mut backoff = policy.backoff(); + let made = AtomicU32::new(0); + if let Start::After(failed) = start { + let Some(wait) = backoff.next() else { + return Err(failed); + }; + tokio::time::sleep(policy.delay(wait, failed.retry_after)).await; + made.store(1, Ordering::Relaxed); + } + let attempt = || { + made.fetch_add(1, Ordering::Relaxed); + async { + match &source { + Source::Http(http) => attempt::http(client, http.clone(), policy.timeout).await, + Source::S3(object) => attempt::s3(client, s3, object, policy.timeout).await, + } + } + }; + attempt + .retry(backoff) + .sleep(tokio::time::sleep) + .when(|rejected| rejected.retryable) + .adjust(|rejected, wait| wait.map(|wait| policy.delay(wait, rejected.retry_after))) + .notify(|rejected, wait| { + litellm_tracing::warn!( + "LiteLLM: model cost map fetch attempt {}/{} failed ({}); retrying in {:.1}s", + made.load(Ordering::Relaxed), + policy.max_attempts, + reason(&shown, &rejected.cause), + wait.as_secs_f64(), + ); + }) + .await + .map_err(|rejected| Failed::of(&shown, &rejected, made.load(Ordering::Relaxed))) +} + +impl Failed { + fn of(url: &str, rejected: &Rejected, attempts: u32) -> Self { + let reason = reason(url, &rejected.cause); + Self { + reason: match rejected.retryable && attempts > 1 { + true => format!("{reason} (after {attempts} attempts)"), + false => reason, + }, + retryable: rejected.retryable, + retry_after: rejected.retry_after, + } + } +} + +fn reason(url: &str, cause: &Cause) -> String { + match cause { + Cause::Status(status) => format!("HTTP {status} from {url}"), + Cause::Transport(error) => format!("error fetching {url}: {}", Report::new(error)), + } +} + +/// Drops userinfo, query and fragment, where presigned URLs and tokens carry credentials +fn redacted(url: &str) -> Cow<'_, str> { + let Ok(mut parsed) = Url::parse(url) else { + return Cow::Borrowed(url); + }; + if parsed.username().is_empty() + && parsed.password().is_none() + && parsed.query().is_none() + && parsed.fragment().is_none() + { + return Cow::Borrowed(url); + } + let _ = parsed.set_username(""); + let _ = parsed.set_password(None); + parsed.set_query(None); + parsed.set_fragment(None); + Cow::Owned(parsed.into()) +} diff --git a/litellm-rust/crates/model-catalog-refresh/src/lib.rs b/litellm-rust/crates/model-catalog-refresh/src/lib.rs index 006c4887fad..1742a669483 100644 --- a/litellm-rust/crates/model-catalog-refresh/src/lib.rs +++ b/litellm-rust/crates/model-catalog-refresh/src/lib.rs @@ -1,9 +1,11 @@ +mod attempt; mod error; +mod fetch; mod policy; mod source; -mod step; +pub use attempt::Fetched; pub use error::Error; +pub use fetch::*; pub use policy::*; pub use source::*; -pub use step::*; diff --git a/litellm-rust/crates/model-catalog-refresh/src/policy.rs b/litellm-rust/crates/model-catalog-refresh/src/policy.rs index bc32e40408e..1cdb737ecd8 100644 --- a/litellm-rust/crates/model-catalog-refresh/src/policy.rs +++ b/litellm-rust/crates/model-catalog-refresh/src/policy.rs @@ -1,20 +1,76 @@ use std::time::Duration; -use litellm_model_catalog::IntegrityLimits; +use backon::{BackoffBuilder, ExponentialBackoff, ExponentialBuilder}; -/// Main's defaults, 3 attempts, 30 s max wait, 5 s timeout, backoff base 2.0, jitter 0.0 to 1.0 s -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct RetryPolicy { pub max_attempts: u32, - pub max_wait: Duration, pub timeout: Duration, - pub backoff_base_seconds: f64, - pub jitter_min_seconds: f64, - pub jitter_max_seconds: f64, - /// Main's 50 model floor and 0.5 shrink ratio - pub limits: IntegrityLimits, + pub min_delay: Duration, + pub max_delay: Duration, } +pub const RETRYABLE_STATUSES: [u16; 5] = [429, 500, 502, 503, 504]; + /// Main's default `LITELLM_MODEL_COST_MAP_URL` pub const DEFAULT_REGISTRY_URL: &str = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"; + +impl RetryPolicy { + pub(crate) fn backoff(&self) -> ExponentialBackoff { + ExponentialBuilder::new() + .with_min_delay(self.min_delay) + .with_max_delay(self.max_delay) + .with_max_times(self.max_attempts.saturating_sub(1) as usize) + .with_jitter() + .build() + } + + pub(crate) fn delay(&self, backoff: Duration, retry_after: Option) -> Duration { + retry_after.map_or(backoff, |server| server.min(self.max_delay)) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + const POLICY: RetryPolicy = RetryPolicy { + max_attempts: 3, + timeout: Duration::from_secs(5), + min_delay: Duration::from_secs(1), + max_delay: Duration::from_secs(30), + }; + + #[rstest] + #[case::backoff_without_retry_after(None, 4)] + #[case::retry_after_wins(Some(7), 7)] + #[case::retry_after_zero(Some(0), 0)] + #[case::retry_after_capped(Some(120), 30)] + fn the_server_retry_after_overrides_the_backoff_up_to_max_delay( + #[case] retry_after: Option, + #[case] seconds: u64, + ) { + assert_eq!( + POLICY.delay(Duration::from_secs(4), retry_after.map(Duration::from_secs)), + Duration::from_secs(seconds) + ); + } + + #[rstest] + #[case::one_attempt(1, 0)] + #[case::three_attempts(3, 2)] + #[case::zero_attempts(0, 0)] + fn the_backoff_allows_one_retry_fewer_than_the_attempts( + #[case] max_attempts: u32, + #[case] retries: usize, + ) { + let policy = RetryPolicy { + max_attempts, + ..POLICY + }; + assert_eq!(policy.backoff().count(), retries); + } +} diff --git a/litellm-rust/crates/model-catalog-refresh/src/source.rs b/litellm-rust/crates/model-catalog-refresh/src/source.rs index 713e0386941..28849ba2841 100644 --- a/litellm-rust/crates/model-catalog-refresh/src/source.rs +++ b/litellm-rust/crates/model-catalog-refresh/src/source.rs @@ -1,17 +1,59 @@ -use std::path::PathBuf; - +use litellm_s3::S3Uri; use url::Url; +use crate::Error; + /// Where a fetched catalog comes from #[derive(Clone, Debug, PartialEq, Eq)] pub enum Source { Http(Url), - File(PathBuf), - S3(S3Object), + S3(S3Uri), } -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct S3Object { - pub bucket: String, - pub key: String, +impl Source { + pub fn parse(url: &str) -> Result { + let parsed = Url::parse(url)?; + match parsed.scheme() { + "http" | "https" => Ok(Self::Http(parsed)), + "s3" => Ok(Self::S3(S3Uri::try_from(url)?)), + scheme => Err(Error::unsupported_scheme(scheme.to_owned())), + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::https("https://example.com/cost.json")] + #[case::http("http://127.0.0.1:8000/cost.json")] + fn http_urls_are_fetched_over_http(#[case] url: &str) { + assert_eq!( + Source::parse(url).unwrap(), + Source::Http(Url::parse(url).unwrap()) + ); + } + + #[rstest] + fn s3_urls_name_a_bucket_and_key() { + assert_eq!( + Source::parse("s3://bucket/prices/cost.json").unwrap(), + Source::S3(S3Uri { + bucket: "bucket".into(), + key: "prices/cost.json".into(), + }) + ); + } + + #[rstest] + #[case::s3_without_a_key("s3://bucket/")] + #[case::unsupported("ftp://example.com/cost.json")] + #[case::file("file:///tmp/cost.json")] + #[case::not_a_url("not a url")] + fn other_urls_are_refused(#[case] url: &str) { + assert!(Source::parse(url).is_err()); + } } diff --git a/litellm-rust/crates/model-catalog-refresh/src/step.rs b/litellm-rust/crates/model-catalog-refresh/src/step.rs deleted file mode 100644 index 822d1807916..00000000000 --- a/litellm-rust/crates/model-catalog-refresh/src/step.rs +++ /dev/null @@ -1,66 +0,0 @@ -use std::time::Duration; - -use litellm_model_catalog::Catalog; - -use crate::Error; -use crate::source::Source; - -/// One attempt. Where to fetch from is the `Source`, so the request carries no URL -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct FetchRequest { - pub if_none_match: Option, - pub timeout: Duration, -} - -#[derive(Debug)] -pub enum Reply { - /// An HTTP, S3 or file answer. The body is read only on 200 - Response { - status: u16, - etag: Option, - retry_after: Option, - body: Option>, - }, - /// No answer reached us - Transport { error: Error }, -} - -/// What the fetch state machine asks its driver to do next -#[derive(Debug)] -pub enum Step { - Fetch(FetchRequest), - Sleep(Duration), - /// A parsed document to install, with its rejected row errors - Adopt { - catalog: Box, - installed: Installed, - errors: Vec, - }, - /// 304, a final status, or retries exhausted - Done { - reason: DoneReason, - }, -} - -#[derive(Debug)] -pub enum DoneReason { - NotModified, - Status(u16), - Exhausted(Error), -} - -/// What is installed now, the input to the next revalidation -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Installed { - pub origin: Origin, - pub etag: Option, - /// `git_blob_id` of the installed bytes - pub revision: String, - pub model_count: usize, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Origin { - Bundled, - Remote { source: Source }, -} diff --git a/litellm-rust/crates/model-catalog-refresh/tests/fetch.rs b/litellm-rust/crates/model-catalog-refresh/tests/fetch.rs new file mode 100644 index 00000000000..2a4ec996bd4 --- /dev/null +++ b/litellm-rust/crates/model-catalog-refresh/tests/fetch.rs @@ -0,0 +1,541 @@ +use std::io::{BufRead, BufReader, Write}; +use std::net::TcpListener; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use aws_credential_types::Credentials; +use aws_credential_types::provider::error::CredentialsError; +use aws_credential_types::provider::{ProvideCredentials, future}; +use litellm_http::Client; +use litellm_model_catalog_refresh::{Failed, Fetched, RetryPolicy, S3Access, Start, fetch}; +use rstest::rstest; + +const POLICY: RetryPolicy = RetryPolicy { + max_attempts: 3, + timeout: Duration::from_secs(5), + min_delay: Duration::from_millis(10), + max_delay: Duration::from_millis(50), +}; + +const OK: &str = + "HTTP/1.1 200 OK\r\nETag: \"v1\"\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}"; +const UNAVAILABLE: &str = "HTTP/1.1 503 Service Unavailable\r\nRetry-After: 0\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; +const NOT_FOUND: &str = "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + +const SLOW_BODY: &[u8] = b"{ }"; + +#[derive(Clone, Copy)] +enum Answer { + Raw(&'static str), + Stall, + StallBody, + SlowBody, + Oversized, +} + +/// A scripted upstream: the n-th request gets the n-th answer, repeating the last, and every request head is kept +struct Upstream { + base: String, + heads: Arc>>, +} + +impl Upstream { + fn serve(answers: Vec) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let heads = Arc::new(Mutex::new(Vec::new())); + let recorded = Arc::clone(&heads); + std::thread::spawn(move || { + for (index, stream) in listener.incoming().enumerate() { + let answer = answers[index.min(answers.len() - 1)]; + let recorded = Arc::clone(&recorded); + std::thread::spawn(move || { + let mut stream = stream.unwrap(); + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut head = String::new(); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap() <= 2 { + break; + } + head.push_str(&line); + } + recorded.lock().unwrap().push(head); + match answer { + Answer::Raw(raw) => stream.write_all(raw.as_bytes()).unwrap(), + Answer::Stall => std::thread::sleep(Duration::from_secs(5)), + Answer::Oversized => { + let chunk = vec![b' '; 1024 * 1024]; + let written = stream + .write_all(b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n") + .and_then(|()| { + (0..=100).try_for_each(|_| stream.write_all(&chunk)) + }); + drop(written); + } + Answer::StallBody => { + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\n{") + .unwrap(); + std::thread::sleep(Duration::from_secs(5)); + } + Answer::SlowBody => { + stream.set_nodelay(true).unwrap(); + stream + .write_all( + b"HTTP/1.1 200 OK\r\nETag: \"v1\"\r\nContent-Length: 8\r\nConnection: close\r\n\r\n", + ) + .unwrap(); + for byte in SLOW_BODY { + std::thread::sleep(Duration::from_millis(60)); + stream.write_all(&[*byte]).unwrap(); + } + } + } + }); + } + }); + Self { base, heads } + } + + fn url(&self) -> String { + format!("{}/cost.json", self.base) + } + + fn heads(&self) -> Vec { + self.heads.lock().unwrap().clone() + } +} + +fn header<'a>(head: &'a str, name: &str) -> Option<&'a str> { + head.lines().find_map(|line| { + let (key, value) = line.split_once(':')?; + key.eq_ignore_ascii_case(name).then(|| value.trim()) + }) +} + +#[derive(Clone, Debug)] +struct NoCredentials; + +impl ProvideCredentials for NoCredentials { + fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a> + where + Self: 'a, + { + future::ProvideCredentials::ready(Err(CredentialsError::not_loaded("no credentials"))) + } +} + +fn keys() -> Credentials { + Credentials::new( + "AKIDMODELCOST", + "model-cost-secret", + None, + None, + "model-cost-test", + ) +} + +async fn fetched(url: &str, policy: RetryPolicy, start: Start) -> Result { + fetch( + &Client::for_test(), + &S3Access::Environment { region: None }, + url, + policy, + start, + ) + .await +} + +#[rstest] +#[case::every_etag_joined( + "HTTP/1.1 200 OK\r\nETag: \"a\"\r\nETag: \"b\"\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}", + Some("\"a\", \"b\"") +)] +#[case::no_etag( + "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}", + None +)] +#[tokio::test] +async fn a_200_returns_the_body_and_its_etags( + #[case] response: &'static str, + #[case] etag: Option<&str>, +) { + let upstream = Upstream::serve(vec![Answer::Raw(response)]); + assert_eq!( + fetched(&upstream.url(), POLICY, Start::Fresh).await, + Ok(Fetched { + body: b"{}".to_vec(), + etag: etag.map(str::to_owned), + }) + ); + assert_eq!(upstream.heads().len(), 1); +} + +#[rstest] +#[tokio::test(flavor = "multi_thread")] +async fn a_fetch_runs_as_a_task_on_a_multi_thread_runtime() { + let upstream = Upstream::serve(vec![Answer::Raw(OK)]); + let url = upstream.url(); + let fetched = tokio::spawn(async move { + fetch( + &Client::for_test(), + &S3Access::Environment { region: None }, + &url, + POLICY, + Start::Fresh, + ) + .await + }) + .await + .unwrap(); + assert!(fetched.is_ok()); +} + +#[rstest] +#[tokio::test] +async fn a_redirect_is_not_followed_and_its_body_is_the_document() { + let upstream = Upstream::serve(vec![ + Answer::Raw( + "HTTP/1.1 301 Moved Permanently\r\nLocation: /elsewhere.json\r\nContent-Length: 7\r\nConnection: close\r\n\r\n{\"a\":1}", + ), + Answer::Raw(OK), + ]); + assert_eq!( + fetched(&upstream.url(), POLICY, Start::Fresh).await, + Ok(Fetched { + body: b"{\"a\":1}".to_vec(), + etag: None, + }) + ); + assert_eq!(upstream.heads().len(), 1); +} + +#[rstest] +#[tokio::test] +async fn a_retry_after_shorter_than_the_backoff_is_honored() { + let upstream = Upstream::serve(vec![Answer::Raw(UNAVAILABLE), Answer::Raw(OK)]); + let policy = RetryPolicy { + min_delay: Duration::from_secs(10), + max_delay: Duration::from_secs(10), + ..POLICY + }; + let fetched = tokio::time::timeout( + Duration::from_secs(2), + fetched(&upstream.url(), policy, Start::Fresh), + ) + .await + .expect("Retry-After: 0 replaces the 10s backoff"); + assert!(fetched.is_ok(), "{fetched:?}"); + assert_eq!(upstream.heads().len(), 2); +} + +#[rstest] +#[tokio::test] +async fn retryable_failures_stop_at_the_attempt_limit() { + let upstream = Upstream::serve(vec![Answer::Raw(UNAVAILABLE)]); + let url = upstream.url(); + assert_eq!( + fetched(&url, POLICY, Start::Fresh).await, + Err(Failed { + reason: format!("HTTP 503 from {url} (after 3 attempts)"), + retryable: true, + retry_after: Some(Duration::ZERO), + }) + ); + assert_eq!(upstream.heads().len(), 3); +} + +#[rstest] +#[tokio::test] +async fn a_404_fails_on_the_first_attempt() { + let upstream = Upstream::serve(vec![Answer::Raw(NOT_FOUND)]); + let url = upstream.url(); + assert_eq!( + fetched(&url, POLICY, Start::Fresh).await, + Err(Failed { + reason: format!("HTTP 404 from {url}"), + retryable: false, + retry_after: None, + }) + ); + assert_eq!(upstream.heads().len(), 1); +} + +#[rstest] +#[tokio::test] +async fn a_stalled_response_times_out_and_is_retried() { + let upstream = Upstream::serve(vec![Answer::Stall]); + let url = upstream.url(); + let policy = RetryPolicy { + max_attempts: 2, + timeout: Duration::from_millis(100), + ..POLICY + }; + let failed = fetched(&url, policy, Start::Fresh).await.unwrap_err(); + assert!( + failed + .reason + .starts_with(&format!("error fetching {url}: ")) + && failed.reason.ends_with(" (after 2 attempts)"), + "{}", + failed.reason + ); + assert!(failed.retryable); + assert_eq!(upstream.heads().len(), 2); +} + +#[rstest] +#[tokio::test] +async fn a_resumed_fetch_makes_only_the_attempts_left() { + let upstream = Upstream::serve(vec![Answer::Raw(UNAVAILABLE)]); + let url = upstream.url(); + let first = Failed { + reason: "first".into(), + retryable: true, + retry_after: Some(Duration::ZERO), + }; + let failed = fetched(&url, POLICY, Start::After(first)) + .await + .unwrap_err(); + assert_eq!( + failed.reason, + format!("HTTP 503 from {url} (after 3 attempts)") + ); + assert_eq!(upstream.heads().len(), 2); +} + +#[rstest] +#[tokio::test] +async fn a_resumed_fetch_with_no_attempts_left_returns_the_failure_it_resumed() { + let upstream = Upstream::serve(vec![Answer::Raw(OK)]); + let first = Failed { + reason: "first".into(), + retryable: true, + retry_after: None, + }; + let policy = RetryPolicy { + max_attempts: 1, + ..POLICY + }; + assert_eq!( + fetched(&upstream.url(), policy, Start::After(first.clone())).await, + Err(first) + ); + assert!(upstream.heads().is_empty()); +} + +#[rstest] +#[tokio::test] +async fn a_failure_reason_drops_url_credentials() { + let upstream = Upstream::serve(vec![Answer::Raw(NOT_FOUND)]); + let address = upstream.base.trim_start_matches("http://"); + let url = format!("http://user:hunter2@{address}/cost.json?X-Amz-Signature=sig-secret#frag"); + let failed = fetched(&url, POLICY, Start::Fresh).await.unwrap_err(); + assert_eq!(failed.reason, format!("HTTP 404 from {}", upstream.url())); +} + +#[rstest] +#[case::unsupported_scheme("ftp://example.com/cost.json")] +#[case::not_a_url("not a url")] +#[case::s3_without_a_key("s3://prices")] +#[tokio::test] +async fn an_unfetchable_url_fails_without_a_request(#[case] url: &str) { + let failed = fetched(url, POLICY, Start::Fresh).await.unwrap_err(); + assert!( + failed + .reason + .starts_with(&format!("error fetching {url}: ")), + "{}", + failed.reason + ); + assert!(!failed.retryable); +} + +fn s3(upstream: &Upstream, credentials: impl ProvideCredentials + 'static) -> S3Access { + S3Access::Config(Box::new( + litellm_s3::config( + "us-west-2".into(), + Some(&upstream.base), + Client::for_test(), + credentials, + ) + .build(), + )) +} + +async fn fetched_from_s3( + url: &str, + upstream: &Upstream, + credentials: impl ProvideCredentials + 'static, +) -> Result { + fetch( + &Client::for_test(), + &s3(upstream, credentials), + url, + POLICY, + Start::Fresh, + ) + .await +} + +#[rstest] +#[tokio::test] +async fn an_s3_fetch_sends_a_signed_get_object_to_the_endpoint() { + let upstream = Upstream::serve(vec![Answer::Raw(OK)]); + assert_eq!( + fetched_from_s3("s3://prices/maps/cost.json", &upstream, keys()).await, + Ok(Fetched { + body: b"{}".to_vec(), + etag: Some("\"v1\"".into()), + }) + ); + let heads = upstream.heads(); + let [head] = heads.as_slice() else { + panic!("expected one request, got {heads:?}"); + }; + assert!(head.starts_with("GET /prices/maps/cost.json"), "{head}"); + let authorization = header(head, "authorization").unwrap_or_default(); + assert!( + authorization.starts_with("AWS4-HMAC-SHA256 Credential=AKIDMODELCOST/") + && authorization.contains("/us-west-2/s3/aws4_request"), + "{authorization}" + ); +} + +#[rstest] +#[tokio::test] +async fn an_s3_body_that_stalls_after_the_headers_times_out() { + let upstream = Upstream::serve(vec![Answer::StallBody]); + let url = "s3://prices/cost.json"; + let policy = RetryPolicy { + timeout: Duration::from_millis(200), + ..POLICY + }; + let failed = tokio::time::timeout( + Duration::from_secs(2), + fetch( + &Client::for_test(), + &s3(&upstream, keys()), + url, + policy, + Start::Fresh, + ), + ) + .await + .expect("the attempt timeout bounds the body read") + .unwrap_err(); + assert!( + failed + .reason + .starts_with(&format!("error fetching {url}: ")), + "{}", + failed.reason + ); + assert!(!failed.retryable); + assert_eq!(upstream.heads().len(), 1); +} + +#[rstest] +#[case::http(|upstream: &Upstream| (S3Access::Environment { region: None }, upstream.url()))] +#[case::s3(|upstream: &Upstream| (s3(upstream, keys()), "s3://prices/cost.json".to_owned()))] +#[tokio::test] +async fn a_body_that_keeps_arriving_finishes_after_the_timeout( + #[case] source: fn(&Upstream) -> (S3Access, String), +) { + let upstream = Upstream::serve(vec![Answer::SlowBody]); + let (access, url) = source(&upstream); + let policy = RetryPolicy { + timeout: Duration::from_millis(300), + ..POLICY + }; + assert_eq!( + fetch(&Client::for_test(), &access, &url, policy, Start::Fresh).await, + Ok(Fetched { + body: SLOW_BODY.to_vec(), + etag: Some("\"v1\"".into()), + }) + ); + assert_eq!(upstream.heads().len(), 1); +} + +#[rstest] +#[case::missing_key(NOT_FOUND)] +#[case::denied("HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")] +#[tokio::test] +async fn an_s3_error_fails_without_a_retry(#[case] response: &'static str) { + let upstream = Upstream::serve(vec![Answer::Raw(response), Answer::Raw(OK)]); + let url = "s3://prices/cost.json"; + let failed = fetched_from_s3(url, &upstream, keys()).await.unwrap_err(); + assert!( + failed + .reason + .starts_with(&format!("error fetching {url}: ")), + "{}", + failed.reason + ); + assert!(!failed.retryable); + assert_eq!(upstream.heads().len(), 1); +} + +#[rstest] +#[tokio::test] +async fn an_s3_fetch_without_credentials_fails_without_a_request() { + let upstream = Upstream::serve(vec![Answer::Raw(OK)]); + let url = "s3://prices/cost.json"; + let failed = fetched_from_s3(url, &upstream, NoCredentials) + .await + .unwrap_err(); + assert!( + failed + .reason + .starts_with(&format!("error fetching {url}: ")), + "{}", + failed.reason + ); + assert!(!failed.retryable); + assert!(upstream.heads().is_empty()); +} + +const DECLARED_OVERSIZED: &str = + "HTTP/1.1 200 OK\r\nContent-Length: 104857601\r\nConnection: close\r\n\r\n"; + +#[rstest] +#[case::http_declared(Answer::Raw(DECLARED_OVERSIZED), false)] +#[case::http_streamed(Answer::Oversized, false)] +#[case::s3_declared(Answer::Raw(DECLARED_OVERSIZED), true)] +#[case::s3_streamed(Answer::Oversized, true)] +#[tokio::test] +async fn a_body_over_100_mib_fails_without_a_retry(#[case] answer: Answer, #[case] s3: bool) { + let upstream = Upstream::serve(vec![answer, Answer::Raw(OK)]); + let failed = match s3 { + true => fetched_from_s3("s3://prices/cost.json", &upstream, keys()).await, + false => fetched(&upstream.url(), POLICY, Start::Fresh).await, + } + .unwrap_err(); + assert!( + failed + .reason + .contains("response body is over the 104857600 byte limit"), + "{}", + failed.reason + ); + assert!(!failed.retryable); + assert_eq!(upstream.heads().len(), 1); +} + +#[rstest] +#[tokio::test] +async fn a_body_of_exactly_100_mib_is_fetched() { + let upstream = Upstream::serve(vec![Answer::Raw(Box::leak( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: 104857600\r\nConnection: close\r\n\r\n{}", + " ".repeat(100 * 1024 * 1024) + ) + .into_boxed_str(), + ))]); + let fetched = fetched(&upstream.url(), POLICY, Start::Fresh) + .await + .unwrap(); + assert_eq!(fetched.body.len(), 100 * 1024 * 1024); +} diff --git a/litellm-rust/crates/model-catalog/AGENTS.md b/litellm-rust/crates/model-catalog/AGENTS.md index abc5d198729..a9c473f2ca3 100644 --- a/litellm-rust/crates/model-catalog/AGENTS.md +++ b/litellm-rust/crates/model-catalog/AGENTS.md @@ -46,7 +46,14 @@ Every price in a row lowers to one `Price { meter, channel, when }` and its `Uni `store/` is the catalog's in-memory store and the only place that indexes model names. Its rules are in [src/store/AGENTS.md](src/store/AGENTS.md) -`layers.rs`, `resolve/`, `model_info.rs` and `register.rs` hold types only. The functions over them land with their first caller +`cost_map.rs` holds a host's inputs, the installed document and the `register_model` overlay in `register.rs`. It serves the document in the key order and alias sharing Python's `litellm.model_cost` has, without the overlay, since where a registration lands is a lookup in the published table. It never holds the live table, which belongs to the host + +- `CostMap` is immutable behind an `Arc`. A new document is a new `CostMap` +- `Registrations` is generic over the host's value type, because Python registrations can hold `datetime` and other values JSON cannot. The host keeps its own objects (`Py` in python-bridge) and does the merging + - Entry order is latest-registration order. `landed_on` records which registered keys landed on each row key +- `write_pass` stops at the first `place` that returns `None` and reports `Pass::Swapped`. Retrying is the host's job + +`resolve/` and `model_info.rs` hold types only. The functions over them land with their first caller Catalog errors use one opaque `Error` with typed causes and JSON Pointer accessors (`path`, `key`, `field`). `Error::rejects` alone decides whether an error fails the document, rejects its row or neither, and `ErrorKind` is the stable name hosts report it by diff --git a/litellm-rust/crates/model-catalog/Cargo.toml b/litellm-rust/crates/model-catalog/Cargo.toml index 63b9ad14768..8cd8ee5760b 100644 --- a/litellm-rust/crates/model-catalog/Cargo.toml +++ b/litellm-rust/crates/model-catalog/Cargo.toml @@ -11,6 +11,7 @@ schema = ["dep:schemars", "litellm-cost/schema"] [dependencies] litellm-cost = { workspace = true, features = ["serde"] } litellm-error.workspace = true +litellm-tracing.workspace = true indexmap = { version = "2.14.0", features = ["serde"] } jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] } @@ -19,6 +20,7 @@ schemars = { workspace = true, optional = true } serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } serde_with.workspace = true +sha1.workspace = true strum.workspace = true thiserror.workspace = true time.workspace = true diff --git a/litellm-rust/crates/model-catalog/src/catalog.rs b/litellm-rust/crates/model-catalog/src/catalog.rs index 98af785b17e..6c11a39b0eb 100644 --- a/litellm-rust/crates/model-catalog/src/catalog.rs +++ b/litellm-rust/crates/model-catalog/src/catalog.rs @@ -12,7 +12,8 @@ use crate::store::{Rows, Store}; use crate::wire::{EntryWire, Fields}; const SAMPLE_SPEC: &str = "sample_spec"; -const FALLBACK_GENERALIZATIONS: &str = "fallback_generalizations"; +/// The top-level key of the rules block, which is never a model row +pub const FALLBACK_GENERALIZATIONS: &str = "fallback_generalizations"; #[derive(Clone, Copy, Debug, PartialEq)] pub struct IntegrityLimits { @@ -111,6 +112,11 @@ impl Rows for CatalogRows { } impl IntegrityLimits { + pub const UNCHECKED: Self = Self { + min_model_count: 0, + min_reference_ratio: 0.0, + }; + pub fn new(min_model_count: usize, min_reference_ratio: f64) -> Option { (0.0..=1.0).contains(&min_reference_ratio).then_some(Self { min_model_count, @@ -142,6 +148,7 @@ impl Catalog { let mut sections = Vec::new(); let mut entries = IndexMap::with_capacity(document.len()); let mut alias_lists = Vec::new(); + let mut pointer_rows = Vec::new(); for (index, (name, value)) in document.into_iter().enumerate() { let fields = match value { Value::Object(fields) if !is_section(&name) => fields, @@ -157,16 +164,34 @@ impl Catalog { continue; } }; - let (model, aliases, row_problems) = read_row(&fields); + let row = read_row(&fields); + let readable = row.problems.is_empty(); problems.extend( - row_problems + row.problems .into_iter() .map(|problem| problem.prepend(&name)), ); - if let Some(aliases) = aliases { + if let Some(target) = row.alias_of { + if readable { + pointer_rows.push((name.clone(), target)); + } + sections.push(Section { + index, + key: name, + value: Value::Object(fields), + }); + continue; + } + if let Some(aliases) = row.aliases { alias_lists.push((name.clone(), aliases)); } - entries.insert(name, ModelEntry { fields, model }); + entries.insert( + name, + ModelEntry { + fields, + model: row.model, + }, + ); } if entries.is_empty() { return Err(Error::new("", Repr::Empty)); @@ -190,6 +215,21 @@ impl Catalog { } } } + for (alias, target) in pointer_rows { + if !entries.contains_key(&target) { + problems.push(Error::from_key(&alias, Repr::AliasTargetMissing { target })); + } else if let Some(taken_by) = aliases.get(&alias) { + problems.push(Error::from_key( + &alias, + Repr::AliasTaken { + alias: alias.clone(), + taken_by: taken_by.clone(), + }, + )); + } else { + aliases.insert(alias, target); + } + } let fallback_generalizations = sections .iter() @@ -242,8 +282,8 @@ impl Catalog { } pub fn row_errors(model: &str, fields: &Fields) -> Vec { - let (_, _, errors) = read_row(fields); - errors + read_row(fields) + .problems .into_iter() .map(|error| error.prepend(model)) .collect() @@ -255,7 +295,13 @@ impl Catalog { (0..self.rows().entries.len() + self.sections.len()).map_while(move |index| match sections .next_if(|section| section.index == index) { - Some(section) => Some((section.key.as_str(), Written::Section(§ion.value))), + Some(section) => Some(( + section.key.as_str(), + match §ion.value { + Value::Object(fields) if !is_section(§ion.key) => Written::Entry(fields), + value => Written::Section(value), + }, + )), None => entries .next() .map(|(name, entry)| (name.as_str(), Written::Entry(&entry.fields))), @@ -378,6 +424,20 @@ impl Catalog { }) } + /// The row under an exact model or alias key, an alias reading its owner's row + pub fn get(&self, key: &str) -> Option<&Fields> { + self.rows().fields(key) + } + + /// A top-level value that is not a model row, such as `sample_spec` + pub fn section(&self, key: &str) -> Option<&Value> { + self.sections + .iter() + .filter(|section| is_section(§ion.key)) + .find(|section| section.key == key) + .map(|section| §ion.value) + } + pub fn model_count(&self) -> usize { self.rows().entries.len() } @@ -406,16 +466,41 @@ impl Catalog { } } -fn read_row(fields: &Fields) -> (Option, Option>, Vec) { +struct RowRead { + model: Option, + aliases: Option>, + alias_of: Option, + problems: Vec, +} + +fn read_row(fields: &Fields) -> RowRead { let wire = EntryWire::read(fields); + if let Some(target) = wire.attributes.alias_of.clone() { + let other_fields = fields + .keys() + .filter(|key| *key != "alias_of") + .map(|key| Error::from_key(key, Repr::AliasRowField)); + return RowRead { + model: None, + aliases: None, + alias_of: Some(target), + problems: wire.problems.into_iter().chain(other_fields).collect(), + }; + } let aliases = wire.attributes.aliases.clone(); let timezone_error = unknown_timezone(&wire); - match Model::try_from(wire) { - Ok(model) => (Some(model), aliases, timezone_error.into_iter().collect()), + let (model, problems) = match Model::try_from(wire) { + Ok(model) => (Some(model), timezone_error.into_iter().collect()), Err(mut errors) => { errors.extend(timezone_error); - (None, aliases, errors) + (None, errors) } + }; + RowRead { + model, + aliases, + alias_of: None, + problems, } } diff --git a/litellm-rust/crates/model-catalog/src/cost_map.rs b/litellm-rust/crates/model-catalog/src/cost_map.rs new file mode 100644 index 00000000000..bec417acae7 --- /dev/null +++ b/litellm-rust/crates/model-catalog/src/cost_map.rs @@ -0,0 +1,130 @@ +use std::sync::{Arc, LazyLock}; + +use indexmap::IndexMap; +use serde_json::Value; +use sha1::{Digest, Sha1}; + +use crate::bundled::BUNDLED_REGISTRY; +use crate::catalog::{Catalog, FALLBACK_GENERALIZATIONS, IntegrityLimits}; +use crate::error::Error; + +/// Which document is installed: the `git_blob_id` of its bytes and the ETag it was served with +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Provenance { + pub revision: String, + pub etag: Option, +} + +/// An installed cost map document. A host builds its live table from it +#[derive(Clone, Debug)] +pub struct CostMap { + base: Arc, + provenance: Provenance, +} + +#[derive(Debug)] +struct Base { + catalog: Catalog, + /// Each owner's names, the owner first, for owners with at least one alias + groups: IndexMap>, +} + +impl Base { + fn new(catalog: Catalog) -> Self { + let groups = catalog.aliases().iter().fold( + IndexMap::>::new(), + |mut groups, (alias, owner)| { + groups + .entry(owner.clone()) + .or_insert_with(|| vec![owner.clone()]) + .push(alias.clone()); + groups + }, + ); + Self { catalog, groups } + } +} + +static BUNDLED: LazyLock = LazyLock::new(|| CostMap { + base: Arc::new(Base::new( + Catalog::parse(BUNDLED_REGISTRY) + .expect("the bundled registry parses") + .value, + )), + provenance: Provenance { + revision: git_blob_id(BUNDLED_REGISTRY), + etag: None, + }, +}); + +impl CostMap { + pub fn bundled() -> Self { + BUNDLED.clone() + } + + /// Reads a fetched document. One that does not parse, or that has fewer models than `limits` allow against the + /// bundled one, is refused. Every alias problem it has is logged + pub fn read(body: &[u8], etag: Option, limits: IntegrityLimits) -> Result { + let loaded = Catalog::parse(body)?; + loaded + .value + .check_size(BUNDLED.base.catalog.model_count(), limits)?; + for problem in loaded.errors.iter().filter(|error| error.is_alias()) { + litellm_tracing::warn!( + "LiteLLM: model cost map alias problem: {}", + litellm_error::Report::new(problem) + ); + } + Ok(Self { + base: Arc::new(Base::new(loaded.value)), + provenance: Provenance { + revision: git_blob_id(body), + etag, + }, + }) + } + + pub fn catalog(&self) -> &Catalog { + &self.base.catalog + } + + pub fn provenance(&self) -> &Provenance { + &self.provenance + } + + /// Every name the row under `key` is served under, the owner first. Empty for a row without aliases + pub fn names(&self, key: &str) -> &[String] { + let owner = self + .catalog() + .aliases() + .get(key) + .map_or(key, String::as_str); + self.base + .groups + .get(owner) + .map(Vec::as_slice) + .unwrap_or_default() + } + + /// Each owner with its names, the owner first, in the order the document declared the aliases + pub fn alias_groups(&self) -> impl Iterator { + self.base + .groups + .iter() + .map(|(owner, names)| (owner.as_str(), names.as_slice())) + } + + /// The `fallback_generalizations` block as the document wrote it + pub fn fallback_generalizations(&self) -> Option<&Value> { + self.catalog().section(FALLBACK_GENERALIZATIONS) + } +} + +/// `git hash-object` of the bytes, which main reports as `source_revision` +pub fn git_blob_id(body: &[u8]) -> String { + let digest = Sha1::new() + .chain_update(format!("blob {}\0", body.len())) + .chain_update(body) + .finalize(); + format!("{digest:x}") +} diff --git a/litellm-rust/crates/model-catalog/src/error.rs b/litellm-rust/crates/model-catalog/src/error.rs index 89e27c05f46..a4b54b5a551 100644 --- a/litellm-rust/crates/model-catalog/src/error.rs +++ b/litellm-rust/crates/model-catalog/src/error.rs @@ -67,6 +67,12 @@ pub(crate) enum Repr { #[error("alias {alias:?} already belongs to {taken_by}")] #[strum_discriminants(strum(serialize = "alias_taken"))] AliasTaken { alias: String, taken_by: String }, + #[error("alias_of {target:?} is not a model name")] + #[strum_discriminants(strum(serialize = "alias_target_missing"))] + AliasTargetMissing { target: String }, + #[error("an alias_of row carries no other field")] + #[strum_discriminants(strum(serialize = "alias_row_field"))] + AliasRowField, #[error("{first} and {second} price the same meter under the same conditions")] #[strum_discriminants(strum(serialize = "duplicate_price"))] DuplicatePrice { first: String, second: String }, @@ -141,6 +147,8 @@ impl Error { | Repr::UnknownTimezone { .. } | Repr::AliasShadowsModel { .. } | Repr::AliasTaken { .. } + | Repr::AliasTargetMissing { .. } + | Repr::AliasRowField | Repr::DuplicatePrice { .. } | Repr::SuffixInsideBlock { .. } | Repr::TierRange @@ -159,6 +167,74 @@ impl Error { } } +impl Error { + /// The document parsed but holds fewer models than the integrity limits trust + pub fn is_integrity(&self) -> bool { + match self.repr() { + Repr::Empty | Repr::BelowMinimum { .. } | Repr::Shrunk { .. } => true, + Repr::Json(_) + | Repr::NotAnObject + | Repr::Invalid(_) + | Repr::UnknownBlockKey + | Repr::UnknownTimezone { .. } + | Repr::AliasShadowsModel { .. } + | Repr::AliasTaken { .. } + | Repr::AliasTargetMissing { .. } + | Repr::AliasRowField + | Repr::DuplicatePrice { .. } + | Repr::SuffixInsideBlock { .. } + | Repr::TierRange + | Repr::TierReversed + | Repr::TierGap { .. } + | Repr::MixedTierBasis + | Repr::NotPricedByResults + | Repr::BandsFromTwoSources + | Repr::NotAnOffPeakRate + | Repr::NoOffPeakPrices + | Repr::NoOffPeakHours + | Repr::EmptyList + | Repr::InvalidDate { .. } + | Repr::MissingProvider + | Repr::UnreadField + | Repr::NoFallbackRules => false, + } + } + + /// A problem with a row's aliases, which the served map skips + pub fn is_alias(&self) -> bool { + match self.repr() { + Repr::AliasShadowsModel { .. } + | Repr::AliasTaken { .. } + | Repr::AliasTargetMissing { .. } + | Repr::AliasRowField => true, + Repr::Invalid(_) => matches!(self.field().as_deref(), Some("aliases" | "alias_of")), + Repr::Json(_) + | Repr::Empty + | Repr::BelowMinimum { .. } + | Repr::Shrunk { .. } + | Repr::NotAnObject + | Repr::UnknownBlockKey + | Repr::UnknownTimezone { .. } + | Repr::DuplicatePrice { .. } + | Repr::SuffixInsideBlock { .. } + | Repr::TierRange + | Repr::TierReversed + | Repr::TierGap { .. } + | Repr::MixedTierBasis + | Repr::NotPricedByResults + | Repr::BandsFromTwoSources + | Repr::NotAnOffPeakRate + | Repr::NoOffPeakPrices + | Repr::NoOffPeakHours + | Repr::EmptyList + | Repr::InvalidDate { .. } + | Repr::MissingProvider + | Repr::UnreadField + | Repr::NoFallbackRules => false, + } + } +} + #[derive(Debug)] pub struct Loaded { pub value: T, @@ -327,6 +403,8 @@ mod tests { #[case::unknown_timezone(Repr::UnknownTimezone { zone: "invalid".into() }, Rejects::Row)] #[case::alias_shadows_model(Repr::AliasShadowsModel { alias: "model".into() }, Rejects::Row)] #[case::alias_taken(Repr::AliasTaken { alias: "alias".into(), taken_by: "owner".into() }, Rejects::Row)] + #[case::alias_target_missing(Repr::AliasTargetMissing { target: "owner".into() }, Rejects::Row)] + #[case::alias_row_field(Repr::AliasRowField, Rejects::Row)] #[case::duplicate_price(Repr::DuplicatePrice { first: "a".into(), second: "b".into() }, Rejects::Row)] #[case::suffix_inside_block(Repr::SuffixInsideBlock { block: "tiered_pricing" }, Rejects::Row)] #[case::tier_range(Repr::TierRange, Rejects::Row)] diff --git a/litellm-rust/crates/model-catalog/src/layers.rs b/litellm-rust/crates/model-catalog/src/layers.rs deleted file mode 100644 index 881f0f6af9b..00000000000 --- a/litellm-rust/crates/model-catalog/src/layers.rs +++ /dev/null @@ -1,32 +0,0 @@ -use std::sync::Arc; - -use indexmap::IndexMap; -use serde_json::Value; - -use crate::catalog::Catalog; -use crate::register::Registrations; -use crate::wire::Fields; - -/// One field of a fetched row over the layer below, RFC 7386. An absent field inherits, JSON `null` is `Remove` -#[derive(Clone, Debug, PartialEq)] -pub enum Patch { - Set(Value), - Remove, -} - -/// The four layers served rows are composed from, lowest first -#[derive(Clone, Debug)] -#[expect( - dead_code, - reason = "02 and 07 add the functions that read these layers" -)] -pub struct Layers { - /// 0, the wheel's embedded map - bundled: Arc, - /// 1, the last installed fetch, rejected rows left out - fetched: Option>, - /// 2, `register(persist=False)`, cleared by every install and refilled by the router replay - transient: IndexMap, - /// 3, `register(persist=True)`, applied over every install - persisted: Registrations, -} diff --git a/litellm-rust/crates/model-catalog/src/lib.rs b/litellm-rust/crates/model-catalog/src/lib.rs index fe419dde054..9b869a078db 100644 --- a/litellm-rust/crates/model-catalog/src/lib.rs +++ b/litellm-rust/crates/model-catalog/src/lib.rs @@ -17,6 +17,9 @@ //! failing the document. //! - `FallbackGeneralizations` / `FallbackRule`: regex rules that map unknown model ids onto known //! families. They are data in the file, not entries. +//! - `CostMap`: an installed document with its provenance, which a host builds its live table from. +//! `Registrations` is the ordered `register_model` overlay over the host's own values, and +//! `write_pass` writes registrations into a host's table through `RegistrationTable`. //! - Validation: `Catalog::check_size` checks a catalog against `IntegrityLimits` (a remote copy //! that lost most of its models is refused), and `Catalog::row_errors` validates one row. //! @@ -37,9 +40,9 @@ macro_rules_attribute::attribute_alias! { mod bundled; mod catalog; +mod cost_map; mod error; mod fallback; -mod layers; mod lower; mod model; mod model_info; @@ -51,9 +54,9 @@ pub mod wire; pub use bundled::*; pub use catalog::*; +pub use cost_map::*; pub use error::*; pub use fallback::*; -pub use layers::*; pub use model::*; pub use model_info::*; pub use register::*; diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index afbbfec59e6..fcd0c027323 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -28,6 +28,7 @@ pub struct ModelInfoFields { pub cache_read_input_audio_token_cost_flex: Option, pub cache_read_input_audio_token_cost_priority: Option, pub cache_read_input_audio_token_cost_batches: Option, + pub cache_read_input_image_token_cost_batches: Option, pub explicit_cache_read_input_token_cost: Option, pub prompt_cache_min_tokens: Option, pub cache_read_input_token_cost_above_200k_tokens: Option, diff --git a/litellm-rust/crates/model-catalog/src/register.rs b/litellm-rust/crates/model-catalog/src/register.rs index ed34082ae57..7aaf2ed3978 100644 --- a/litellm-rust/crates/model-catalog/src/register.rs +++ b/litellm-rust/crates/model-catalog/src/register.rs @@ -1,26 +1,252 @@ -use std::collections::{BTreeSet, HashMap}; +use std::collections::BTreeMap; use indexmap::IndexMap; -use crate::wire::Fields; - -/// Where one `register` call writes and which row it inherits from -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RegistrationTarget { - /// Always the caller's key, never a case variant of it - pub write: String, - /// The first candidate that is an exact key whose row passes `provider_matches` - pub inherit_from: Option, +/// `register_model(persist_across_reloads=True)` calls, replayed above every installed document +#[derive(Clone, Debug, PartialEq)] +pub struct Registrations { + /// Each registered key's merged value, ordered by its latest registration + entries: IndexMap, + /// The registered keys whose write landed on each row key, in landing order + landed_on: IndexMap>, } -/// Python's inheritance candidates in main's order, `get_model_info`'s then `_resolve_builtin_model_cost_entry`'s -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RegistrationCandidates(pub Vec); - -/// Layer 3, and the fields `register` last wrote, which litellm owns again -#[derive(Clone, Debug, Default)] -#[expect(dead_code, reason = "07 adds the functions that read registrations")] -pub struct Registrations { - persisted: IndexMap, - registered_fields: HashMap>, +impl Default for Registrations { + fn default() -> Self { + Self { + entries: IndexMap::new(), + landed_on: IndexMap::new(), + } + } +} + +/// One registration written to a host's table: the key it was registered under and the row key it landed on +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Landed { + pub registered: String, + pub written: String, +} + +impl Registrations { + /// Keeps `value` as the key's registration, already merged with `get(key)` by the host, and makes it the latest + pub fn register(&mut self, key: &str, value: V) { + self.entries.shift_remove(key); + self.entries.insert(key.to_owned(), value); + } + + pub fn get(&self, key: &str) -> Option<&V> { + self.entries.get(key) + } + + pub fn record(&mut self, landed: &[Landed]) { + for Landed { + registered, + written, + } in landed + { + let names = self.landed_on.entry(written.clone()).or_default(); + if !names.contains(registered) { + names.push(registered.clone()); + } + } + } + + /// Every registration, oldest first + pub fn entries(&self) -> impl Iterator { + self.entries + .iter() + .map(|(key, value)| (key.as_str(), value)) + } + + /// The registrations that apply to the row under `key`, oldest first: `key`'s own and every one that landed on it + pub fn over<'a>(&'a self, key: &str) -> Vec<&'a V> { + let landed = self.landed_on.get(key).into_iter().flatten(); + let by_order: BTreeMap = landed + .map(String::as_str) + .chain([key]) + .filter_map(|name| self.entries.get_full(name)) + .map(|(order, _, value)| (order, value)) + .collect(); + by_order.into_values().collect() + } + + pub fn map(&self, mut convert: impl FnMut(&V) -> W) -> Registrations { + Registrations { + entries: self + .entries + .iter() + .map(|(key, value)| (key.clone(), convert(value))) + .collect(), + landed_on: self.landed_on.clone(), + } + } +} + +/// A host's live table that registrations are written into +pub trait RegistrationTable { + type Registration; + type Error; + + /// Writes `registration` where it lands, `None` when this table is no longer the one the host publishes + fn place( + &mut self, + key: &str, + registration: &Self::Registration, + ) -> Result, Self::Error>; +} + +/// Where a registration was written and whether a built-in row matched it +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Placed { + pub key: String, + pub matched_builtin: bool, +} + +#[derive(Debug, PartialEq)] +pub enum Pass<'a, R> { + /// Every entry was written. `unmatched` are those no built-in row matched + Finished { + landed: Vec, + unmatched: Vec<(&'a str, &'a R)>, + }, + /// The table stopped being the published one before every entry was written + Swapped { landed: Vec }, +} + +impl Pass<'_, R> { + pub fn landed(&self) -> &[Landed] { + match self { + Pass::Finished { landed, .. } | Pass::Swapped { landed } => landed, + } + } +} + +/// Writes `entries` into `table` in order +pub fn write_pass<'a, T: RegistrationTable>( + table: &mut T, + entries: &'a [(String, T::Registration)], +) -> Result, T::Error> { + let mut landed = Vec::with_capacity(entries.len()); + let mut unmatched = Vec::new(); + for (key, registration) in entries { + let Some(written) = table.place(key, registration)? else { + return Ok(Pass::Swapped { landed }); + }; + if !written.matched_builtin { + unmatched.push((key.as_str(), registration)); + } + landed.push(Landed { + registered: key.clone(), + written: written.key, + }); + } + Ok(Pass::Finished { landed, unmatched }) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn landed(registered: &str, written: &str) -> Landed { + Landed { + registered: registered.to_owned(), + written: written.to_owned(), + } + } + + #[rstest] + fn reregistering_a_key_makes_it_the_latest() { + let mut registrations = Registrations::default(); + registrations.register("a", 1); + registrations.register("b", 2); + registrations.register("a", 3); + assert_eq!( + registrations.entries().collect::>(), + [("b", &2), ("a", &3)] + ); + } + + #[rstest] + fn over_applies_names_landed_on_the_row_in_registration_order() { + let mut registrations = Registrations::default(); + registrations.register("openai/o3-pro", 1); + registrations.register("o3-pro", 2); + registrations.register("other", 3); + registrations.record(&[ + landed("openai/o3-pro", "o3-pro"), + landed("o3-pro", "o3-pro"), + ]); + registrations.register("openai/o3-pro", 4); + assert_eq!(registrations.over("o3-pro"), [&2, &4]); + } + + #[rstest] + fn over_applies_a_keys_own_registration_that_landed_on_another_row() { + let mut registrations = Registrations::default(); + registrations.register("openai/o3-pro", 1); + registrations.record(&[landed("openai/o3-pro", "o3-pro")]); + assert_eq!( + ( + registrations.over("openai/o3-pro"), + registrations.over("o3-pro") + ), + (vec![&1], vec![&1]) + ); + } + + /// A table published until `swaps_after` writes, whose rows match a built-in only when the key starts with `b` + struct Table { + writes: Vec<(String, i32)>, + swaps_after: usize, + } + + impl RegistrationTable for Table { + type Registration = i32; + type Error = (); + + fn place(&mut self, key: &str, registration: &i32) -> Result, ()> { + if self.writes.len() >= self.swaps_after { + return Ok(None); + } + let written = key.to_uppercase(); + self.writes.push((written.clone(), *registration)); + Ok(Some(Placed { + matched_builtin: written.starts_with('B'), + key: written, + })) + } + } + + #[rstest] + #[case::published_throughout( + usize::MAX, + Pass::Finished { landed: vec![landed("a", "A"), landed("b", "B")], unmatched: vec![("a", &1)] }, + &[("A", 1), ("B", 2)], + )] + #[case::swapped_after_the_first_write( + 1, + Pass::Swapped { landed: vec![landed("a", "A")] }, + &[("A", 1)], + )] + fn a_pass_writes_entries_in_order_until_the_table_is_swapped( + #[case] swaps_after: usize, + #[case] expected: Pass<'static, i32>, + #[case] writes: &[(&str, i32)], + ) { + let entries = [("a".to_owned(), 1), ("b".to_owned(), 2)]; + let mut table = Table { + writes: Vec::new(), + swaps_after, + }; + assert_eq!(write_pass(&mut table, &entries), Ok(expected)); + assert_eq!( + table.writes, + writes + .iter() + .map(|(key, value)| ((*key).to_owned(), *value)) + .collect::>() + ); + } } diff --git a/litellm-rust/crates/model-catalog/src/schema.rs b/litellm-rust/crates/model-catalog/src/schema.rs index 6fba985a8bd..60c755fe94f 100644 --- a/litellm-rust/crates/model-catalog/src/schema.rs +++ b/litellm-rust/crates/model-catalog/src/schema.rs @@ -104,9 +104,18 @@ pub fn registry_json_schema() -> Schema { "title".into(), "LiteLLM model prices and context window registry".into(), ); + definitions.insert( + "aliasRow".into(), + json!({ + "type": "object", + "required": ["alias_of"], + "properties": {"alias_of": {"type": "string"}}, + "additionalProperties": false, + }), + ); root.insert( "additionalProperties".into(), - json!({"$ref": "#/$defs/modelEntry"}), + json!({"anyOf": [{"$ref": "#/$defs/modelEntry"}, {"$ref": "#/$defs/aliasRow"}]}), ); root.insert("$defs".into(), Value::Object(definitions)); Schema::from(root.clone()) @@ -153,6 +162,7 @@ fn decorate_model_entry(schema: &mut Value) { let rates = json!({FLAT_KEY_PATTERN: {"type": "number", "minimum": 0}}); let object = schema.as_object_mut().unwrap(); object.insert("required".into(), json!(["litellm_provider"])); + object.insert("not".into(), json!({"required": ["alias_of"]})); object.insert("additionalProperties".into(), Value::Bool(true)); object.insert( "patternProperties".into(), diff --git a/litellm-rust/crates/model-catalog/src/store/AGENTS.md b/litellm-rust/crates/model-catalog/src/store/AGENTS.md index f1dacd2e217..580798da895 100644 --- a/litellm-rust/crates/model-catalog/src/store/AGENTS.md +++ b/litellm-rust/crates/model-catalog/src/store/AGENTS.md @@ -24,7 +24,7 @@ | Source | Stamp | Blind spot | |---|---|---| | `Catalog`'s `CatalogRows` | `()`, every change goes through `Store::edit`, which drops the index | none | -| python-bridge's bound `litellm.model_cost` | the dict object and its `len` | a delete plus an add between two reads, or a `litellm_provider` rewrite on an existing key | +| python-bridge's bound `litellm.model_cost` | the dict object, its `len` and its last key | deletes plus adds between two reads that put the old last key back last, or a `litellm_provider` rewrite on an existing key | A source with a blind spot calls `Store::invalidate` when it knows keys changed. Until then a key added in the blind spot is not found, and a key moved to another provider stays listed under the old one. Removed keys are still never returned diff --git a/litellm-rust/crates/model-catalog/src/wire/attributes.rs b/litellm-rust/crates/model-catalog/src/wire/attributes.rs index 401634f8c62..52cd2e8525f 100644 --- a/litellm-rust/crates/model-catalog/src/wire/attributes.rs +++ b/litellm-rust/crates/model-catalog/src/wire/attributes.rs @@ -47,6 +47,8 @@ macro_rules! attributes { } attributes! { + /// Model this row is another name for; such a row carries no other key. + alias_of: String, /// Other names this entry is served under. aliases: Vec, audio_transcription_config: String, diff --git a/litellm-rust/crates/model-catalog/tests/catalog.rs b/litellm-rust/crates/model-catalog/tests/catalog.rs index 31f907689d3..679c67412c8 100644 --- a/litellm-rust/crates/model-catalog/tests/catalog.rs +++ b/litellm-rust/crates/model-catalog/tests/catalog.rs @@ -139,6 +139,18 @@ fn a_bad_entry_is_reported_by_model_and_key_and_kept_raw() { json!({"First": {"litellm_provider": "t", "aliases": ["shared"]}, "Second": {"litellm_provider": "t", "aliases": ["shared"]}}), r#"Second: alias "shared" already belongs to First"# )] +#[case::alias_of_target_missing( + json!({"Real": {"litellm_provider": "t"}, "ghost-latest": {"alias_of": "Ghost"}}), + r#"ghost-latest: alias_of "Ghost" is not a model name"# +)] +#[case::alias_of_row_with_other_fields( + json!({"Real": {"litellm_provider": "t"}, "real-latest": {"alias_of": "Real", "mode": "chat"}}), + "real-latest.mode: an alias_of row carries no other field" +)] +#[case::alias_of_claimed_by_a_list( + json!({"Real": {"litellm_provider": "t"}, "real-latest": {"alias_of": "Real"}, "Other": {"litellm_provider": "t", "aliases": ["real-latest"]}}), + r#"real-latest: alias "real-latest" already belongs to Other"# +)] #[case::aliases_not_a_list( json!({"a": {"litellm_provider": "t", "aliases": "bad"}}), "a.aliases: invalid value: invalid type: string \"bad\", expected a sequence" @@ -399,3 +411,70 @@ fn a_bad_value_keeps_its_json_error_as_the_source_and_names_its_key_and_field() assert_eq!(error.key().as_deref(), Some("open/router")); assert_eq!(error.field().as_deref(), Some("input_cost_per_token")); } + +#[rstest] +fn an_alias_of_row_is_served_as_its_target_and_written_as_itself() { + let catalog = catalog(json!({ + "Real": {"litellm_provider": "t", "input_cost_per_token": 1e-6}, + "real-latest": {"alias_of": "Real"}, + "After": {"litellm_provider": "t"} + })); + let found = catalog.lookup("REAL-LATEST").unwrap(); + assert_eq!( + (found.matched_key, found.canonical_key), + ("real-latest", "Real") + ); + assert_eq!( + found.entry.field("input_cost_per_token"), + Some(&json!(1e-6)) + ); + assert_eq!(catalog.model_count(), 2); + assert_eq!( + catalog.served().map(|(key, _)| key).collect::>(), + ["Real", "After", "real-latest"] + ); + assert_eq!( + written(&catalog), + r#"{"Real":{"litellm_provider":"t","input_cost_per_token":1e-6},"real-latest":{"alias_of":"Real"},"After":{"litellm_provider":"t"}}"# + ); +} + +#[rstest] +fn a_dangling_alias_of_row_is_rejected_and_dropped_from_the_written_document() { + let parsed = loaded(json!({ + "Real": {"litellm_provider": "t"}, + "ghost-latest": {"alias_of": "Ghost"} + })); + let (catalog, removed) = parsed.value.without_rejected(&parsed.errors); + assert_eq!(removed.len(), 1); + assert!(catalog.lookup("ghost-latest").is_none()); + assert_eq!(written(&catalog), r#"{"Real":{"litellm_provider":"t"}}"#); +} + +#[rstest] +fn an_alias_of_row_with_other_fields_is_never_served() { + let parsed = loaded(json!({ + "Real": {"litellm_provider": "t"}, + "real-latest": {"alias_of": "Real", "mode": "chat"} + })); + assert!(parsed.value.lookup("real-latest").is_none()); + let (catalog, removed) = parsed.value.without_rejected(&parsed.errors); + assert_eq!(removed.len(), 1); + assert!(catalog.lookup("real-latest").is_none()); + assert_eq!(written(&catalog), r#"{"Real":{"litellm_provider":"t"}}"#); +} + +#[rstest] +fn a_pointer_row_that_repeats_a_listed_alias_is_dropped_and_the_model_is_kept() { + let parsed = loaded(json!({ + "Real": {"litellm_provider": "t", "aliases": ["real-latest"]}, + "real-latest": {"alias_of": "Real"} + })); + let (catalog, removed) = parsed.value.without_rejected(&parsed.errors); + assert_eq!(removed.len(), 1); + assert_eq!(catalog.lookup("real-latest").unwrap().canonical_key, "Real"); + assert_eq!( + written(&catalog), + r#"{"Real":{"litellm_provider":"t","aliases":["real-latest"]}}"# + ); +} diff --git a/litellm-rust/crates/model-catalog/tests/cost_map.rs b/litellm-rust/crates/model-catalog/tests/cost_map.rs new file mode 100644 index 00000000000..04a04170df6 --- /dev/null +++ b/litellm-rust/crates/model-catalog/tests/cost_map.rs @@ -0,0 +1,53 @@ +use litellm_model_catalog::{ + BUNDLED_REGISTRY, CostMap, ErrorKind, IntegrityLimits, Provenance, git_blob_id, +}; +use rstest::rstest; +use serde_json::json; + +fn any_size() -> IntegrityLimits { + IntegrityLimits::new(0, 0.0).unwrap() +} + +#[rstest] +#[case::not_json(b"{".as_slice(), false, ErrorKind::Json)] +#[case::empty(b"{}".as_slice(), true, ErrorKind::Empty)] +fn a_refused_document_says_whether_it_failed_the_integrity_limits( + #[case] body: &[u8], + #[case] integrity: bool, + #[case] kind: ErrorKind, +) { + let error = CostMap::read(body, None, any_size()).unwrap_err(); + assert_eq!((error.is_integrity(), error.kind()), (integrity, kind)); +} + +#[rstest] +fn provenance_is_the_read_bytes_and_etag() { + let body = br#"{"m":{"litellm_provider":"p"}}"#; + let cost_map = CostMap::read(body, Some("\"v1\"".into()), any_size()).unwrap(); + assert_eq!( + cost_map.provenance(), + &Provenance { + revision: "eaa747f210a1527d5ea7138be0f768c47578ca68".to_owned(), + etag: Some("\"v1\"".into()), + } + ); + assert_eq!( + CostMap::bundled().provenance().revision, + git_blob_id(BUNDLED_REGISTRY) + ); +} + +#[rstest] +fn every_name_of_an_aliased_row_lists_its_owner_first( + #[values("owner", "short", "owner-latest")] key: &str, +) { + let body = json!({ + "owner": {"litellm_provider": "p", "aliases": ["short"]}, + "owner-latest": {"alias_of": "owner"}, + "plain": {"litellm_provider": "p"}, + }) + .to_string(); + let cost_map = CostMap::read(body.as_bytes(), None, any_size()).unwrap(); + assert_eq!(cost_map.names(key), ["owner", "short", "owner-latest"]); + assert!(cost_map.names("plain").is_empty()); +} diff --git a/litellm-rust/crates/model-catalog/tests/schema.rs b/litellm-rust/crates/model-catalog/tests/schema.rs index 464cf069cac..5fe5832cb58 100644 --- a/litellm-rust/crates/model-catalog/tests/schema.rs +++ b/litellm-rust/crates/model-catalog/tests/schema.rs @@ -35,6 +35,7 @@ fn generated_registry_schema_validates_checked_in_catalog() { #[case(json!({"example": {"litellm_provider": "test"}}))] #[case(json!({"example": {"litellm_provider": "test", "future_field": true}}))] #[case(json!({"sample_spec": {"litellm_provider": "placeholder"}}))] +#[case(json!({"example": {"litellm_provider": "test"}, "example-latest": {"alias_of": "example"}}))] fn generated_registry_schema_keeps_reader_compatibility(#[case] document: Value) { assert!(registry_validator().is_valid(&document)); } @@ -54,6 +55,9 @@ fn generated_registry_schema_keeps_reader_compatibility(#[case] document: Value) #[case::empty_windows(json!({"litellm_provider": "test", "off_peak_pricing": {"windows": []}}))] #[case::invalid_weekday(json!({"litellm_provider": "test", "off_peak_pricing": {"windows": [{"hours_utc": "00:00-01:00", "weekdays": [0]}]}}))] #[case::invalid_aliases(json!({"litellm_provider": "test", "aliases": "wrong"}))] +#[case::alias_of_on_a_model(json!({"litellm_provider": "test", "alias_of": "other"}))] +#[case::alias_row_with_other_fields(json!({"alias_of": "other", "mode": "chat"}))] +#[case::alias_of_not_a_string(json!({"alias_of": 4}))] #[case::non_object_model(json!(4))] fn generated_registry_schema_rejects_invalid_entries(#[case] entry: Value) { assert!(!registry_validator().is_valid(&json!({"example": entry}))); diff --git a/litellm-rust/crates/python-bridge/AGENTS.md b/litellm-rust/crates/python-bridge/AGENTS.md index 26858b2afe2..b18a1e2ca19 100644 --- a/litellm-rust/crates/python-bridge/AGENTS.md +++ b/litellm-rust/crates/python-bridge/AGENTS.md @@ -81,6 +81,7 @@ GIL handling to `litellm-host-python`. ## Bridge Shape +- Reach Python only through `litellm/rust_bridge/host/` and the `litellm.rust_bridge.execution` driver. Never import another LiteLLM module or read a `litellm` global. `litellm/rust_bridge/host/AGENTS.md` lists what may be imported directly - Prefer one stable method per top-level LiteLLM route, for example `messages(...)`, calling the matching `litellm-inference-` entrypoint. - Do not add one exported PyO3 function per provider helper unless there is a diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index fd826f549e5..48206d242e4 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -19,16 +19,26 @@ huggingface = ["litellm-token-counter/huggingface"] tiktoken = ["litellm-token-counter/tiktoken"] [dependencies] +rustls.workspace = true litellm-secret-value.workspace = true fancy-regex.workspace = true litellm-tracing.workspace = true litellm-spend-clickhouse.workspace = true litellm-storage-clickhouse.workspace = true litellm-host.workspace = true +async-trait.workspace = true bytes.workspace = true futures-util.workspace = true litellm-cache.workspace = true litellm-error.workspace = true +litellm-circuit-breaker.workspace = true +litellm-redis-client.workspace = true +litellm-redis-connect.workspace = true +litellm-redis-store.workspace = true +litellm-redis-resolve.workspace = true +litellm-python-compat.workspace = true +litellm-redis-types.workspace = true +tracing.workspace = true litellm-cache-response.workspace = true serde.workspace = true litellm-auth.workspace = true @@ -47,10 +57,13 @@ litellm-secrets = { workspace = true, features = ["aws", "azure", "google", "has litellm-secrets-types.workspace = true litellm-llms-types.workspace = true litellm-router-types.workspace = true +litellm-router-core.workspace = true litellm-host-python.workspace = true litellm-model-catalog.workspace = true +litellm-model-catalog-refresh.workspace = true litellm-token-counter = { path = "../token-counter", default-features = false } -pyo3.workspace = true +indexmap = "2.14.0" +pyo3 = { workspace = true, features = ["indexmap"] } prost.workspace = true pyo3-async-runtimes.workspace = true reqwest.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/cache/AGENTS.md b/litellm-rust/crates/python-bridge/src/cache/AGENTS.md index e79562b8762..a95229c5bf4 100644 --- a/litellm-rust/crates/python-bridge/src/cache/AGENTS.md +++ b/litellm-rust/crates/python-bridge/src/cache/AGENTS.md @@ -2,6 +2,6 @@ This folder owns how Rust inference reaches the selected Python cache. Cache algorithms, storage protocols and response-cache semantics belong to their cache crates -`mod.rs` exposes the cache boundary to routes. `selection.rs` asks `litellm.rust_bridge.host.cache` for the live configured cache and owns route admission and inference protocol composition +`mod.rs` exposes the cache boundary to routes. `selection.rs` asks `litellm.rust_bridge.host.cache` for the live configured cache and owns route admission. A route's protocol carries cache operations through any host call type that converts from `CacheCall` `python/` delegates operations to the selected Python cache without discovering configuration. Routes use only the parent module's exports diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs index 4e3847951d8..959c211b1d3 100644 --- a/litellm-rust/crates/python-bridge/src/cache/mod.rs +++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs @@ -2,10 +2,12 @@ mod python; mod selection; pub(crate) use python::{CacheCall, PythonCache}; -pub(crate) use selection::{Cached, Selection, configure, ensure_cache_disabled}; +pub(crate) use selection::{Selection, configure, ensure_cache_disabled}; use pyo3::{exceptions::PyRuntimeError, prelude::*}; +const CACHE_HOST_MODULE: &str = "litellm.rust_bridge.host.cache"; + #[derive(Debug, thiserror::Error)] pub(crate) enum HandleError { #[error("this inference route does not support configured caches")] diff --git a/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md b/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md index 81a0997f02a..9aba11a3376 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md +++ b/litellm-rust/crates/python-bridge/src/cache/python/AGENTS.md @@ -1,9 +1,9 @@ # Python cache delegation -This directory lets Rust inference use a selected Python cache. `service.rs` implements the injected Rust response-cache service and yields typed cache operations. `host.rs` calls the Python cache's sync or async API and delivers the result back to Rust. +This directory lets Rust inference use a selected Python cache. `service.rs` implements the injected Rust response-cache service and yields typed cache operations. `host.rs` delivers each operation's result back to Rust. Asynchronous lookups and stream stores go through `litellm.rust_bridge.host.cache`, which reads with the `LLMCachingHandler` the `@client` wrapper uses, for the legacy function the route names, and writes with its dual cache. `configure` disables the cache for synchronous calls. Keep the shared inference protocol wrapper and adapter selection in the parent module. Receive the configured cache and prepared arguments from the parent module. Do not discover global configuration, choose native backends, or move inference to Python. Core remains independent of Python objects and cache implementation details -Await asynchronous cache operations through the existing host driver in the caller's task. Do not create another asyncio task or event loop. Cancellation must prevent subsequent provider requests and cache writes. Preserve ordinary cache failure handling without swallowing cancellation or other Python base exceptions +Await asynchronous cache operations through the existing host driver in the caller's task. Do not create another asyncio task or event loop. The one exception is the asynchronous unary messages store: Python's `_async_set_cache` writes the returned response from `create_cache_write_task` after the router has stamped its `_hidden_params`, so `host.rs` defers that store until the deployment transform has returned the response the caller receives and hands that response to `litellm.rust_bridge.host.cache.store_response`, which schedules the same write task. Lookups, sync stores and stream stores stay in the caller's task. Cancellation must prevent subsequent provider requests and cache writes. Preserve ordinary cache failure handling without swallowing cancellation or other Python base exceptions Traverse retained Python references for GC and release pending replies when the call closes. Regression tests must require native inference with Python fallback disabled and assert observable hits, provider request counts, cache-key headers, task identity and cancellation diff --git a/litellm-rust/crates/python-bridge/src/cache/python/host.rs b/litellm-rust/crates/python-bridge/src/cache/python/host.rs index 82b7d881df6..208f40a324b 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/host.rs +++ b/litellm-rust/crates/python-bridge/src/cache/python/host.rs @@ -1,6 +1,6 @@ use litellm_cache::Error; use litellm_host::protocol::Reply; -use litellm_host_python::{from_py, to_py}; +use litellm_host_python::{json_from_py, to_py}; use pyo3::{ gc::{PyTraverseError, PyVisit}, prelude::*, @@ -9,7 +9,7 @@ use pyo3::{ use serde_json::Value; use super::service::CacheCall; -use crate::cache::HandleError; +use crate::cache::{CACHE_HOST_MODULE, HandleError}; enum Pending { Lookup(Reply, Error>>), @@ -18,27 +18,39 @@ enum Pending { pub(crate) struct PythonCache { cache: Option>, + function: Option>, arguments: Option>, + read_handler: Option>, pending: Option, asynchronous: bool, + deferred: bool, } impl PythonCache { pub fn new(asynchronous: bool) -> Self { Self { cache: None, + function: None, arguments: None, + read_handler: None, pending: None, asynchronous, + deferred: false, } } + pub(in crate::cache) fn asynchronous(&self) -> bool { + self.asynchronous + } + pub(in crate::cache) fn bind( &mut self, cache: Bound<'_, PyAny>, + function: &Bound<'_, PyAny>, arguments: &Bound<'_, PyDict>, ) { self.cache = Some(cache.unbind()); + self.function = Some(function.clone().unbind()); self.arguments = Some(arguments.clone().unbind()); } @@ -57,18 +69,25 @@ impl PythonCache { self.pending = Some(Pending::Lookup(reply)); ( if self.asynchronous { - "async_get_cache" + "lookup" } else { "get_cache" }, None, ) } - CacheCall::Store { value, reply } => { + CacheCall::Store { + unary: true, reply, .. + } if self.asynchronous => { + self.deferred = true; + reply.send(Ok(())); + return Ok(None); + } + CacheCall::Store { value, reply, .. } => { self.pending = Some(Pending::Store(reply)); ( if self.asynchronous { - "async_add_cache" + "store_stream" } else { "add_cache" }, @@ -77,9 +96,20 @@ impl PythonCache { } }; let result = match result { + Some(value) if self.asynchronous => py + .import(CACHE_HOST_MODULE)? + .getattr(method)? + .call((cache, value), Some(&arguments)), Some(value) => cache .bind(py) .call_method(method, (value,), Some(&arguments)), + None if self.asynchronous => py.import(CACHE_HOST_MODULE)?.getattr(method)?.call( + ( + cache, + self.function.as_ref().ok_or(HandleError::Unconfigured)?, + ), + Some(&arguments), + ), None => cache.bind(py).call_method(method, (), Some(&arguments)), } .map(Bound::unbind); @@ -89,6 +119,43 @@ impl PythonCache { self.resume(py, result) } + pub fn store_response( + &mut self, + py: Python<'_>, + response: &Bound<'_, PyAny>, + verbatim: bool, + ) -> PyResult<()> { + if !std::mem::take(&mut self.deferred) { + return Ok(()); + } + let (Some(cache), Some(arguments)) = (self.cache.as_ref(), self.arguments.as_ref()) else { + return Err(HandleError::Unconfigured.into()); + }; + py.import(CACHE_HOST_MODULE)? + .getattr(if verbatim { + "store_verbatim" + } else { + "store_response" + })? + .call((cache, response), Some(&arguments.bind(py).copy()?))?; + Ok(()) + } + + pub fn wrap_stream(&self, py: Python<'_>, stream: Py) -> PyResult> { + let (Some(cache), Some(function), Some(arguments)) = + (&self.cache, &self.function, &self.arguments) + else { + return Ok(stream); + }; + py.import(CACHE_HOST_MODULE)? + .getattr("wrap_stream")? + .call( + (cache, function, stream, &self.read_handler), + Some(&arguments.bind(py).copy()?), + ) + .map(Bound::unbind) + } + pub fn resume( &mut self, py: Python<'_>, @@ -103,11 +170,14 @@ impl PythonCache { }; match self.pending.take() { Some(Pending::Lookup(reply)) => { + if self.asynchronous { + self.read_handler = self.caching_handler(py).ok().flatten(); + } let value = result.map_err(Error::unavailable).and_then(|value| { if value.bind(py).is_none() { Ok(None) } else { - from_py(value.bind(py)) + json_from_py(value.bind(py)) .map(Some) .map_err(Error::invalid_entry) } @@ -122,14 +192,30 @@ impl PythonCache { Ok(None) } + fn caching_handler(&self, py: Python<'_>) -> PyResult>> { + let Some(arguments) = self.arguments.as_ref() else { + return Ok(None); + }; + let handler = py + .import(CACHE_HOST_MODULE)? + .getattr("caching_handler")? + .call((), Some(arguments.bind(py)))?; + Ok((!handler.is_none()).then(|| handler.unbind())) + } + pub fn close(&mut self) { self.pending = None; + self.read_handler = None; + self.deferred = false; self.cache = None; + self.function = None; self.arguments = None; } pub fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { visit.call(&self.cache)?; - visit.call(&self.arguments) + visit.call(&self.function)?; + visit.call(&self.arguments)?; + visit.call(&self.read_handler) } } diff --git a/litellm-rust/crates/python-bridge/src/cache/python/service.rs b/litellm-rust/crates/python-bridge/src/cache/python/service.rs index 38872b85d30..bd54accc0dc 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/service.rs +++ b/litellm-rust/crates/python-bridge/src/cache/python/service.rs @@ -45,6 +45,7 @@ pub(crate) enum CacheCall { }, Store { value: Value, + unary: bool, reply: Reply>, }, } @@ -54,11 +55,12 @@ struct PythonCacheService { config: ResponseCacheConfig, } -pub(in crate::cache) fn service>( +pub(in crate::cache) fn service( services: HostServices

, namespace: String, ) -> std::sync::Arc where + P::HostCall: From, P::Error: From + std::error::Error, { std::sync::Arc::new(PythonCacheService { @@ -70,14 +72,19 @@ where }) } -impl> ResponseCacheService for PythonCacheService

+impl ResponseCacheService for PythonCacheService

where + P::HostCall: From, P::Error: From + std::error::Error, { fn config(&self) -> &ResponseCacheConfig { &self.config } + fn keyed_by_caller(&self) -> bool { + true + } + fn lookup<'a>( &'a self, _: &'a ResponseCacheRequest, @@ -85,7 +92,7 @@ where ) -> Pin, Error>> + Send + 'a>> { Box::pin(async move { self.services - .call(|reply| CacheCall::Lookup { reply }) + .call(|reply| CacheCall::Lookup { reply }.into()) .await .map_err(Error::unavailable)?? .map(from_python) @@ -101,8 +108,16 @@ where ) -> Pin> + Send + 'a>> { Box::pin(async move { let value = to_python(value)?; + let unary = value.get(STREAM_EVENTS_KEY).is_none(); self.services - .call(|reply| CacheCall::Store { value, reply }) + .call(|reply| { + CacheCall::Store { + value, + unary, + reply, + } + .into() + }) .await .map_err(Error::unavailable)? }) diff --git a/litellm-rust/crates/python-bridge/src/cache/selection.rs b/litellm-rust/crates/python-bridge/src/cache/selection.rs index e96f079e5da..6e4d222a1d0 100644 --- a/litellm-rust/crates/python-bridge/src/cache/selection.rs +++ b/litellm-rust/crates/python-bridge/src/cache/selection.rs @@ -1,22 +1,10 @@ -use super::{HandleError, python}; +use super::{CACHE_HOST_MODULE, HandleError, python}; use litellm_cache_response::{CacheOptions, CachePolicy, CacheScope, ScopedCache}; use litellm_host::{ machine::{HostServices, MachineFault}, protocol::Protocol, }; use pyo3::{prelude::*, types::PyDict}; -const CACHE_HOST_MODULE: &str = "litellm.rust_bridge.host.cache"; - -pub(crate) struct Cached

(std::marker::PhantomData

); - -impl Protocol for Cached

{ - type Request = (P::Request, Selection); - type Response = P::Response; - type Error = P::Error; - type HostCall = python::CacheCall; - type Chunk = P::Chunk; - type StreamHead = P::StreamHead; -} enum Backend { Disabled, @@ -29,11 +17,12 @@ pub(crate) struct Selection { } impl Selection { - pub(crate) fn attach>( + pub(crate) fn attach( self, services: HostServices

, ) -> (Option, CacheOptions) where + P::HostCall: From, P::Error: From + std::error::Error, { let service = match self.backend { @@ -52,15 +41,21 @@ fn selected_cache<'py>( kwargs: &Bound<'py, PyDict>, call_type: &str, ) -> PyResult>> { + if kwargs + .get_item("caching")? + .is_some_and(|value| value.is(pyo3::types::PyBool::new(py, false))) + { + return Ok(None); + } + configured_cache(py, call_type) +} + +fn configured_cache<'py>(py: Python<'py>, call_type: &str) -> PyResult>> { let configured = py .import(CACHE_HOST_MODULE)? .getattr("configured_cache")? .call0()?; - if configured.is_none() - || kwargs - .get_item("caching")? - .is_some_and(|value| value.is(pyo3::types::PyBool::new(py, false))) - { + if configured.is_none() { return Ok(None); } let supported = configured.getattr("supported_call_types")?; @@ -91,9 +86,14 @@ pub(crate) fn configure( python: &mut python::PythonCache, py: Python<'_>, arguments: &Bound<'_, PyDict>, - call_type: &str, + function: &Bound<'_, PyAny>, ) -> PyResult { - let selected = selected_cache(py, arguments, call_type)?; + let call_type: String = function.getattr("__name__")?.extract()?; + let selected = if python.asynchronous() { + configured_cache(py, &call_type)? + } else { + None + }; let Some(cache) = selected else { return Ok(Selection { backend: Backend::Disabled, @@ -103,6 +103,9 @@ pub(crate) fn configure( let enabled = cache .call_method("should_use_cache", (), Some(arguments))? .extract::()?; + let skips_read = arguments + .get_item("caching")? + .is_some_and(|value| !value.is_none() && !value.is(pyo3::types::PyBool::new(py, true))); let controls = arguments .get_item("cache")? .filter(|value| !value.is_none()); @@ -118,7 +121,7 @@ pub(crate) fn configure( }; let options = CacheOptions { policy: CachePolicy { - no_cache: boolean("no-cache")?, + no_cache: boolean("no-cache")? || skips_read, no_store: boolean("no-store")?, ..CachePolicy::default() }, @@ -130,7 +133,7 @@ pub(crate) fn configure( .map(|value| value.extract()) .transpose()? .unwrap_or_default(); - python.bind(cache, arguments); + python.bind(cache, function, arguments); Ok(Selection { backend: if enabled { Backend::Python { namespace } diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index 426cd861729..29c26ed30a3 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -3,7 +3,7 @@ use std::error::Error as StdError; use litellm_error::Report; use litellm_inference::Error; use pyo3::{ - exceptions::{PyRuntimeError, PyValueError}, + exceptions::{PyRuntimeError, PyTypeError, PyValueError}, prelude::*, }; use reqwest::header::HeaderMap; @@ -29,11 +29,44 @@ const MESSAGES_UNSUPPORTED_PARAM: &str = "messages_unsupported_param"; /// The phase label (`Connection` or `Read`) of a timed-out call, for Python's timeout text, or /// `Body` for a streamed body read that timed out after the head arrived. const MESSAGES_TIMEOUT: &str = "messages_timeout"; +// `(received, expected, reset)` of a streamed body the connection ended early. +const MESSAGES_BODY_CUT: &str = "messages_body_cut"; /// No credential reached the provider, which Python raises as `AuthenticationError`. const MESSAGES_MISSING_CREDENTIAL: &str = "messages_missing_credential"; /// Python refuses the request only once its logger knows the resolved model and provider (the /// provider config's request transform), so the failure logs them. const MESSAGES_LATE_REJECTION: &str = "messages_late_rejection"; +/// Python's request transform raises a builtin exception (`TypeError`) that its handler does not +/// map, so the caller gets the native exception as it is. +const MESSAGES_PYTHON_RAISED: &str = "messages_python_raised"; +/// The text of a refusal Python's provider config raises from its request transform, which its +/// HTTP handler maps as a provider error. +const MESSAGES_PROVIDER_REJECTION: &str = "messages_provider_rejection"; + +fn provider_rejection(rejection: &(dyn StdError + 'static)) -> Option { + match rejection.downcast_ref::() { + Some(litellm_llms::Error::MissingField("max_tokens")) => { + return Some("max_tokens is required for Anthropic /v1/messages API".to_owned()); + } + Some(_) => return None, + None => {} + } + match rejection.downcast_ref::() { + Some( + detail @ litellm_llms::ErrorDetail::InvalidChoice { + field: "reasoning effort", + .. + }, + ) => Some(format!("litellm.BadRequestError: {detail}")), + Some( + detail @ (litellm_llms::ErrorDetail::InvalidReasoningEffort { .. } + | litellm_llms::ErrorDetail::UnsupportedValue { + field: "effort", .. + }), + ) => Some(detail.to_string()), + _ => None, + } +} pub(crate) fn upstream_error( py: Python<'_>, @@ -88,6 +121,9 @@ pub(crate) fn route_failure( if let Some(raised) = crate::secrets::python_error(py, &error) { return Ok(raised); } + if let Some(raised) = crate::routes::messages::workload_identity::python_error(py, &error) { + return Ok(raised); + } if let Some((status, body)) = error.upstream_response() { let headers = error .upstream_headers() @@ -102,6 +138,18 @@ pub(crate) fn route_failure( return public(upstream_error(py, status, body, &headers)?); } let native = match (marker, error.rejected_request()) { + (RequestMarker::Messages, Some(rejection)) + if matches!( + rejection.downcast_ref::(), + Some(litellm_llms::ErrorDetail::UnorderableTypes { .. }) + ) => + { + let native = PyTypeError::new_err(rejection.to_string()); + let value = native.value(py); + value.setattr(MESSAGES_LATE_REJECTION, true)?; + value.setattr(MESSAGES_PYTHON_RAISED, true)?; + native + } (RequestMarker::Messages, Some(rejection)) => { let native = PyValueError::new_err(Report::new(rejection).to_string()); let value = native.value(py); @@ -121,6 +169,9 @@ pub(crate) fn route_failure( ) { value.setattr(MESSAGES_LATE_REJECTION, true)?; } + if let Some(text) = provider_rejection(rejection) { + value.setattr(MESSAGES_PROVIDER_REJECTION, text)?; + } native } (RequestMarker::Messages, None) => { @@ -134,6 +185,9 @@ pub(crate) fn route_failure( }; value.setattr(MESSAGES_TIMEOUT, phase)?; } + if let Some(cut) = error.body_cut() { + value.setattr(MESSAGES_BODY_CUT, (cut.received, cut.expected, cut.reset))?; + } if error.is_missing_credential() { value.setattr(MESSAGES_MISSING_CREDENTIAL, true)?; } @@ -319,6 +373,72 @@ mod tests { }); } + #[rstest::rstest] + #[case::missing_max_tokens( + litellm_llms::Error::MissingField("max_tokens").into(), + Some("max_tokens is required for Anthropic /v1/messages API") + )] + #[case::unmapped_effort( + Error::invalid_request(litellm_llms::ErrorDetail::InvalidChoice { + field: "reasoning effort", + actual: "'ultra'".into(), + choices: vec!["low", "none"], + }), + Some("litellm.BadRequestError: Unmapped reasoning effort: 'ultra'. Must be one of: 'low', 'none'.") + )] + #[case::other_missing_field(litellm_llms::Error::MissingField("model").into(), None)] + #[case::unsupported_param( + Error::invalid_request(litellm_llms::ErrorDetail::UnsupportedParameter { + model: "claude".into(), + param: "top_k".into(), + value: "5".into(), + hint: String::new(), + }), + None + )] + fn provider_config_rejections_carry_python_text( + #[case] error: Error, + #[case] expected: Option<&str>, + ) { + Python::initialize(); + Python::attach(|py| { + let native = route_failure(py, error, RequestMarker::Messages, Ok).unwrap(); + let text = native + .value(py) + .getattr_opt(MESSAGES_PROVIDER_REJECTION) + .unwrap() + .map(|value| value.extract::().unwrap()); + assert_eq!(text.as_deref(), expected); + }); + } + + #[test] + fn an_unorderable_comparison_raises_python_type_error_as_it_is() { + Python::initialize(); + Python::attach(|py| { + let error = Error::invalid_request(litellm_llms::ErrorDetail::UnorderableTypes { + op: "<=", + left: "str", + right: "int", + }); + let native = route_failure(py, error, RequestMarker::Messages, Ok).unwrap(); + assert!(native.is_instance_of::(py)); + let value = native.value(py); + assert_eq!( + value.str().unwrap().to_string(), + "'<=' not supported between instances of 'str' and 'int'" + ); + assert!(value.getattr_opt(MESSAGES_PYTHON_RAISED).unwrap().is_some()); + assert!( + value + .getattr_opt(MESSAGES_LATE_REJECTION) + .unwrap() + .is_some() + ); + assert!(value.getattr_opt(MESSAGES_REQUEST_ERROR).unwrap().is_none()); + }); + } + #[rstest::rstest] #[case::rejected_request( Error::invalid_request(litellm_llms::ErrorDetail::UnsupportedParameter { diff --git a/litellm-rust/crates/python-bridge/src/http.rs b/litellm-rust/crates/python-bridge/src/http.rs index b4e20a8f750..e7cc8e35172 100644 --- a/litellm-rust/crates/python-bridge/src/http.rs +++ b/litellm-rust/crates/python-bridge/src/http.rs @@ -101,20 +101,73 @@ pub(crate) fn call_config( kwargs: &Bound<'_, PyDict>, asynchronous: bool, ) -> PyResult { - let settings = HttpSettings::from_layers([ + Ok(resolved(py, call_settings(py, kwargs, asynchronous)?)) +} + +fn call_settings( + py: Python<'_>, + kwargs: &Bound<'_, PyDict>, + asynchronous: bool, +) -> PyResult { + Ok(HttpSettings::from_layers([ for_call(call_ssl_verify(kwargs)?, asynchronous), HttpSettingsLayer::from_environment(&ProcessEnvironment), match PythonSettings::Http.read_or_unset(py)? { Some(snapshot) => configured(&snapshot)?, None => HttpSettingsLayer::default(), }, - ]) - .without_missing_files(&|path: &Path| path.exists()); - let resolution = Resolution::from(&settings); + ])) +} + +pub(crate) fn settings_config( + py: Python<'_>, + use_http_settings: bool, +) -> PyResult { + let snapshot = if use_http_settings { + PythonSettings::Http.read_or_unset(py)? + } else { + None + }; + let layer = match snapshot { + Some(snapshot) => configured(&snapshot)?, + None => HttpSettingsLayer::default(), + }; + let settings = HttpSettings::from_layers([ + HttpSettingsLayer::from_environment(&ProcessEnvironment), + layer, + ]); + Ok(resolved(py, settings)) +} + +#[pyfunction] +#[pyo3(signature = (ssl_security_level=None, ssl_ecdh_curve=None))] +pub(crate) fn unsupported_tls_settings( + ssl_security_level: Option, + ssl_ecdh_curve: Option, +) -> Vec { + let settings = HttpSettings::from_layers([HttpSettingsLayer { + ssl_security_level, + ssl_ecdh_curve, + ..HttpSettingsLayer::default() + }]); + Resolution::from(&settings) + .unsupported + .iter() + .map(ToString::to_string) + .collect() +} + +fn resolution(settings: HttpSettings) -> Resolution { + Resolution::from(&settings.without_missing_files(&|path: &Path| path.exists())) +} + +/// The client config `settings` resolve to, warning once per process about each setting the client cannot apply +fn resolved(py: Python<'_>, settings: HttpSettings) -> HttpClientConfig { + let resolution = resolution(settings); for unsupported in unreported(&REPORTED_UNSUPPORTED, resolution.unsupported) { crate::logger::capture(py).scope(|| litellm_tracing::warn!("{unsupported}")); } - Ok(resolution.config) + resolution.config } pub(crate) fn provider_client( @@ -126,6 +179,15 @@ pub(crate) fn provider_client( Ok(pool().client(&config, ClientVariant::Provider)) } +pub(crate) fn messages_client( + py: Python<'_>, + kwargs: &Bound<'_, PyDict>, + asynchronous: bool, +) -> PyResult> { + let config = resolution(call_settings(py, kwargs, asynchronous)?).config; + Ok(pool().client(&config, ClientVariant::Undecoded)) +} + pub(crate) fn host_client(py: Python<'_>, variant: ClientVariant) -> PyResult { let config = call_config(py, &PyDict::new(py), true)?; pool().client(&config, variant).map_err(client_error) diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index de35419308a..45ea95560e1 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -9,9 +9,12 @@ mod execution; mod http; mod logger; mod marshal; +mod model_cost; mod model_registry; mod preflight; mod python_settings; +mod redis; +mod router; mod routes; mod secrets; mod tokenizer; @@ -32,10 +35,45 @@ mod _native { #[pymodule_export] use crate::errors::{RustBridgeDeclined, RustUpstreamError}; #[pymodule_export] + use crate::http::unsupported_tls_settings; + #[pymodule_export] use crate::logger::NativeDiagnosticProcessor; #[pymodule_export] + use crate::model_cost::{ + NativeCostMapSource, NativeFetchFailed, NativeFetchPolicy, NativeFetched, NativeInstalled, + NativeModelCostMap, NativeModelCostMapState, NativeProvenance, NativeRegistered, + NativeRejected, NativeTableSwapped, + }; + #[pymodule_export] use crate::model_registry::bundled_model_cost_map; #[pymodule_export] + use crate::redis::breaker::{NativeAdmission, NativeRedisBreaker, native_redis_breaker_states}; + #[pymodule_export] + use crate::redis::client::NativeRedisClient; + #[pymodule_export] + use crate::redis::store::auth_cache::{NativeAuthCache, PyLocalSizes}; + #[pymodule_export] + use crate::redis::store::rate_limits::{ + NativeRateLimits, PyAtomicApplied, PyAtomicOverLimit, PyGaugeOverLimit, PyGaugesAcquired, + PyGuardedReply, PyWindowCount, + }; + #[pymodule_export] + use crate::redis::store::request::{NativeRequestRedis, NativeSend}; + #[pymodule_export] + use crate::redis::store::router_usage::NativeRouterUsage; + #[pymodule_export] + use crate::redis::store::sensitive_route_pins::NativeSensitiveRoutePins; + #[pymodule_export] + use crate::redis::store::spend_counters::{ + NativeSpendCounters, PyPostCallSettlement, PySpendRead, + }; + #[pymodule_export] + use crate::redis::store::{ + NativePending, NativeRedisStoreError, router_cooldowns::NativeRouterCooldowns, + }; + #[pymodule_export] + use crate::router::{RouterCall, RouterCoreHandle}; + #[pymodule_export] use crate::routes::audio_transcription::{atranscription, transcription}; #[pymodule_export] use crate::routes::chat_completions::{acompletion, completion}; @@ -63,6 +101,7 @@ mod _native { #[pymodule_init] fn init(module: &Bound<'_, PyModule>) -> PyResult<()> { + let _ = rustls::crypto::ring::default_provider().install_default(); let py = module.py(); let dict = module.dict(); dict.set_item( @@ -108,16 +147,53 @@ mod tests { "aresponses", "ResponsesWebSocketConnection", "NativeDiagnosticProcessor", + "NativeRedisClient", + "NativeRedisBreaker", + "NativeAdmission", + "NativeRequestRedis", + "NativeSend", + "NativeAuthCache", + "NativeRateLimits", + "WindowCount", + "AtomicApplied", + "AtomicOverLimit", + "GuardedReply", + "GaugesAcquired", + "GaugeOverLimit", + "NativePending", + "NativeRouterCooldowns", + "NativeRouterUsage", + "NativeSensitiveRoutePins", + "NativeRedisStoreError", + "NativeSpendCounters", + "PostCallSettlement", + "SpendRead", + "LocalSizes", + "native_redis_breaker_states", "NativeClickHouseSpendConfig", "NativeClickHouseSpendStorage", "NativeDatabase", "trace_encode_error", "bundled_model_cost_map", + "NativeModelCostMap", + "NativeModelCostMapState", + "NativeInstalled", + "NativeRejected", + "NativeRegistered", + "NativeTableSwapped", + "NativeProvenance", + "NativeCostMapSource", + "NativeFetchPolicy", + "NativeFetched", + "NativeFetchFailed", "TokenCounter", "Tokenizer", + "RouterCoreHandle", + "RouterCall", "gil_stats", "process_state_started", "reserve_process_for_forking", + "unsupported_tls_settings", ]; #[cfg(feature = "huggingface")] expected.push("HuggingFaceEncoding"); @@ -135,4 +211,18 @@ mod tests { assert_eq!(public_names, expected); }); } + #[rstest::rstest] + fn module_import_installs_ring_as_the_process_crypto_provider() { + Python::initialize(); + Python::attach(|py| { + native_module(py); + }); + + let installed = rustls::crypto::CryptoProvider::get_default() + .expect("importing _native should install a process crypto provider"); + assert_eq!( + format!("{installed:?}"), + format!("{:?}", rustls::crypto::ring::default_provider()) + ); + } } diff --git a/litellm-rust/crates/python-bridge/src/logger/mod.rs b/litellm-rust/crates/python-bridge/src/logger/mod.rs index 8f40d94f458..5032b418404 100644 --- a/litellm-rust/crates/python-bridge/src/logger/mod.rs +++ b/litellm-rust/crates/python-bridge/src/logger/mod.rs @@ -7,7 +7,7 @@ use litellm_tracing::{DiagnosticInput, Level, Logger, Metadata, Policy, Processo use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::*; -const MODULE: &str = "litellm.rust_bridge.logger"; +const MODULE: &str = "litellm.rust_bridge.host.logger"; type NativeDiagnosticOutput = (String, Option, Option, Vec, bool); #[pyclass] diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-bridge/src/marshal.rs index dec8ff0b0ce..927ed741669 100644 --- a/litellm-rust/crates/python-bridge/src/marshal.rs +++ b/litellm-rust/crates/python-bridge/src/marshal.rs @@ -4,7 +4,7 @@ use std::{ }; use litellm_auth::InputSource; -use litellm_host_python::{from_py, from_py_argument, to_py}; +use litellm_host_python::{from_py, from_py_argument, json_to_py, to_py}; use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; use serde::{Serialize, de::DeserializeOwned}; use serde_json::{Map, Value}; @@ -143,6 +143,19 @@ pub(crate) fn public_response( .map(Bound::unbind) } +pub(crate) fn public_json_response( + py: Python<'_>, + module: &str, + response: &(impl Serialize + ?Sized), +) -> PyResult> { + let factory = py.import(module)?.getattr("response")?; + let document = + serde_json::to_value(response).map_err(|error| PyValueError::new_err(error.to_string()))?; + factory + .call1((json_to_py(py, &document)?,)) + .map(Bound::unbind) +} + struct RequestFieldSources<'py> { body: Option>, credentials: Option>, diff --git a/litellm-rust/crates/python-bridge/src/model_cost/AGENTS.md b/litellm-rust/crates/python-bridge/src/model_cost/AGENTS.md new file mode 100644 index 00000000000..4ba23aac1ce --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_cost/AGENTS.md @@ -0,0 +1,16 @@ +# Model cost bridge + +- `NativeModelCostMap` holds inputs only, the installed `CostMap`, kept `Registrations` and the name index. Python's `litellm.model_cost` is the only copy of the live rows +- Every call that reads or writes rows takes the dict as an argument. Never store rows or build a second table here +- `LiveDict` holds a reference to that same dict and reads it under the GIL on every query, so a Python write is seen on the next read + - It needs the GIL, so Rust code running without the GIL cannot use it + - Its stamp is the dict's address, length and last key. [model-catalog/src/store/AGENTS.md](../../../model-catalog/src/store/AGENTS.md) says what that misses +- `host.rs` is the only code that calls Python, by name through `MODULE`. Calls happen with the GIL held and no `NativeModelCostMap` lock held + - `resolve` reaches `find_key` on this same object, so holding a lock across a host call deadlocks + - Test `a_host_lookup_that_reaches_find_key_on_the_same_map_finishes` guards this +- Rows and registrations cross as Python objects, never JSON, because router `model_info` holds `datetime` +- `table::build` serves the document as main's `_expand_model_aliases` did, so all names of a model share one row object made by `host.entry` +- `install` touches no state until `CostMap::read` succeeds. `NativeRejected` means nothing changed +- `register` keeps a registration only after its pass finished, so an exception in a host call keeps nothing +- `rows=None` writes each entry into whichever table is published at that moment and never reports a swap +- `fetch_blocking` runs on a throwaway runtime with an unpooled client so an importing process can still fork. `fetch` is for workers after the fork diff --git a/litellm-rust/crates/python-bridge/src/model_cost/fetch.rs b/litellm-rust/crates/python-bridge/src/model_cost/fetch.rs new file mode 100644 index 00000000000..dff6296cf91 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_cost/fetch.rs @@ -0,0 +1,238 @@ +use std::time::Duration; + +use litellm_http::{ClientVariant, HttpClientConfig}; +use litellm_model_catalog_refresh::{Failed, Fetched, RetryPolicy, S3Access, Start}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyBytes; + +/// A cost map URL and how to connect to it. An `s3://` URL reaches S3 as an AWS SDK finds it in the environment, +/// with LiteLLM's `AWS_REGION_NAME` winning over the region it finds +#[pyclass( + frozen, + name = "NativeCostMapSource", + module = "litellm.rust_bridge._native" +)] +pub(crate) struct NativeCostMapSource { + url: String, + http: HttpClientConfig, +} + +#[pymethods] +impl NativeCostMapSource { + #[new] + #[pyo3(signature = (url, *, use_http_settings))] + fn new(py: Python<'_>, url: String, use_http_settings: bool) -> PyResult { + Ok(Self { + url, + http: crate::http::settings_config(py, use_http_settings)?, + }) + } + + /// Fetches on the shared runtime with a pooled client + fn fetch<'py>( + &self, + py: Python<'py>, + policy: PyRef<'_, NativeFetchPolicy>, + ) -> PyResult> { + let client = crate::http::pool() + .client(&self.http, ClientVariant::Provider) + .map_err(crate::http::client_error)?; + let url = self.url.clone(); + let policy = policy.0; + crate::execution::run_async_value(py, async move { + Ok(Outcome( + litellm_model_catalog_refresh::fetch(&client, &s3(), &url, policy, Start::Fresh) + .await, + )) + }) + } + + /// Fetches on a throwaway runtime with a client of its own. `resume_from` continues a fetch whose first attempt + /// failed retryably + #[pyo3(signature = (policy, resume_from = None))] + fn fetch_blocking( + &self, + py: Python<'_>, + policy: PyRef<'_, NativeFetchPolicy>, + resume_from: Option>, + ) -> PyResult> { + let start = resume_from.map_or(Start::Fresh, |failed| Start::After(failed.0.clone())); + let (config, url, policy) = (self.http.clone(), self.url.clone(), policy.0); + let logger = crate::logger::capture(py); + let fetched = litellm_host_python::release_gil(py, move || { + logger.scope(|| { + let client = crate::http::pool() + .unpooled(&config, ClientVariant::Provider) + .map_err(crate::http::client_error)?; + litellm_host_python::run_isolated(litellm_model_catalog_refresh::fetch( + &client, + &s3(), + &url, + policy, + start, + )) + .map_err(PyErr::from) + }) + })?; + Outcome(fetched).into_pyobject(py).map(Bound::unbind) + } +} + +fn s3() -> S3Access { + S3Access::Environment { + region: std::env::var("AWS_REGION_NAME").ok(), + } +} + +#[pyclass( + frozen, + name = "NativeFetchPolicy", + module = "litellm.rust_bridge._native" +)] +pub(crate) struct NativeFetchPolicy(RetryPolicy); + +#[pymethods] +impl NativeFetchPolicy { + #[new] + /// Retries wait an exponential backoff with jitter from `min_delay_seconds` up to `max_wait_seconds`, or the + /// server's `Retry-After` up to that cap + #[pyo3(signature = (*, max_attempts, max_wait_seconds, timeout_seconds, min_delay_seconds))] + fn new( + max_attempts: u32, + max_wait_seconds: f64, + timeout_seconds: f64, + min_delay_seconds: f64, + ) -> PyResult { + if max_attempts == 0 { + return Err(PyValueError::new_err("max_attempts must be at least 1")); + } + let seconds = |name: &str, value: f64| { + Duration::try_from_secs_f64(value).map_err(|_| { + PyValueError::new_err(format!("{name} must be a non-negative number of seconds")) + }) + }; + let min_delay = seconds("min_delay_seconds", min_delay_seconds)?; + let max_delay = seconds("max_wait_seconds", max_wait_seconds)?; + if min_delay > max_delay { + return Err(PyValueError::new_err( + "min_delay_seconds must not exceed max_wait_seconds", + )); + } + Ok(Self(RetryPolicy { + max_attempts, + timeout: seconds("timeout_seconds", timeout_seconds)?, + min_delay, + max_delay, + })) + } +} + +#[pyclass(frozen, name = "NativeFetched", module = "litellm.rust_bridge._native")] +pub(crate) struct NativeFetched { + #[pyo3(get)] + body: Py, + #[pyo3(get)] + etag: Option, +} + +#[pyclass( + frozen, + skip_from_py_object, + name = "NativeFetchFailed", + module = "litellm.rust_bridge._native" +)] +pub(crate) struct NativeFetchFailed(Failed); + +#[pymethods] +impl NativeFetchFailed { + #[getter] + fn reason(&self) -> &str { + &self.0.reason + } + + /// Another attempt could succeed. Only the attempt limit ended the fetch + #[getter] + fn retryable(&self) -> bool { + self.0.retryable + } + + #[getter] + fn retry_after(&self) -> Option { + self.0.retry_after.map(|wait| wait.as_secs_f64()) + } +} + +struct Outcome(Result); + +impl<'py> IntoPyObject<'py> for Outcome { + type Target = PyAny; + type Output = Bound<'py, PyAny>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> PyResult { + match self.0 { + Ok(fetched) => Bound::new( + py, + NativeFetched { + body: PyBytes::new(py, &fetched.body).unbind(), + etag: fetched.etag, + }, + ) + .map(Bound::into_any), + Err(failed) => Bound::new(py, NativeFetchFailed(failed)).map(Bound::into_any), + } + } +} + +#[cfg(test)] +mod tests { + use std::io::{BufRead, BufReader, Write}; + use std::net::TcpListener; + + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::equal_bounds(2.0, 2.0, true)] + #[case::min_below_max(2.0, 30.0, true)] + #[case::min_above_max(31.0, 30.0, false)] + fn a_policy_accepts_a_min_delay_only_up_to_the_max_wait( + #[case] min_delay_seconds: f64, + #[case] max_wait_seconds: f64, + #[case] accepted: bool, + ) { + Python::initialize(); + let policy = NativeFetchPolicy::new(3, max_wait_seconds, 5.0, min_delay_seconds); + assert_eq!(policy.is_ok(), accepted); + } + + #[rstest] + fn a_blocking_fetch_hands_python_the_document_without_starting_the_shared_runtime() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/cost.json", listener.local_addr().unwrap()); + std::thread::spawn(move || { + let mut stream = listener.incoming().next().unwrap().unwrap(); + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut line = String::new(); + while reader.read_line(&mut line).unwrap() > 2 { + line.clear(); + } + stream + .write_all(b"HTTP/1.1 200 OK\r\nETag: \"v1\"\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}") + .unwrap(); + }); + Python::initialize(); + Python::attach(|py| { + let source = NativeCostMapSource::new(py, url, false).unwrap(); + let policy = + Bound::new(py, NativeFetchPolicy::new(3, 30.0, 5.0, 2.0).unwrap()).unwrap(); + let outcome = source.fetch_blocking(py, policy.borrow(), None).unwrap(); + let fetched = outcome.bind(py).cast::().unwrap().get(); + assert_eq!(fetched.body.bind(py).as_bytes(), b"{}"); + assert_eq!(fetched.etag.as_deref(), Some("\"v1\"")); + }); + assert!(!litellm_host_python::runtime_started()); + } +} diff --git a/litellm-rust/crates/python-bridge/src/model_cost/host.rs b/litellm-rust/crates/python-bridge/src/model_cost/host.rs new file mode 100644 index 00000000000..25fbbefbb73 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_cost/host.rs @@ -0,0 +1,81 @@ +//! Calls into `litellm.rust_bridge.host.model_cost` + +use pyo3::call::PyCallArgs; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyTuple}; + +pub(crate) const MODULE: &str = "litellm.rust_bridge.host.model_cost"; + +/// Where a registration lands in the published table and the row it starts from +#[derive(FromPyObject)] +pub(crate) struct Landing<'py> { + #[pyo3(attribute)] + pub(crate) key: String, + #[pyo3(attribute)] + pub(crate) base: Bound<'py, PyAny>, + #[pyo3(attribute)] + pub(crate) matched_builtin: bool, +} + +fn call<'py>( + py: Python<'py>, + name: &str, + args: impl PyCallArgs<'py>, +) -> PyResult> { + py.import(MODULE)?.getattr(name)?.call1(args) +} + +/// The row object every name in `names` shares, the owner first +pub(crate) fn entry<'py>( + fields: Bound<'py, PyAny>, + names: &[String], +) -> PyResult> { + let py = fields.py(); + call(py, "entry", (fields, PyTuple::new(py, names)?)) +} + +/// Where `registration` lands in `rows`, `None` when `rows` is no longer the published table. `rows` of `None` is the +/// published table +pub(crate) fn resolve<'py>( + rows: Option<&Bound<'py, PyDict>>, + key: &str, + registration: &Bound<'py, PyAny>, + custom_llm_provider: Option<&str>, +) -> PyResult>> { + call( + registration.py(), + "resolve", + (rows, key, registration, custom_llm_provider), + )? + .extract() +} + +/// The row `registration` writes over `current` +pub(crate) fn merged<'py>( + current: Option<&Bound<'py, PyAny>>, + landing: &Landing<'py>, + registration: &Bound<'py, PyAny>, +) -> PyResult> { + call( + registration.py(), + "merged", + (current, &landing.base, registration), + ) +} + +/// The registration kept for a key: `registration` merged into the one kept before it +pub(crate) fn kept<'py>( + previous: Option<&Bound<'py, PyDict>>, + registration: &Bound<'py, PyAny>, +) -> PyResult> { + Ok(call(registration.py(), "kept", (previous, registration))?.cast_into::()?) +} + +pub(crate) fn published(py: Python<'_>) -> PyResult> { + Ok(call(py, "published", ())?.cast_into::()?) +} + +/// Writes `written` into `rows`, the published table, through the host, which tracks the table's version +pub(crate) fn write(rows: &Bound<'_, PyDict>, written: &Bound<'_, PyDict>) -> PyResult<()> { + call(rows.py(), "write", (rows, written)).map(drop) +} diff --git a/litellm-rust/crates/python-bridge/src/model_cost/live.rs b/litellm-rust/crates/python-bridge/src/model_cost/live.rs new file mode 100644 index 00000000000..0b7f071127d --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_cost/live.rs @@ -0,0 +1,176 @@ +use std::borrow::Cow; + +use litellm_model_catalog::Rows; +use pyo3::intern; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +/// The dict Python last handed in as `litellm.model_cost`, read under the GIL on every query +pub(crate) struct LiveDict(Py); + +/// One row exactly as the dict holds it +pub(crate) struct LiveRow(Py); + +impl Clone for LiveRow { + fn clone(&self) -> Self { + Python::attach(|py| Self(self.0.clone_ref(py))) + } +} + +impl LiveDict { + pub(crate) fn new(dict: &Bound<'_, PyDict>) -> Self { + Self(dict.clone().unbind()) + } + + pub(crate) fn is(&self, dict: &Bound<'_, PyDict>) -> bool { + self.0.bind(dict.py()).is(dict) + } +} + +impl Rows for LiveDict { + type Row = LiveRow; + /// The dict's address, length and last key + type Stamp = (usize, usize, Option); + + fn fold(name: &str) -> String { + name.to_lowercase() + } + + fn stamp(&self) -> Self::Stamp { + Python::attach(|py| { + let rows = self.0.bind(py); + let last = rows + .call_method0(intern!(py, "__reversed__")) + .and_then(|keys| keys.try_iter()) + .ok() + .and_then(|mut keys| keys.next()) + .and_then(Result::ok) + .and_then(|key| key.extract::().ok()); + (self.0.as_ptr() as usize, rows.len(), last) + }) + } + + fn contains(&self, key: &str) -> bool { + Python::attach(|py| self.0.bind(py).contains(key).unwrap_or(false)) + } + + fn row(&self, key: &str) -> Option> { + Python::attach(|py| { + let row = self.0.bind(py).get_item(key).ok().flatten()?; + Some(Cow::Owned(LiveRow(row.unbind()))) + }) + } + + /// Groups each key by its row's `litellm_provider` when that is a string + fn scan(&self, visit: &mut dyn FnMut(&str, Option<&str>)) { + Python::attach(|py| { + for (key, row) in self.0.bind(py).iter() { + let Ok(key) = key.extract::() else { + continue; + }; + let provider = row + .cast::() + .ok() + .and_then(|row| row.get_item("litellm_provider").ok().flatten()) + .and_then(|provider| provider.extract::().ok()); + visit(&key, provider.as_deref()); + } + }); + } +} + +#[cfg(test)] +mod tests { + use litellm_model_catalog::Store; + use pyo3::ffi::c_str; + use rstest::rstest; + + use super::*; + + fn dict<'py>(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> { + py.eval(source, None, None) + .unwrap() + .cast_into::() + .unwrap() + } + + #[rstest] + fn a_key_added_to_the_dict_is_found_on_the_next_query() { + Python::initialize(); + Python::attach(|py| { + let rows = dict(py, c_str!("{'gpt-4o': {'litellm_provider': 'openai'}}")); + let store = Store::new(LiveDict::new(&rows)); + assert_eq!(store.find_key("GPT-4O"), Some("gpt-4o".to_owned())); + rows.set_item("Claude-X", PyDict::new(py)).unwrap(); + assert_eq!(store.find_key("claude-x"), Some("Claude-X".to_owned())); + }); + } + + #[rstest] + fn a_removed_key_is_never_returned() { + Python::initialize(); + Python::attach(|py| { + let rows = dict( + py, + c_str!("{'gpt-4o': {'litellm_provider': 'openai'}, 'o3': {}}"), + ); + let store = Store::new(LiveDict::new(&rows)); + assert_eq!(store.keys_in("openai"), ["gpt-4o"]); + rows.del_item("gpt-4o").unwrap(); + rows.set_item("o4", PyDict::new(py)).unwrap(); + assert_eq!(store.find_key("GPT-4O"), None); + assert!(store.keys_in("openai").is_empty()); + }); + } + + #[rstest] + fn a_key_swapped_in_for_a_removed_one_is_found() { + Python::initialize(); + Python::attach(|py| { + let rows = dict(py, c_str!("{'gpt-4o': {}, 'Lit-Gone': {}}")); + let store = Store::new(LiveDict::new(&rows)); + assert_eq!(store.find_key("lit-gone"), Some("Lit-Gone".to_owned())); + rows.del_item("Lit-Gone").unwrap(); + rows.set_item("Lit-New", PyDict::new(py)).unwrap(); + assert_eq!(store.find_key("lit-new"), Some("Lit-New".to_owned())); + assert_eq!(store.find_key("lit-gone"), None); + }); + } + + #[rstest] + fn two_keys_folding_alike_resolve_to_the_later_one() { + Python::initialize(); + Python::attach(|py| { + let rows = dict(py, c_str!("{'Model': {}, 'MODEL': {}}")); + assert_eq!( + Store::new(LiveDict::new(&rows)).find_key("model"), + Some("MODEL".to_owned()) + ); + }); + } + + #[rstest] + fn rows_are_the_objects_the_dict_holds() { + Python::initialize(); + Python::attach(|py| { + let rows = dict(py, c_str!("{'gpt-4o': {'litellm_provider': 'openai'}}")); + let store = Store::new(LiveDict::new(&rows)); + let row = store.get("gpt-4o").unwrap().into_owned(); + assert!(row.0.bind(py).is(rows.get_item("gpt-4o").unwrap().unwrap())); + }); + } + + #[rstest] + fn providers_group_only_string_providers() { + Python::initialize(); + Python::attach(|py| { + let rows = dict( + py, + c_str!( + "{'a': {'litellm_provider': 'p'}, 'b': {'litellm_provider': None}, 'c': 1, 1: {'litellm_provider': 'p'}}" + ), + ); + assert_eq!(Store::new(LiveDict::new(&rows)).keys_in("p"), ["a"]); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/model_cost/mod.rs b/litellm-rust/crates/python-bridge/src/model_cost/mod.rs new file mode 100644 index 00000000000..99451033122 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_cost/mod.rs @@ -0,0 +1,497 @@ +mod fetch; +mod host; +mod live; +mod table; + +use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard}; + +use litellm_host_python::to_py; +use litellm_model_catalog::{ + CostMap, IntegrityLimits, Landed, Pass, Placed, Provenance, RegistrationTable, Registrations, + Store, write_pass, +}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +pub(crate) use fetch::{NativeCostMapSource, NativeFetchFailed, NativeFetchPolicy, NativeFetched}; +use live::LiveDict; + +/// The installed document and the `register_model` calls kept above it, each as the dict Python merged +struct Inputs { + cost_map: CostMap, + registrations: Registrations>, +} + +impl Inputs { + fn clone_ref(&self, py: Python<'_>) -> Self { + Self { + cost_map: self.cost_map.clone(), + registrations: self.registrations.map(|kept| kept.clone_ref(py)), + } + } +} + +/// The cost map's inputs and the name index over `litellm.model_cost` +#[pyclass( + frozen, + name = "NativeModelCostMap", + module = "litellm.rust_bridge._native" +)] +pub(crate) struct NativeModelCostMap { + inputs: RwLock, + limits: IntegrityLimits, + index: RwLock>>>, +} + +#[pyclass( + frozen, + name = "NativeInstalled", + module = "litellm.rust_bridge._native" +)] +pub(crate) struct NativeInstalled { + /// The new live table, the document without registrations + #[pyo3(get)] + rows: Py, + /// How many keys the document itself serves, aliases included and registrations left out + #[pyo3(get)] + model_count: usize, + /// The document's `fallback_generalizations` block as written, `None` when it has none + #[pyo3(get)] + fallback_generalizations: Py, +} + +#[pyclass( + frozen, + name = "NativeRejected", + module = "litellm.rust_bridge._native" +)] +pub(crate) struct NativeRejected { + #[pyo3(get)] + reason: String, + /// The document parsed but has fewer models than the integrity limits allow + #[pyo3(get)] + integrity: bool, +} + +/// A finished registration pass, with the registrations no built-in row matched, by registered key +#[pyclass( + frozen, + name = "NativeRegistered", + module = "litellm.rust_bridge._native" +)] +pub(crate) struct NativeRegistered { + #[pyo3(get)] + unmatched: Py, +} + +/// A registration pass that stopped because the table it was handed is no longer the published one +#[pyclass( + frozen, + name = "NativeTableSwapped", + module = "litellm.rust_bridge._native" +)] +pub(crate) struct NativeTableSwapped; + +/// The inputs at one moment, for a caller that has to put them back, such as test isolation +#[pyclass( + frozen, + name = "NativeModelCostMapState", + module = "litellm.rust_bridge._native" +)] +pub(crate) struct NativeModelCostMapState(Inputs); + +#[pyclass( + frozen, + name = "NativeProvenance", + module = "litellm.rust_bridge._native" +)] +pub(crate) struct NativeProvenance { + #[pyo3(get)] + revision: String, + #[pyo3(get)] + etag: Option, +} + +#[pymethods] +impl NativeModelCostMap { + /// Starts from the bundled document. A fetched document with fewer than `min_model_count` models, or fewer than + /// `max_shrink_ratio` times the bundled count, is rejected + #[staticmethod] + fn bundled(min_model_count: i64, max_shrink_ratio: f64) -> PyResult { + let min_model_count = usize::try_from(min_model_count).map_err(|_| { + PyValueError::new_err(format!( + "min_model_count must be 0 or more, got {min_model_count}" + )) + })?; + let limits = IntegrityLimits::new(min_model_count, max_shrink_ratio).ok_or_else(|| { + PyValueError::new_err(format!( + "max_shrink_ratio must be between 0 and 1, got {max_shrink_ratio}" + )) + })?; + Ok(Self { + inputs: RwLock::new(Inputs { + cost_map: CostMap::bundled(), + registrations: Registrations::default(), + }), + limits, + index: RwLock::new(None), + }) + } + + /// The live table built from a fetched document, or why it was rejected, in which case nothing changed + #[pyo3(signature = (body, etag = None))] + fn install(&self, py: Python<'_>, body: &[u8], etag: Option) -> PyResult> { + match crate::logger::capture(py).scope(|| CostMap::read(body, etag, self.limits)) { + Ok(cost_map) => self.install_document(py, cost_map), + Err(error) => rejected(py, &error), + } + } + + /// A caller's document as a table, read without size limits and installed nowhere + #[staticmethod] + fn finalize(py: Python<'_>, body: &[u8]) -> PyResult> { + match crate::logger::capture(py) + .scope(|| CostMap::read(body, None, IntegrityLimits::UNCHECKED)) + { + Ok(cost_map) => Ok(Py::new(py, built(py, &cost_map)?)?.into_any()), + Err(error) => rejected(py, &error), + } + } + + /// The live table built from the bundled document, which is never size checked + fn install_bundled(&self, py: Python<'_>) -> PyResult> { + self.install_document(py, CostMap::bundled()) + } + + /// Writes each entry into `rows`, or into the published table when `rows` is `None` + #[pyo3(signature = (rows, entries, persist, custom_llm_provider = None))] + fn register( + &self, + py: Python<'_>, + rows: Option<&Bound<'_, PyDict>>, + entries: &Bound<'_, PyDict>, + persist: bool, + custom_llm_provider: Option<&str>, + ) -> PyResult> { + let entries: Vec<(String, Bound<'_, PyAny>)> = entries + .iter() + .map(|(key, registration)| Ok((key.str()?.to_string(), registration))) + .collect::>()?; + let kept = match persist { + true => self.merged_with_kept(&entries)?, + false => Vec::new(), + }; + let pass = self.write_pass(rows, &entries, custom_llm_provider)?; + if persist { + self.keep(kept, pass.landed()); + } + into_py(py, pass) + } + + /// Writes every kept registration into `rows` again, oldest first + fn reapply(&self, rows: &Bound<'_, PyDict>) -> PyResult> { + let py = rows.py(); + let entries: Vec<(String, Bound<'_, PyAny>)> = self + .read_inputs() + .registrations + .entries() + .map(|(key, kept)| (key.to_owned(), kept.bind(py).clone().into_any())) + .collect(); + let pass = self.write_pass(Some(rows), &entries, None)?; + self.write_inputs().registrations.record(pass.landed()); + into_py(py, pass) + } + + /// The installed row under `key` with the kept registrations that apply to it, without anything the router wrote + fn registered_row<'py>( + &self, + py: Python<'py>, + key: &str, + ) -> PyResult>> { + let (cost_map, applied) = { + let inputs = self.read_inputs(); + let applied: Vec> = inputs + .registrations + .over(key) + .into_iter() + .map(|kept| kept.bind(py).clone()) + .collect(); + (inputs.cost_map.clone(), applied) + }; + table::registered_row(py, &cost_map, key, &applied) + } + + fn registered_keys(&self) -> Vec { + let inputs = self.read_inputs(); + table::registered_keys( + &inputs.cost_map, + inputs.registrations.entries().map(|(key, _)| key), + ) + } + + /// The installed document as a table, without any registration + fn loaded<'py>(&self, py: Python<'py>) -> PyResult> { + let cost_map = self.read_inputs().cost_map.clone(); + table::build(py, &cost_map) + } + + /// The key itself when `rows` has it, otherwise the key that differs from `name` only in case + fn find_key(&self, rows: &Bound<'_, PyDict>, name: &str) -> Option { + self.index_over(rows).find_key(name) + } + + /// Keys whose row's `litellm_provider` is exactly `provider`, in row order + fn keys_in(&self, rows: &Bound<'_, PyDict>, provider: &str) -> Vec { + self.index_over(rows).keys_in(provider) + } + + /// Drops the name index, for a caller that changed keys in a way the dict's length does not show + fn invalidate(&self) { + if let Some(index) = self + .index + .read() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + { + index.invalidate(); + } + } + + fn snapshot(&self, py: Python<'_>) -> NativeModelCostMapState { + NativeModelCostMapState(self.read_inputs().clone_ref(py)) + } + + /// Puts back the document, registrations and provenance a `snapshot` saw + fn restore(&self, py: Python<'_>, state: PyRef<'_, NativeModelCostMapState>) { + *self.write_inputs() = state.0.clone_ref(py); + self.invalidate(); + } + + #[getter] + fn provenance(&self) -> NativeProvenance { + let Provenance { revision, etag } = self.read_inputs().cost_map.provenance().clone(); + NativeProvenance { revision, etag } + } +} + +impl NativeModelCostMap { + fn read_inputs(&self) -> RwLockReadGuard<'_, Inputs> { + self.inputs.read().unwrap_or_else(PoisonError::into_inner) + } + + fn write_inputs(&self) -> RwLockWriteGuard<'_, Inputs> { + self.inputs.write().unwrap_or_else(PoisonError::into_inner) + } + + fn install_document(&self, py: Python<'_>, cost_map: CostMap) -> PyResult> { + let installed = built(py, &cost_map)?; + self.write_inputs().cost_map = cost_map; + *self.index.write().unwrap_or_else(PoisonError::into_inner) = + Some(Arc::new(Store::new(LiveDict::new(installed.rows.bind(py))))); + Ok(Py::new(py, installed)?.into_any()) + } + + /// Each entry merged into the registration kept for its key + fn merged_with_kept<'py>( + &self, + entries: &[(String, Bound<'py, PyAny>)], + ) -> PyResult)>> { + entries + .iter() + .map(|(key, registration)| { + let previous = self + .read_inputs() + .registrations + .get(key) + .map(|kept| kept.bind(registration.py()).clone()); + Ok((key.clone(), host::kept(previous.as_ref(), registration)?)) + }) + .collect() + } + + fn keep(&self, kept: Vec<(String, Bound<'_, PyDict>)>, landed: &[Landed]) { + let mut inputs = self.write_inputs(); + for (key, registration) in kept { + inputs.registrations.register(&key, registration.unbind()); + } + inputs.registrations.record(landed); + } + + fn write_pass<'a, 'py>( + &self, + rows: Option<&Bound<'py, PyDict>>, + entries: &'a [(String, Bound<'py, PyAny>)], + custom_llm_provider: Option<&str>, + ) -> PyResult>> { + let mut table = PublishedTable { + map: self, + rows, + custom_llm_provider, + }; + write_pass(&mut table, entries) + } + + /// The index over `rows`, rebuilt when `rows` is not the dict it was built over + fn index_over(&self, rows: &Bound<'_, PyDict>) -> Arc> { + let current = self + .index + .read() + .unwrap_or_else(PoisonError::into_inner) + .clone(); + if let Some(index) = current.filter(|index| index.rows().is(rows)) { + return index; + } + let index = Arc::new(Store::new(LiveDict::new(rows))); + *self.index.write().unwrap_or_else(PoisonError::into_inner) = Some(Arc::clone(&index)); + index + } +} + +/// The dict Python publishes as `litellm.model_cost`, written through the host +struct PublishedTable<'a, 'py> { + map: &'a NativeModelCostMap, + rows: Option<&'a Bound<'py, PyDict>>, + custom_llm_provider: Option<&'a str>, +} + +impl<'py> RegistrationTable for PublishedTable<'_, 'py> { + type Registration = Bound<'py, PyAny>; + type Error = PyErr; + + fn place(&mut self, key: &str, registration: &Self::Registration) -> PyResult> { + let Some(landing) = host::resolve(self.rows, key, registration, self.custom_llm_provider)? + else { + return Ok(None); + }; + let rows = match self.rows { + Some(rows) => rows.clone(), + None => host::published(registration.py())?, + }; + table::write(&rows, &landing, registration)?; + self.map.invalidate(); + Ok(Some(Placed { + key: landing.key, + matched_builtin: landing.matched_builtin, + })) + } +} + +fn built(py: Python<'_>, cost_map: &CostMap) -> PyResult { + let rows = table::build(py, cost_map)?; + let fallback_generalizations = match cost_map.fallback_generalizations() { + Some(block) => to_py(py, block)?, + None => py.None(), + }; + Ok(NativeInstalled { + model_count: rows.len(), + rows: rows.unbind(), + fallback_generalizations, + }) +} + +fn rejected(py: Python<'_>, error: &litellm_model_catalog::Error) -> PyResult> { + let rejected = NativeRejected { + reason: litellm_error::Report::new(error).to_string(), + integrity: error.is_integrity(), + }; + Ok(Py::new(py, rejected)?.into_any()) +} + +fn into_py<'py>(py: Python<'py>, pass: Pass<'_, Bound<'py, PyAny>>) -> PyResult> { + match pass { + Pass::Swapped { .. } => Ok(Py::new(py, NativeTableSwapped)?.into_any()), + Pass::Finished { unmatched, .. } => { + let table = PyDict::new(py); + for (key, registration) in unmatched { + table.set_item(key, registration)?; + } + let registered = NativeRegistered { + unmatched: table.unbind(), + }; + Ok(Py::new(py, registered)?.into_any()) + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::mpsc; + use std::time::Duration; + + use pyo3::ffi::c_str; + use rstest::rstest; + + use super::*; + + const HOST: &std::ffi::CStr = c_str!( + r#" +from types import SimpleNamespace + +class Row(dict): + names = () + +found = [] + +def resolve(rows, key, registration, custom_llm_provider): + found.append(cost_map.find_key(rows, key)) + return SimpleNamespace(key=key, base={}, matched_builtin=True) + +def merged(current, base, registration): + return Row({**(current or {}), **base, **registration}) + +def write(rows, written): + rows.update(written) +"# + ); + + #[rstest] + fn a_host_lookup_that_reaches_find_key_on_the_same_map_finishes() { + Python::initialize(); + let (sender, receiver) = mpsc::channel(); + std::thread::spawn(move || { + let outcome = Python::attach(|py| -> PyResult<(Vec>, f64)> { + let cost_map = Py::new(py, NativeModelCostMap::bundled(0, 0.0)?)?; + let module_name = std::ffi::CString::new(host::MODULE).unwrap(); + let host = + PyModule::from_code(py, HOST, c_str!("host_model_cost.py"), &module_name)?; + host.setattr("cost_map", &cost_map)?; + let modules = py.import("sys")?.getattr("modules")?; + let packages = ["litellm", "litellm.rust_bridge", "litellm.rust_bridge.host"]; + for package in packages { + modules.set_item(package, PyModule::new(py, package)?)?; + } + modules.set_item(host::MODULE, &host)?; + let rows = PyDict::new(py); + rows.set_item("Lit-Reentry", PyDict::new(py))?; + let entry = PyDict::new(py); + entry.set_item("input_cost_per_token", 1.5)?; + let entries = PyDict::new(py); + entries.set_item("lit-reentry", entry)?; + let registered = cost_map + .bind(py) + .call_method1("register", (&rows, &entries, false)); + for name in [host::MODULE].into_iter().chain(packages) { + modules.del_item(name)?; + } + registered?; + let price = rows + .get_item("lit-reentry")? + .expect("the registration landed") + .get_item("input_cost_per_token")? + .extract()?; + Ok((host.getattr("found")?.extract()?, price)) + }); + sender + .send(outcome.map_err(|error| error.to_string())) + .unwrap(); + }); + + let (found, price) = receiver + .recv_timeout(Duration::from_secs(30)) + .expect("register finished instead of deadlocking on its own lock") + .unwrap(); + assert_eq!(found, [Some("Lit-Reentry".to_owned())]); + assert!((price - 1.5).abs() < f64::EPSILON); + } +} diff --git a/litellm-rust/crates/python-bridge/src/model_cost/table.rs b/litellm-rust/crates/python-bridge/src/model_cost/table.rs new file mode 100644 index 00000000000..67b5f581b49 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/model_cost/table.rs @@ -0,0 +1,166 @@ +//! Builds `litellm.model_cost` from an installed document + +use std::borrow::Cow; +use std::collections::HashMap; + +use indexmap::IndexSet; +use litellm_host_python::to_py; +use litellm_model_catalog::wire::Fields; +use litellm_model_catalog::{CostMap, FALLBACK_GENERALIZATIONS, Written}; +use pyo3::prelude::*; +use pyo3::types::PyDict; +use serde_json::Value; + +use super::host::{self, Landing}; + +const ALIASES: &str = "aliases"; + +enum Served<'a> { + Model, + AliasOf(&'a str), + /// A top-level value that is not a model row, such as `sample_spec` + Section(&'a Value), +} + +/// Every served key in dict order. An `alias_of` row repeats among the aliases and keeps its document position +fn served(cost_map: &CostMap) -> impl Iterator)> { + let catalog = cost_map.catalog(); + let documented = catalog + .written() + .filter(|(key, _)| *key != FALLBACK_GENERALIZATIONS) + .filter_map(|(key, written)| { + let served = match written { + Written::Section(value) => Served::Section(value), + Written::Entry(_) => match catalog.aliases().get(key) { + Some(owner) => Served::AliasOf(owner), + None => catalog.get(key).map(|_| Served::Model)?, + }, + }; + Some((key, served)) + }); + let listed = cost_map.alias_groups().flat_map(|(owner, names)| { + names + .iter() + .skip(1) + .map(move |alias| (alias.as_str(), Served::AliasOf(owner))) + }); + documented.chain(listed) +} + +/// The installed document as a new dict +pub(crate) fn build<'py>(py: Python<'py>, cost_map: &CostMap) -> PyResult> { + let table = PyDict::new(py); + let mut rows = HashMap::new(); + for (key, served) in served(cost_map) { + let value = match served { + Served::Model => row(py, cost_map, key, &mut rows)?, + Served::AliasOf(owner) => row(py, cost_map, owner, &mut rows)?, + Served::Section(value) => to_py(py, value)?.into_bound(py), + }; + table.set_item(key, value)?; + } + Ok(table) +} + +fn row<'a, 'py>( + py: Python<'py>, + cost_map: &'a CostMap, + owner: &'a str, + rows: &mut HashMap<&'a str, Bound<'py, PyAny>>, +) -> PyResult> { + if let Some(row) = rows.get(owner) { + return Ok(row.clone()); + } + let fields = cost_map + .catalog() + .get(owner) + .expect("a served model or alias owner is a catalog row"); + let fields = to_py(py, without_alias_list(fields).as_ref())?.into_bound(py); + let row = match cost_map.names(owner) { + [] => fields, + names => host::entry(fields, names)?, + }; + rows.insert(owner, row.clone()); + Ok(row) +} + +fn without_alias_list(fields: &Fields) -> Cow<'_, Fields> { + if fields.get(ALIASES).is_none_or(Value::is_null) { + return Cow::Borrowed(fields); + } + Cow::Owned( + fields + .iter() + .filter(|(name, _)| *name != ALIASES) + .map(|(name, value)| (name.clone(), value.clone())) + .collect(), + ) +} + +/// The installed row under `key` with `applied` written over it in order, `None` when neither has one +pub(crate) fn registered_row<'py>( + py: Python<'py>, + cost_map: &CostMap, + key: &str, + applied: &[Bound<'py, PyDict>], +) -> PyResult>> { + let catalog = cost_map.catalog(); + let installed = catalog + .get(key) + .or_else(|| match catalog.section(key) { + Some(Value::Object(fields)) if key != FALLBACK_GENERALIZATIONS => Some(fields), + _ => None, + }) + .map(without_alias_list); + let row = match (installed, applied) { + (None, []) => return Ok(None), + (Some(fields), _) => to_py(py, fields.as_ref())? + .into_bound(py) + .cast_into::()?, + (None, _) => PyDict::new(py), + }; + for registration in applied { + row.update(registration.as_mapping())?; + } + Ok(Some(row)) +} + +/// The keys `registered_row` answers for, installed keys first +pub(crate) fn registered_keys<'a>( + cost_map: &'a CostMap, + registered: impl Iterator, +) -> Vec { + let served: IndexSet<&str> = served(cost_map).map(|(key, _)| key).collect(); + let only_registered: Vec<&str> = registered.filter(|key| !served.contains(key)).collect(); + served + .into_iter() + .chain(only_registered) + .map(str::to_owned) + .collect() +} + +/// Writes `registration` over the row under `landing.key`, and under every name that shares that row object +pub(crate) fn write<'py>( + rows: &Bound<'py, PyDict>, + landing: &Landing<'py>, + registration: &Bound<'py, PyAny>, +) -> PyResult<()> { + let current = rows.get_item(&landing.key)?; + let row = host::merged(current.as_ref(), landing, registration)?; + let names: Vec = row.getattr("names")?.extract()?; + let sharing: Vec<&String> = names + .iter() + .filter(|name| { + let served = rows.get_item(name.as_str()).ok().flatten(); + served + .zip(current.as_ref()) + .is_some_and(|(served, current)| served.is(current)) + }) + .collect(); + let written = PyDict::new(rows.py()); + written.set_item(&landing.key, &row)?; + sharing + .into_iter() + .try_for_each(|name| written.set_item(name, &row))?; + host::write(rows, &written) +} diff --git a/litellm-rust/crates/python-bridge/src/preflight.rs b/litellm-rust/crates/python-bridge/src/preflight.rs index bc692498880..2a07723b94c 100644 --- a/litellm-rust/crates/python-bridge/src/preflight.rs +++ b/litellm-rust/crates/python-bridge/src/preflight.rs @@ -7,7 +7,7 @@ use pyo3::{ }; use strum::{IntoStaticStr, VariantArray}; -const MODULE: &str = "litellm.rust_bridge.preflight"; +const MODULE: &str = "litellm.rust_bridge.host.preflight"; /// The litellm globals the preflight still reads through Python. `preflight_contract.json` /// pins each function's parameters on both sides. @@ -115,7 +115,7 @@ mod tests { /// tests that install it run one at a time. static PREFLIGHT_MODULE: Mutex<()> = Mutex::new(()); - /// A fresh stand-in for `litellm.rust_bridge.preflight` that records every call, then + /// A fresh stand-in for `litellm.rust_bridge.host.preflight` that records every call, then /// `script` run against it with the module bound as `preflight`. fn preflight_stubs<'py>(py: Python<'py>, script: &std::ffi::CStr) -> Bound<'py, PyDict> { let locals = PyDict::new(py); @@ -124,15 +124,15 @@ mod tests { import sys import types -for name in ('litellm', 'litellm.rust_bridge'): +for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host'): sys.modules.setdefault(name, types.ModuleType(name)) -preflight = types.ModuleType('litellm.rust_bridge.preflight') +preflight = types.ModuleType('litellm.rust_bridge.host.preflight') preflight.warnings = [] preflight.checked = [] preflight.credential_list = lambda: [] preflight.warn_unknown_credential = lambda name, loaded: preflight.warnings.append((name, loaded)) preflight.check_limits = lambda kwargs: preflight.checked.append(kwargs) -sys.modules['litellm.rust_bridge.preflight'] = preflight +sys.modules['litellm.rust_bridge.host.preflight'] = preflight ", Some(&locals), Some(&locals), diff --git a/litellm-rust/crates/python-bridge/src/python_settings.rs b/litellm-rust/crates/python-bridge/src/python_settings.rs index eacec6859b5..1cb1813049c 100644 --- a/litellm-rust/crates/python-bridge/src/python_settings.rs +++ b/litellm-rust/crates/python-bridge/src/python_settings.rs @@ -13,6 +13,8 @@ pub(crate) enum PythonSettings { UrlPolicy, #[strum(serialize = "provider_defaults")] ProviderDefaults, + #[strum(serialize = "router_tiers")] + RouterTiers, #[strum(serialize = "secret_manager")] SecretManager, #[strum(serialize = "secret_manager_binding")] @@ -24,10 +26,15 @@ pub(crate) struct Snapshot<'py> { value: Bound<'py, PyAny>, } -impl Snapshot<'_> { +impl<'py> Snapshot<'py> { pub(crate) fn read(&self, spec: &FieldSpec) -> Result { spec.read(&self.value, self.group.into()) } + + pub(crate) fn attribute(&self, name: &str) -> PyResult>> { + let value = self.value.getattr(name)?; + Ok((!value.is_none()).then_some(value)) + } } impl PythonSettings { diff --git a/litellm-rust/crates/python-bridge/src/redis/breaker.rs b/litellm-rust/crates/python-bridge/src/redis/breaker.rs new file mode 100644 index 00000000000..5fdac76cb02 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/breaker.rs @@ -0,0 +1,134 @@ +use std::{ + sync::{Arc, Mutex, PoisonError}, + time::{Duration, Instant}, +}; + +use litellm_circuit_breaker::{Admission, Breaker, Config, Failure}; +use litellm_redis_client::RedisEvents; +use litellm_redis_types::Endpoint; +use pyo3::{exceptions::PyValueError, prelude::*}; + +use crate::redis::{ + client::{RedisSpec, resolved_endpoint}, + events::{ServiceLoggingEvents, state_label}, + hosts, +}; + +/// `litellm.redis.store_settings.RedisBreakerSettings`, filled from `litellm.constants`. +#[derive(FromPyObject)] +pub(crate) struct BreakerSettings { + failure_threshold: u32, + recovery_timeout: f64, + timeout_min_duration: f64, +} + +impl BreakerSettings { + pub(crate) fn config(&self) -> PyResult { + let seconds = |seconds: f64| { + Duration::try_from_secs_f64(seconds) + .map_err(|error| PyValueError::new_err(error.to_string())) + }; + Ok(Config { + failure_threshold: self.failure_threshold, + recovery_timeout: seconds(self.recovery_timeout)?, + timeout_min_duration: seconds(self.timeout_min_duration)?, + }) + } +} + +/// The one health signal for a Redis, shared by its `RedisCache` and its native client, so a Redis the +/// cache found wedged is fast-failed by every store too. +#[pyclass(frozen, name = "NativeRedisBreaker")] +pub(crate) struct NativeRedisBreaker(pub(crate) Breaker); + +impl NativeRedisBreaker { + fn build( + py: Python<'_>, + settings: &BreakerSettings, + endpoint: Option, + ) -> PyResult { + let config = settings.config()?; + let runtime = litellm_host_python::runtime_handle()?; + let events: Arc = + Arc::new(ServiceLoggingEvents::transitions_only(py, runtime)?); + Ok(Self(py.detach(|| { + hosts().breaker(endpoint, config, Some(events)) + }))) + } +} + +#[pymethods] +impl NativeRedisBreaker { + #[new] + fn new(py: Python<'_>, settings: BreakerSettings) -> PyResult { + Self::build(py, &settings, None) + } + + /// The breaker of the Redis server `redis_kwargs` names, which every cache and client on that server shares + /// whatever its db, credentials or namespace. The first caller's settings win. + #[staticmethod] + fn shared( + py: Python<'_>, + redis_kwargs: RedisSpec, + settings: BreakerSettings, + ) -> PyResult { + let endpoint = resolved_endpoint(&redis_kwargs); + Self::build(py, &settings, endpoint) + } + + fn admit(&self) -> Option { + self.0 + .admit(Instant::now()) + .ok() + .map(|admission| NativeAdmission(Mutex::new(Some(admission)))) + } + + fn failed(&self, timeout: bool) { + self.0.failed(failure(timeout), Instant::now()); + } + + #[getter] + fn state(&self) -> &'static str { + state_label(&self.0.state()) + } +} + +/// One admitted call. Settling it twice records nothing more, and dropping an unsettled probe reopens the breaker. +#[pyclass(frozen, name = "NativeAdmission")] +pub(crate) struct NativeAdmission(Mutex>); + +impl NativeAdmission { + fn take(&self) -> Option { + self.0.lock().unwrap_or_else(PoisonError::into_inner).take() + } +} + +#[pymethods] +impl NativeAdmission { + fn succeeded(&self) { + if let Some(admission) = self.take() { + admission.succeeded(); + } + } + + fn failed(&self, timeout: bool) { + if let Some(admission) = self.take() { + admission.failed(failure(timeout), Instant::now()); + } + } +} + +fn failure(timeout: bool) -> Failure { + match timeout { + true => Failure::Timeout, + false => Failure::Hard, + } +} + +#[pyfunction] +pub(crate) fn native_redis_breaker_states(py: Python<'_>) -> Vec<&'static str> { + py.detach(|| hosts().breakers()) + .iter() + .map(|breaker| state_label(&breaker.state())) + .collect() +} diff --git a/litellm-rust/crates/python-bridge/src/redis/client/credentials.rs b/litellm-rust/crates/python-bridge/src/redis/client/credentials.rs new file mode 100644 index 00000000000..6995981a03e --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/client/credentials.rs @@ -0,0 +1,47 @@ +use std::sync::Arc; + +use litellm_host_python::{PythonContext, attach_blocking}; +use litellm_redis_connect::{CredentialSource, CredentialStage, Error, Expiring, SourceFuture}; +use litellm_secret_value::SecretValue; +use pyo3::prelude::*; + +const CREDENTIALS_MODULE: &str = "litellm.rust_bridge.host.redis_credentials"; + +/// A redis-py `CredentialProvider` the SDK caller passed in `cache_kwargs`, asked through the host +/// module the way redis-py's async connection asks it. +pub(crate) struct PythonCredentialSource { + provider: Arc>, + context: PythonContext, +} + +impl PythonCredentialSource { + pub(crate) fn new(provider: Py, context: PythonContext) -> Self { + Self { + provider: Arc::new(provider), + context, + } + } +} + +impl CredentialSource for PythonCredentialSource { + fn fetch(&self) -> SourceFuture<'_> { + let provider = Arc::clone(&self.provider); + let context = self.context.clone(); + Box::pin(async move { + let (username, password) = attach_blocking(context, move |py| { + py.import(CREDENTIALS_MODULE)? + .getattr("credentials")? + .call1((provider.bind(py),))? + .extract::<(Option, String)>() + }) + .await + .and_then(|credentials| credentials) + .map_err(Error::credential(CredentialStage::Provider))?; + Ok(Expiring { + username, + password: SecretValue::new(password), + expires_at: None, + }) + }) + } +} diff --git a/litellm-rust/crates/python-bridge/src/redis/client/mod.rs b/litellm-rust/crates/python-bridge/src/redis/client/mod.rs new file mode 100644 index 00000000000..5e5ab7e6f5a --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/client/mod.rs @@ -0,0 +1,294 @@ +mod credentials; + +use std::{collections::BTreeMap, sync::Arc, time::Duration}; + +use litellm_circuit_breaker::Config as BreakerConfig; +use litellm_core_utils::settings::{Lookup, ProcessEnvironment}; +use litellm_host_python::PythonContext; +use litellm_python_compat::number::{self, SecondsError}; +use litellm_redis_client::{RedisClient, RedisEvents}; +use litellm_redis_connect::{CredentialSource, HostServices, SharedEnvironment}; +use litellm_redis_resolve::{Kwargs, PythonException}; +use litellm_redis_types::{Endpoint, ProviderHandle, Resolution}; +use pyo3::{ + exceptions::{PyOverflowError, PyTypeError, PyValueError}, + prelude::*, + types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyMapping, PyString, PyTuple}, +}; + +use self::credentials::PythonCredentialSource; +use crate::redis::{breaker::BreakerSettings, events::ServiceLoggingEvents, hosts}; + +pyo3::import_exception!(redis.exceptions, RedisClusterException); + +/// The kwargs `_merge_redis_kwargs` returns for a `RedisCache`, which the resolver reads the way +/// redis-py's constructors do. A key whose value is a Python object, such as `retry`, reaches the +/// resolver by name only, so it can name the setting it cannot honor. A `credential_provider` object +/// is also kept, so the connector can ask it for credentials, and reaches the resolver as a handle. +pub(crate) struct RedisSpec { + kwargs: Kwargs, + credential_provider: Option>, +} + +impl<'a, 'py> FromPyObject<'a, 'py> for RedisSpec { + type Error = PyErr; + + fn extract(kwargs: Borrowed<'a, 'py, PyAny>) -> PyResult { + let py = kwargs.py(); + let source = PyDict::new(py); + let mapping = kwargs.cast::()?; + source.update(&mapping)?; + let plain = PyDict::new(py); + let mut host_objects = Vec::new(); + for (name, value) in source.iter() { + match value.is_none() || is_plain(&value) { + true => plain.set_item(&name, value)?, + false => host_objects.push(name.extract::()?), + } + } + let credential_provider = source + .get_item("credential_provider")? + .filter(|provider| !provider.is_none() && !is_plain(provider)) + .map(Bound::unbind); + let fields: BTreeMap = + litellm_host_python::from_py_argument(plain.as_any())?; + let objects = host_objects + .into_iter() + .map(|name| (name, serde_json::Value::Bool(true))); + Ok(Self { + kwargs: litellm_redis_resolve::kwargs(fields.into_iter().chain(objects)), + credential_provider, + }) + } +} + +/// `litellm.redis.store_settings.RedisStoreSettings`, which a `RedisCache` captures when it is built. +#[derive(FromPyObject)] +pub(crate) struct StoreSettings { + redis_kwargs: RedisSpec, + namespace: Option, + default_ttl: f64, + breaker: Option, +} + +fn is_plain(value: &Bound<'_, PyAny>) -> bool { + value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() +} + +/// The one client for a resolved Redis config and credential provider, so every family in this process shares its pool +/// and its breaker. A config redis-py refuses building its client raises the same class here. One it only fails on +/// once a command connects, where `RedisCache` logs every failed call, leaves the family on memory with an error. +fn shared_client( + py: Python<'_>, + spec: RedisSpec, + namespace: Option, + breaker: Option, +) -> PyResult> { + let environment: SharedEnvironment = Arc::new(ProcessEnvironment); + let logger = crate::logger::capture(py); + let resolved = match logger.scope(|| resolve_spec(&spec)) { + Ok(resolution) => { + logger.scope(|| { + for ignored in &resolution.ignored { + litellm_tracing::warn!( + "Redis setting {} is not supported by the Rust client and is ignored: {}", + ignored.setting, + ignored.consequence + ); + } + }); + Resolution { + namespace, + ..resolution + } + } + Err(error) => return unresolved(py, &error), + }; + let runtime = litellm_host_python::runtime_handle()?; + let events: Arc = Arc::new(ServiceLoggingEvents::new(py, runtime.clone())?); + let host = HostServices { + events: Some(events), + http: crate::http::host_client(py, litellm_http::ClientVariant::Provider)?, + environment, + credential_provider: spec + .credential_provider + .as_ref() + .map(|provider| -> PyResult> { + Ok(Arc::new(PythonCredentialSource::new( + provider.clone_ref(py), + PythonContext::capture(py)?, + ))) + }) + .transpose()?, + }; + let connected = py.detach(|| { + let _runtime = runtime.enter(); + hosts().client(resolved, host, breaker) + }); + match connected { + Ok(client) => Ok(Some(client)), + Err(error) => Ok(unconnected(py, &error)), + } +} + +/// The Rust client for one `RedisCache`, built once and handed to every native store, so a store +/// never sees the redis-py kwargs or the namespace. +#[pyclass(frozen, name = "NativeRedisClient")] +pub(crate) struct NativeRedisClient { + pub(crate) client: RedisClient, + /// `RedisCache.default_ttl`, the TTL a store's write takes when its caller gives none. + pub(crate) default_ttl: Duration, +} + +#[pymethods] +impl NativeRedisClient { + /// `None` when redis-py would fail only once a command connects, which leaves every store given it on memory. + #[staticmethod] + fn connect(py: Python<'_>, settings: StoreSettings) -> PyResult> { + let StoreSettings { + redis_kwargs, + namespace, + default_ttl, + breaker, + } = settings; + let default_ttl = number::seconds(default_ttl).map_err(seconds_error)?; + let breaker = breaker.as_ref().map(BreakerSettings::config).transpose()?; + Ok( + shared_client(py, redis_kwargs, namespace, breaker)?.map(|client| Self { + client, + default_ttl, + }), + ) + } + + #[getter] + fn default_ttl_seconds(&self) -> f64 { + self.default_ttl.as_secs_f64() + } +} + +/// The Redis server `spec` names, whatever its db, credentials or namespace, or none when the Rust client cannot reach it. +pub(crate) fn resolved_endpoint(spec: &RedisSpec) -> Option { + resolve_spec(spec) + .ok() + .and_then(|resolution| resolution.target.endpoint()) +} + +// A live registry entry's client keeps its provider alive, so no live entry shares its address +fn resolve_spec(spec: &RedisSpec) -> Result { + let handle = spec + .credential_provider + .as_ref() + .map(|provider| ProviderHandle::new(provider.as_ptr() as usize as u64)); + litellm_redis_resolve::interpret( + &spec.kwargs, + handle, + ProcessEnvironment.get("REDIS_USERNAME").as_deref(), + ) +} + +fn unresolved( + py: Python<'_>, + error: &litellm_redis_resolve::Error, +) -> PyResult> { + let message = || litellm_error::Report::new(error).to_string(); + match error.raised_building_the_python_client() { + Some(PythonException::ValueError) => Err(PyValueError::new_err(message())), + Some(PythonException::TypeError) => Err(PyTypeError::new_err(message())), + Some(PythonException::RedisClusterException) => { + Err(RedisClusterException::new_err(message())) + } + None => Ok(memory_only(py, error)), + } +} + +// redis-py reads TLS files and fetches credentials only once a command connects, so none of these fails building +// its client +fn unconnected(py: Python<'_>, error: &litellm_redis_connect::Error) -> Option { + match error { + litellm_redis_connect::Error::Connect(_) + | litellm_redis_connect::Error::NoHostCredentialProvider + | litellm_redis_connect::Error::ProcessEnvironmentRequired { .. } + | litellm_redis_connect::Error::Credential { .. } + | litellm_redis_connect::Error::CredentialTimeout { .. } + | litellm_redis_connect::Error::ExpiredCredential + | litellm_redis_connect::Error::MissingCredential { .. } => memory_only(py, error), + } +} + +fn memory_only(py: Python<'_>, error: &(dyn std::error::Error + 'static)) -> Option { + let report = litellm_error::Report::new(error).to_string(); + crate::logger::capture(py).scope(|| { + litellm_tracing::error!( + "Redis is not usable from the Rust stores, their state stays in memory: {report}" + ) + }); + None +} + +fn seconds_error(error: SecondsError) -> PyErr { + match error { + SecondsError::Overflow => PyOverflowError::new_err(error.to_string()), + SecondsError::NotANumber => PyValueError::new_err(error.to_string()), + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn kwargs(fields: serde_json::Value) -> Kwargs { + let fields: BTreeMap = serde_json::from_value(fields).unwrap(); + litellm_redis_resolve::kwargs(fields) + } + + #[rstest] + #[case::no_target(json!({"port": 6379}), Some("ValueError"))] + #[case::unenforceable(json!({"host": "redis", "ssl": true, "ssl_validate_ocsp": true}), Some("TypeError"))] + #[case::fails_only_once_a_command_connects( + json!({"host": "redis", "ssl": true, "ssl_cert_reqs": "None"}), + None + )] + fn a_config_raises_where_redis_py_raises_building_its_client( + #[case] fields: serde_json::Value, + #[case] raised: Option<&str>, + ) { + Python::initialize(); + Python::attach(|py| { + let error = litellm_redis_resolve::interpret(&kwargs(fields), None, None).unwrap_err(); + + let outcome = unresolved(py, &error) + .map(|client| client.is_some()) + .map_err(|error| error.get_type(py).name().unwrap().to_string()); + + assert_eq!( + outcome, + raised.map_or(Ok(false), |class| Err(class.to_owned())) + ); + }); + } + + #[rstest] + #[case::overflow(f64::INFINITY, "OverflowError")] + #[case::not_a_number(f64::NAN, "ValueError")] + fn a_default_ttl_python_cannot_make_an_int_raises_like_int( + #[case] seconds: f64, + #[case] raised: &str, + ) { + Python::initialize(); + Python::attach(|py| { + let error = number::seconds(seconds).map_err(seconds_error).unwrap_err(); + + assert_eq!(error.get_type(py).name().unwrap().to_string(), raised); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/redis/events.rs b/litellm-rust/crates/python-bridge/src/redis/events.rs new file mode 100644 index 00000000000..a281d78d5a6 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/events.rs @@ -0,0 +1,403 @@ +use std::{ + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use litellm_circuit_breaker::{State, Transition}; +use litellm_host_python::{PythonContext, attach_blocking}; +use litellm_redis_client::{ + CallEvent, CallTag, FailureKind, Operation, Outcome, RedisEvents, current_tag, tagged, +}; +use pyo3::{prelude::*, types::PyModule}; +use tokio::runtime::Handle; + +/// The asyncio loop a Python caller ran on. `RedisCache` scheduled its service events there with +/// `asyncio.create_task`, so the native events go there too instead of a new loop per event. +#[derive(Clone)] +pub(crate) struct EventLoop(Arc>); + +impl EventLoop { + pub(crate) fn running(py: Python<'_>) -> Option { + pyo3_async_runtimes::get_running_loop(py) + .ok() + .map(|event_loop| Self(Arc::new(event_loop.unbind()))) + } +} + +/// The Python caller of one store operation. Its context carries `with_service_target` and the +/// ambient OTel span, and its `call_type` is the label `RedisCache` reports, so dashboards keep +/// their series. +pub(crate) struct RedisCall { + pub(crate) context: PythonContext, + pub(crate) event_loop: Option, + pub(crate) call_type: &'static str, +} + +impl RedisCall { + pub(crate) fn tag(self) -> CallTag { + Arc::new(self) + } + + pub(crate) async fn scope(self, operation: F) -> F::Output { + tagged(self.tag(), operation).await + } +} + +/// Reports every native Redis call and breaker transition to `ServiceLogging`. +pub(crate) struct ServiceLoggingEvents { + record_call: Arc>, + record_breaker: Arc>, + context: PythonContext, + /// The loop the client was built on, for calls whose caller had none running. + event_loop: Option, + runtime: Handle, +} + +impl ServiceLoggingEvents { + const MODULE: &str = "litellm.rust_bridge.host.redis_events"; + + pub(crate) fn new(py: Python<'_>, runtime: Handle) -> PyResult { + Self::from_module( + py, + &py.import(Self::MODULE)?, + runtime, + EventLoop::running(py), + ) + } + + /// For a breaker, whose transitions never run on the caller's loop, so building it never looks one up. + pub(crate) fn transitions_only(py: Python<'_>, runtime: Handle) -> PyResult { + Self::from_module(py, &py.import(Self::MODULE)?, runtime, None) + } + + fn from_module( + py: Python<'_>, + module: &Bound<'_, PyModule>, + runtime: Handle, + event_loop: Option, + ) -> PyResult { + Ok(Self { + record_call: Arc::new(module.getattr("record_redis_call")?.unbind()), + record_breaker: Arc::new(module.getattr("record_redis_breaker")?.unbind()), + context: PythonContext::capture(py)?, + event_loop, + runtime, + }) + } + + fn report(&self, context: PythonContext, record: Arc>, args: ReportArgs) { + self.runtime.spawn(async move { + let reported = attach_blocking(context, move |py| match args { + ReportArgs::Call(call) => record + .bind(py) + .call1(( + call.event_loop + .as_ref() + .map(|event_loop| event_loop.0.bind(py)), + call.call_type, + call.start_time, + call.end_time, + call.failure, + )) + .map(drop), + ReportArgs::Breaker(state) => record.bind(py).call1((state,)).map(drop), + }) + .await + .and_then(|reported| reported); + if let Err(error) = reported { + tracing::warn!(error = %error, "reporting a Redis event to ServiceLogging failed"); + } + }); + } +} + +/// One call as `record_redis_call` takes it, its times in `time.time()` seconds. +struct CallReport { + event_loop: Option, + call_type: &'static str, + start_time: f64, + end_time: f64, + failure: Option<&'static str>, +} + +enum ReportArgs { + Call(CallReport), + Breaker(&'static str), +} + +/// The wall-clock start and end of a call that just finished after `duration`. +fn wall_clock_span(duration: Duration) -> (f64, f64) { + let end = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + ( + end.saturating_sub(duration).as_secs_f64(), + end.as_secs_f64(), + ) +} + +fn operation_call_type(operation: Operation) -> &'static str { + match operation { + Operation::Command(name) => name, + Operation::Script => "eval", + Operation::Pipeline { .. } => "pipeline", + } +} + +fn failure_label(outcome: Outcome) -> Option<&'static str> { + match outcome { + Outcome::Succeeded => None, + Outcome::Failed(FailureKind::Timeout) => Some("timeout"), + Outcome::Failed(FailureKind::Transport) => Some("transport"), + Outcome::Failed(FailureKind::Server) => Some("server"), + Outcome::Failed(FailureKind::Decode) => Some("decode"), + Outcome::Failed(FailureKind::BreakerOpen) => Some("breaker_open"), + } +} + +pub(crate) fn state_label(state: &State) -> &'static str { + match state { + State::Closed { .. } => "closed", + State::Open { .. } => "open", + State::HalfOpen => "half_open", + } +} + +impl RedisEvents for ServiceLoggingEvents { + fn call(&self, event: &CallEvent) { + let scoped = current_tag().and_then(|tag| tag.downcast::().ok()); + let (context, event_loop, call_type) = match scoped { + Some(call) => ( + call.context.clone(), + call.event_loop.clone().or_else(|| self.event_loop.clone()), + call.call_type, + ), + None => ( + self.context.clone(), + self.event_loop.clone(), + operation_call_type(event.operation), + ), + }; + let (start_time, end_time) = wall_clock_span(event.duration); + self.report( + context, + self.record_call.clone(), + ReportArgs::Call(CallReport { + event_loop, + call_type, + start_time, + end_time, + failure: failure_label(event.outcome), + }), + ); + } + + fn breaker(&self, transition: &Transition) { + self.report( + self.context.clone(), + self.record_breaker.clone(), + ReportArgs::Breaker(state_label(&transition.to)), + ); + } +} + +#[cfg(test)] +mod tests { + use std::{sync::Arc, time::Duration}; + + use litellm_host_python::PythonContext; + use litellm_redis_client::{CallEvent, FailureKind, Operation, Outcome, RedisEvents}; + use pyo3::{ + prelude::*, + types::{PyCFunction, PyModule, PyString}, + }; + use rstest::rstest; + use tokio::runtime::Handle; + + use super::{EventLoop, RedisCall, ServiceLoggingEvents}; + + const RECORDER: &str = " +import contextvars, time +target = contextvars.ContextVar('target', default=None) +seen = [] +def record_redis_call(event_loop, call_type, start_time, end_time, failure): + just_now = abs(time.time() - end_time) < 60 + seen.append((event_loop, call_type, round(end_time - start_time, 3), just_now, failure, target.get())) +def record_redis_breaker(state): + seen.append(('breaker', state)) +"; + + fn stand_in_loop(py: Python<'_>, name: &str) -> EventLoop { + EventLoop(Arc::new(PyString::new(py, name).into_any().unbind())) + } + + fn recorder(name: &str, client_loop: Option<&str>) -> (ServiceLoggingEvents, Py) { + Python::initialize(); + Python::attach(|py| { + let name = std::ffi::CString::new(format!("redis_events_recorder_{name}")).unwrap(); + let module = + PyModule::from_code(py, &std::ffi::CString::new(RECORDER).unwrap(), &name, &name) + .unwrap(); + let events = ServiceLoggingEvents::from_module( + py, + &module, + Handle::current(), + client_loop.map(|client_loop| stand_in_loop(py, client_loop)), + ) + .unwrap(); + (events, module.unbind()) + }) + } + + async fn seen(module: &Py, count: usize) -> String { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + let seen: (usize, String) = Python::attach(|py| { + let seen = module.bind(py).getattr("seen").unwrap(); + (seen.len().unwrap(), seen.repr().unwrap().to_string()) + }); + if seen.0 >= count || tokio::time::Instant::now() > deadline { + return seen.1; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + + fn scoped_call(namespace: &Py, caller_loop: Option<&str>) -> RedisCall { + Python::attach(|py| { + namespace + .bind(py) + .getattr("target") + .unwrap() + .call_method1("set", ("spend_counter",)) + .unwrap(); + RedisCall { + context: PythonContext::capture(py).unwrap(), + event_loop: caller_loop.map(|caller_loop| stand_in_loop(py, caller_loop)), + call_type: "async_increment", + } + }) + } + + #[rstest] + #[case::on_the_callers_loop(Some("caller loop"), "caller loop")] + #[case::on_the_clients_loop_when_the_caller_had_none(None, "client loop")] + #[tokio::test(flavor = "multi_thread")] + async fn a_scoped_call_reports_the_store_labels_and_its_times_in_the_callers_context( + #[case] caller_loop: Option<&str>, + #[case] expected_loop: &str, + ) { + let (events, namespace) = recorder( + expected_loop.replace(' ', "_").as_str(), + Some("client loop"), + ); + let call = scoped_call(&namespace, caller_loop); + let event = CallEvent { + operation: Operation::Command("INCRBYFLOAT"), + duration: Duration::from_millis(250), + outcome: Outcome::Failed(FailureKind::Timeout), + }; + + call.scope(async { events.call(&event) }).await; + + assert_eq!( + seen(&namespace, 1).await, + format!( + "[('{expected_loop}', 'async_increment', 0.25, True, 'timeout', 'spend_counter')]" + ) + ); + } + + #[rstest] + #[case::opened(litellm_circuit_breaker::State::Open { since: std::time::Instant::now() }, "open")] + #[case::probing(litellm_circuit_breaker::State::HalfOpen, "half_open")] + #[tokio::test(flavor = "multi_thread")] + async fn breaker_transitions_report_python_state_names( + #[case] to: litellm_circuit_breaker::State, + #[case] expected: &str, + ) { + let (events, namespace) = recorder(expected, None); + + events.breaker(&litellm_circuit_breaker::Transition { + from: litellm_circuit_breaker::State::HalfOpen, + to, + generation: 1, + }); + + assert_eq!( + seen(&namespace, 1).await, + format!("[('breaker', '{expected}')]") + ); + } + + #[rstest] + #[tokio::test(flavor = "multi_thread")] + async fn a_transition_on_a_thread_without_a_runtime_is_still_reported() { + let (events, namespace) = recorder("plain_thread", None); + + std::thread::spawn(move || { + events.breaker(&litellm_circuit_breaker::Transition { + from: litellm_circuit_breaker::State::HalfOpen, + to: litellm_circuit_breaker::State::HalfOpen, + generation: 1, + }); + }) + .join() + .unwrap(); + + assert_eq!(seen(&namespace, 1).await, "[('breaker', 'half_open')]"); + } + + #[rstest] + #[tokio::test(flavor = "multi_thread")] + async fn an_unscoped_call_falls_back_to_the_command_name_and_the_clients_loop() { + let (events, namespace) = recorder("unscoped", None); + + events.call(&CallEvent { + operation: Operation::Pipeline { ops: 3 }, + duration: Duration::from_millis(5), + outcome: Outcome::Succeeded, + }); + + assert_eq!( + seen(&namespace, 1).await, + "[(None, 'pipeline', 0.005, True, None, None)]" + ); + } + + #[rstest] + fn running_is_the_loop_the_caller_runs_on_and_none_outside_one() { + Python::initialize(); + Python::attach(|py| { + let module = PyModule::from_code( + py, + c"import asyncio\nasync def probe_inside(probe):\n return probe() is asyncio.get_running_loop()\n", + c"event_loop_probe", + c"event_loop_probe", + ) + .unwrap(); + let probe = PyCFunction::new_closure(py, None, None, |_, _| { + Python::attach(|py| -> PyResult>> { + Ok(EventLoop::running(py).map(|event_loop| event_loop.0.clone_ref(py))) + }) + }) + .unwrap(); + let coroutine = module + .getattr("probe_inside") + .unwrap() + .call1((probe,)) + .unwrap(); + + let inside: bool = py + .import("asyncio") + .unwrap() + .call_method1("run", (coroutine,)) + .unwrap() + .extract() + .unwrap(); + + assert!(inside); + assert!(EventLoop::running(py).is_none()); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/redis/mod.rs b/litellm-rust/crates/python-bridge/src/redis/mod.rs new file mode 100644 index 00000000000..808f930dd20 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/mod.rs @@ -0,0 +1,15 @@ +use std::sync::LazyLock; + +use litellm_redis_connect::RedisHosts; + +pub(crate) mod breaker; +pub(crate) mod client; +mod events; +pub(crate) mod store; + +static HOSTS: LazyLock = LazyLock::new(RedisHosts::default); + +/// The Redis servers this process talks to, shared by every `RedisCache` and native store in it. +pub(crate) fn hosts() -> &'static RedisHosts { + &HOSTS +} diff --git a/litellm-rust/crates/python-bridge/src/redis/store/auth_cache.rs b/litellm-rust/crates/python-bridge/src/redis/store/auth_cache.rs new file mode 100644 index 00000000000..8d7788b136e --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/store/auth_cache.rs @@ -0,0 +1,265 @@ +use std::sync::Arc; + +use litellm_python_compat::Value; +use litellm_redis_store::auth_cache::{AuthCache, Settings}; +use pyo3::prelude::*; + +use super::{ + NativePending, PyNumber, kind_error, pending, redis_async, redis_sync, remote, + request::{NativeRequestRedis, NativeSend}, + ttl, + value::PyValue, +}; +use crate::redis::client::NativeRedisClient; + +fn values(values: Vec>) -> Vec> { + values.into_iter().map(|value| value.map(PyValue)).collect() +} + +/// `UserApiKeyCache` with `enable_redis_auth_cache` on. The sync methods back `DualCache`'s sync +/// API and block with the GIL released, like redis-py's sync client. +#[pyclass(frozen, name = "NativeAuthCache")] +pub(crate) struct NativeAuthCache(Arc); + +#[pymethods] +impl NativeAuthCache { + #[new] + #[pyo3(signature = (client, *, memory_entries, memory_ttl_seconds = None))] + fn new( + client: Option>, + memory_entries: usize, + memory_ttl_seconds: Option, + ) -> PyResult { + Ok(Self(Arc::new(AuthCache::new( + client.map(|client| remote(&client)), + Settings { + memory_entries, + memory_ttl: ttl(memory_ttl_seconds)?, + }, + )))) + } + + #[getter] + fn has_redis(&self) -> bool { + self.0.has_redis() + } + + /// `request` answers a key its pre-call reads already found absent without another `GET`. + #[pyo3(signature = (key, request = None))] + fn get<'py>( + &self, + py: Python<'py>, + key: String, + request: Option<&NativeRequestRedis>, + ) -> PyResult> { + let store = self.0.clone(); + let request = request.map(|request| request.0.clone()); + redis_async(py, "async_get_cache", kind_error, async move { + store + .get(request.as_ref(), &key) + .await + .map(|value| value.map(PyValue)) + }) + } + + fn get_sync(&self, py: Python<'_>, key: String) -> PyResult> { + let store = self.0.clone(); + redis_sync(py, "get_cache", kind_error, async move { + store.get(None, &key).await.map(|value| value.map(PyValue)) + }) + } + + /// `update_cache_ttl` and `update_in_memory_max_size`: both partitions keep what they hold. + #[pyo3(signature = (*, memory_entries, memory_ttl_seconds = None))] + fn reconfigure(&self, memory_entries: usize, memory_ttl_seconds: Option) -> PyResult<()> { + self.0.reconfigure(Settings { + memory_entries, + memory_ttl: ttl(memory_ttl_seconds)?, + }); + Ok(()) + } + + #[pyo3(signature = (key, value, ttl_seconds = None))] + fn set<'py>( + &self, + py: Python<'py>, + key: String, + value: PyValue, + ttl_seconds: Option, + ) -> PyResult> { + let store = self.0.clone(); + let ttl = ttl(ttl_seconds)?; + redis_async(py, "async_set_cache", kind_error, async move { + store.set(&key, value.0, ttl).await + }) + } + + #[pyo3(signature = (key, value, ttl_seconds = None))] + fn set_sync( + &self, + py: Python<'_>, + key: String, + value: PyValue, + ttl_seconds: Option, + ) -> PyResult<()> { + let store = self.0.clone(); + let ttl = ttl(ttl_seconds)?; + redis_sync(py, "set_cache", kind_error, async move { + store.set_str(&key, value.0, ttl).await + }) + } + + /// Memory now and the `SET` on the batch `send` picks. + #[pyo3(signature = (key, value, ttl_seconds, send))] + fn declare_set( + &self, + key: String, + value: PyValue, + ttl_seconds: Option, + send: &NativeSend, + ) -> PyResult { + let written = self + .0 + .declare_set(send.send(), &key, value.0, ttl(ttl_seconds)?) + .map_err(kind_error)?; + Ok(pending( + "async_set_cache", + kind_error, + written, + std::convert::identity, + )) + } + + fn declare_delete(&self, key: String, send: &NativeSend) -> PyResult { + let deleted = self + .0 + .declare_delete(send.send(), &key) + .map_err(kind_error)?; + Ok(pending( + "async_delete_cache", + kind_error, + deleted, + std::convert::identity, + )) + } + + #[pyo3(signature = (entries, ttl_seconds = None))] + fn set_many<'py>( + &self, + py: Python<'py>, + entries: Vec<(String, PyValue)>, + ttl_seconds: Option, + ) -> PyResult> { + let store = self.0.clone(); + let entries = entries + .into_iter() + .map(|(key, value)| (key, value.0)) + .collect(); + let ttl = ttl(ttl_seconds)?; + redis_async(py, "async_set_cache_pipeline", kind_error, async move { + store.set_many(entries, ttl).await + }) + } + + /// On a request batch it returns before anything is sent, and `Now` is a pipeline of its own. + fn write_back<'py>( + &self, + py: Python<'py>, + entries: Vec<(String, PyValue, Option)>, + send: &NativeSend, + ) -> PyResult> { + let entries = entries + .into_iter() + .map(|(key, value, seconds)| Ok((key, value.0, ttl(seconds)?))) + .collect::>()?; + let written = self + .0 + .write_back(send.send(), entries) + .map_err(kind_error)?; + redis_async( + py, + "async_set_cache_pipeline_with_ttls", + kind_error, + written, + ) + } + + fn delete<'py>(&self, py: Python<'py>, keys: Vec) -> PyResult> { + let store = self.0.clone(); + redis_async(py, "async_delete_cache", kind_error, async move { + store.delete(&keys).await + }) + } + + fn delete_sync(&self, py: Python<'_>, keys: Vec) -> PyResult<()> { + let store = self.0.clone(); + redis_sync(py, "delete_cache", kind_error, async move { + store.delete(&keys).await + }) + } + + /// One `MGET` of keys the caller found missing in memory, on the batch `send` picks; `None` + /// without Redis. + fn get_many(&self, keys: Vec, send: &NativeSend) -> Option { + let read = self.0.get_many(send.send(), &keys)?; + Some(pending("async_batch_get_cache", kind_error, read, values)) + } + + fn increment_if_present<'py>( + &self, + py: Python<'py>, + key: String, + amount: PyNumber, + ) -> PyResult> { + let store = self.0.clone(); + redis_async(py, "async_increment_if_present", kind_error, async move { + store.increment_if_present(&key, amount.0).await + }) + } + + fn get_local(&self, key: &str) -> PyResult> { + Ok(self.0.get_local(key).map_err(kind_error)?.map(PyValue)) + } + + #[pyo3(signature = (key, value, ttl_seconds = None))] + fn set_local(&self, key: &str, value: PyValue, ttl_seconds: Option) -> PyResult<()> { + self.0 + .set_local(key, value.0, ttl(ttl_seconds)?) + .map_err(kind_error) + } + + fn delete_local(&self, key: &str) -> PyResult<()> { + self.0.delete_local(key).map_err(kind_error) + } + + fn flush_local(&self) -> PyResult<()> { + self.0.flush_local().map_err(kind_error) + } + + fn len_local(&self) -> PyResult { + let (objects, key_objects) = self.0.len_local().map_err(kind_error)?; + Ok(PyLocalSizes { + objects, + key_objects, + }) + } +} + +/// Entries the management-object and the user-key partitions hold in memory. +#[pyclass(frozen, eq, get_all, name = "LocalSizes")] +#[derive(PartialEq)] +pub(crate) struct PyLocalSizes { + objects: usize, + key_objects: usize, +} + +#[pymethods] +impl PyLocalSizes { + #[new] + fn new(objects: usize, key_objects: usize) -> Self { + Self { + objects, + key_objects, + } + } +} diff --git a/litellm-rust/crates/python-bridge/src/redis/store/mod.rs b/litellm-rust/crates/python-bridge/src/redis/store/mod.rs new file mode 100644 index 00000000000..d9c3aabb814 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/store/mod.rs @@ -0,0 +1,271 @@ +pub(crate) mod auth_cache; +mod policy; +pub(crate) mod rate_limits; +pub(crate) mod request; +pub(crate) mod router_cooldowns; +pub(crate) mod router_usage; +pub(crate) mod sensitive_route_pins; +pub(crate) mod spend_counters; +mod value; + +use value::PyValue; + +use std::{ + pin::Pin, + sync::{Mutex, PoisonError}, + time::Duration, +}; + +use litellm_host_python::PythonContext; +use litellm_python_compat::number::{SecondsError, seconds}; +use litellm_redis_store::{Error, FromValue, IntoValue, Number, Pending, Remote}; +use pyo3::{ + IntoPyObjectExt, + exceptions::{PyOverflowError, PyRuntimeError, PyTypeError, PyValueError}, + prelude::*, + types::{PyBool, PyFloat, PyInt}, +}; + +use crate::execution::{run_async_value, run_sync_value}; +use crate::redis::{ + client::NativeRedisClient, + events::{EventLoop, RedisCall}, +}; + +/// The labels `RedisCache` reports for this operation, in the caller's context so its +/// `service_target` reaches the service event, dispatched on the loop the caller runs on. +fn call(py: Python<'_>, call_type: &'static str) -> PyResult { + Ok(RedisCall { + context: PythonContext::capture(py)?, + event_loop: EventLoop::running(py), + call_type, + }) +} + +/// Runs one Redis operation as a Python awaitable, reported to `ServiceLogging` as `call_type`. +fn redis_async<'py, T, E: 'static>( + py: Python<'py>, + call_type: &'static str, + error: fn(E) -> PyErr, + operation: impl Future> + Send + 'static, +) -> PyResult> +where + T: for<'a> IntoPyObject<'a> + Send + 'static, +{ + let scope = call(py, call_type)?; + run_async_value( + py, + async move { scope.scope(operation).await.map_err(error) }, + ) +} + +/// Runs one Redis operation with the GIL released, for the router's sync paths. +fn redis_sync( + py: Python<'_>, + call_type: &'static str, + error: fn(E) -> PyErr, + operation: impl Future> + Send + 'static, +) -> PyResult +where + T: Send + 'static, +{ + let scope = call(py, call_type)?; + run_sync_value( + py, + async move { scope.scope(operation).await.map_err(error) }, + ) +} + +/// A TTL in seconds, raising what `int(seconds)` raises for infinity and NaN. +fn duration(value: f64) -> PyResult { + seconds(value).map_err(|error| match error { + SecondsError::Overflow => PyOverflowError::new_err(error.to_string()), + SecondsError::NotANumber => PyValueError::new_err(error.to_string()), + }) +} + +fn ttl(value: Option) -> PyResult> { + value.map(duration).transpose() +} + +fn remote(client: &NativeRedisClient) -> Remote { + Remote { + client: client.client.clone(), + default_ttl: client.default_ttl, + } +} + +type Resolve = Pin>> + Send>>; + +/// A Redis call declared on a request's batch. `resolve` flushes that batch if it has not gone +/// out and answers the call, once. +#[pyclass(frozen, name = "NativePending")] +pub(crate) struct NativePending { + resolve: Mutex>, + call_type: &'static str, +} + +impl NativePending { + pub(crate) fn new( + call_type: &'static str, + error: fn(E) -> PyErr, + reply: impl Future> + Send + 'static, + ) -> Self + where + T: for<'a> IntoPyObject<'a> + Send + 'static, + { + let resolve = async move { + let reply = reply.await.map_err(error)?; + Python::attach(|py| reply.into_py_any(py)) + }; + Self { + resolve: Mutex::new(Some(Box::pin(resolve))), + call_type, + } + } +} + +#[pymethods] +impl NativePending { + fn resolve<'py>(&self, py: Python<'py>) -> PyResult> { + let resolve = self + .resolve + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + .ok_or_else(|| PyRuntimeError::new_err("this Redis call was already resolved"))?; + redis_async(py, self.call_type, std::convert::identity, resolve) + } +} + +pyo3::create_exception!( + _native, + NativeRedisStoreError, + pyo3::exceptions::PyException, + "A native Redis store operation failed. Args are (kind, message), kind spelled as litellm-redis-store's ErrorKind." +); + +/// What every native store raises, its kind telling a caller which failures it may drop. +fn kind_error(error: Error) -> PyErr { + let kind: &'static str = error.kind().into(); + NativeRedisStoreError::new_err((kind, litellm_error::Report::new(&error).to_string())) +} + +/// A store op declared on the batch the caller's `NativeSend` picked, answered as `reply` shapes it. +fn pending( + call_type: &'static str, + error: fn(Error) -> PyErr, + declared: Pending, + reply: fn(T) -> R, +) -> NativePending +where + T: Send + 'static, + R: for<'a> IntoPyObject<'a> + Send + 'static, +{ + NativePending::new(call_type, error, async move { declared.await.map(reply) }) +} + +/// A Python `int` or `float` across the bridge, keeping which one it is. +#[derive(Clone, Copy, PartialEq)] +pub(crate) struct PyNumber(Number); + +impl<'a, 'py> FromPyObject<'a, 'py> for PyNumber { + type Error = PyErr; + + fn extract(value: Borrowed<'a, 'py, PyAny>) -> PyResult { + if value.is_instance_of::() && !value.is_instance_of::() { + return Ok(Self(Number::Int(value.extract()?))); + } + if value.is_instance_of::() { + return Ok(Self(Number::Float(value.extract()?))); + } + Err(PyTypeError::new_err("expected an int or a float")) + } +} + +impl<'py> IntoPyObject<'py> for PyNumber { + type Target = PyAny; + type Output = Bound<'py, PyAny>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> PyResult { + Ok(match self.0 { + Number::Int(value) => value.into_pyobject(py)?.into_any(), + Number::Float(value) => value.into_pyobject(py)?.into_any(), + }) + } +} + +/// A store enum read from its wire spelling, so the bridge names no variant. +pub(crate) struct Spelled(pub(crate) T); + +impl<'a, 'py, T: std::str::FromStr> FromPyObject<'a, 'py> for Spelled { + type Error = PyErr; + + fn extract(value: Borrowed<'a, 'py, PyAny>) -> PyResult { + let spelling: String = value.extract()?; + spelling + .parse() + .map(Self) + .map_err(|_| PyValueError::new_err(format!("unknown spelling {spelling:?}"))) + } +} + +/// A store type whose Python side is plain data, converted through its `FromValue` and `IntoValue` +/// so the bridge never names a field. A shape the store refuses is `ValueError`. +pub(crate) struct PyData(pub(crate) T); + +impl<'a, 'py, T: FromValue> FromPyObject<'a, 'py> for PyData { + type Error = PyErr; + + fn extract(value: Borrowed<'a, 'py, PyAny>) -> PyResult { + let PyValue(value) = value.extract()?; + T::from_value(&value) + .map(Self) + .map_err(|error| PyValueError::new_err(error.to_string())) + } +} + +impl<'py, T: IntoValue> IntoPyObject<'py> for PyData { + type Target = PyAny; + type Output = Bound<'py, PyAny>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> PyResult { + PyValue(self.0.into_value()).into_pyobject(py) + } +} + +fn unwrap_all(values: Vec>) -> Vec { + values.into_iter().map(|value| value.0).collect() +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use pyo3::prelude::*; + use rstest::rstest; + + use super::ttl; + + #[rstest] + #[case::infinity(f64::INFINITY, "OverflowError")] + #[case::nan(f64::NAN, "ValueError")] + fn a_ttl_int_refuses_raises_what_int_raises(#[case] seconds: f64, #[case] raised: &str) { + Python::initialize(); + Python::attach(|py| { + let error = ttl(Some(seconds)).unwrap_err(); + + assert_eq!(error.get_type(py).name().unwrap().to_string(), raised); + }); + } + + #[rstest] + #[case::fraction(Some(1.5), Some(Duration::from_millis(1500)))] + #[case::negative_clamps_to_zero(Some(-1.0), Some(Duration::ZERO))] + #[case::absent(None, None)] + fn a_finite_ttl_converts(#[case] seconds: Option, #[case] expected: Option) { + assert_eq!(ttl(seconds).unwrap(), expected); + } +} diff --git a/litellm-rust/crates/python-bridge/src/redis/store/policy.rs b/litellm-rust/crates/python-bridge/src/redis/store/policy.rs new file mode 100644 index 00000000000..ffdf7b97a2e --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/store/policy.rs @@ -0,0 +1,151 @@ +use std::time::Duration; + +use litellm_redis_store::TierPolicy; +use pyo3::prelude::*; + +use crate::{ + coercion::{Field, FieldSpec, ProjectionError}, + python_settings::{PythonSettings, Snapshot}, +}; + +const COOLDOWN_REFRESH: FieldSpec = + FieldSpec::new("cooldown_redis_read_interval", seconds); +const USAGE_REFRESH: FieldSpec = FieldSpec::new("usage_redis_read_interval", seconds); +const TRACKED_READS: FieldSpec = FieldSpec::new("max_redis_batch_cache_size", count); +const MEMORY_TTL: FieldSpec> = + FieldSpec::new("default_in_memory_ttl", optional_seconds); + +/// The tier values this Python process runs with, from its environment and `litellm` globals. +#[derive(Debug)] +pub(crate) struct RouterTierPolicies { + pub(crate) cooldowns: TierPolicy, + pub(crate) usage: TierPolicy, +} + +pub(crate) fn router_tier_policies(py: Python<'_>) -> PyResult { + Ok(project(&PythonSettings::RouterTiers.read(py)?)?) +} + +fn project(snapshot: &Snapshot<'_>) -> Result { + let tracked_reads = snapshot.read(&TRACKED_READS)?; + let default_in_memory_ttl = snapshot.read(&MEMORY_TTL)?; + Ok(RouterTierPolicies { + cooldowns: TierPolicy { + refresh_after: snapshot.read(&COOLDOWN_REFRESH)?, + tracked_reads, + default_in_memory_ttl, + }, + usage: TierPolicy { + refresh_after: snapshot.read(&USAGE_REFRESH)?, + tracked_reads, + default_in_memory_ttl, + }, + }) +} + +fn seconds(field: &Field<'_>) -> Result { + match field.value().extract::() { + Ok(seconds) if seconds.is_finite() && seconds >= 0.0 => { + Ok(Duration::from_secs_f64(seconds)) + } + _ => Err(field.invalid("a non-negative number of seconds")), + } +} + +fn optional_seconds(field: &Field<'_>) -> Result, ProjectionError> { + match field.value().is_none() { + true => Ok(None), + false => seconds(field).map(Some), + } +} + +fn count(field: &Field<'_>) -> Result { + field + .value() + .extract::() + .map_err(|_| field.invalid("a non-negative integer")) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use pyo3::{prelude::*, types::PyDict}; + + use super::project; + use crate::python_settings::PythonSettings; + + fn projected(py: Python<'_>, fields: &str) -> Result { + let locals = PyDict::new(py); + let source = std::ffi::CString::new(format!( + "import types\nsettings = types.SimpleNamespace({fields})" + )) + .unwrap(); + py.run(&source, Some(&locals), Some(&locals)).unwrap(); + let snapshot = + PythonSettings::RouterTiers.snapshot(locals.get_item("settings").unwrap().unwrap()); + project(&snapshot).map_err(|error| PyErr::from(error).to_string()) + } + + #[test] + fn the_host_values_are_each_familys_refresh_and_tracked_reads() { + Python::initialize(); + Python::attach(|py| { + let policies = projected( + py, + "cooldown_redis_read_interval=2.5, usage_redis_read_interval=30, max_redis_batch_cache_size=42, default_in_memory_ttl=None", + ) + .unwrap(); + assert_eq!( + policies.cooldowns.refresh_after, + Duration::from_millis(2500) + ); + assert_eq!(policies.usage.refresh_after, Duration::from_secs(30)); + assert_eq!(policies.cooldowns.tracked_reads, 42); + assert_eq!(policies.usage.tracked_reads, 42); + assert_eq!(policies.cooldowns.default_in_memory_ttl, None); + }); + } + + #[test] + fn a_set_default_in_memory_ttl_reaches_both_familys_backfill() { + Python::initialize(); + Python::attach(|py| { + let policies = projected( + py, + "cooldown_redis_read_interval=1, usage_redis_read_interval=10, max_redis_batch_cache_size=1000, default_in_memory_ttl=7.5", + ) + .unwrap(); + assert_eq!( + policies.cooldowns.default_in_memory_ttl, + Some(Duration::from_millis(7500)) + ); + assert_eq!( + policies.usage.default_in_memory_ttl, + Some(Duration::from_millis(7500)) + ); + }); + } + + #[test] + fn a_value_python_would_not_run_with_is_a_configuration_error() { + Python::initialize(); + Python::attach(|py| { + let negative = projected( + py, + "cooldown_redis_read_interval=-1, usage_redis_read_interval=10, max_redis_batch_cache_size=1000, default_in_memory_ttl=None", + ) + .unwrap_err(); + let text = projected( + py, + "cooldown_redis_read_interval=1, usage_redis_read_interval=10, max_redis_batch_cache_size='many', default_in_memory_ttl=None", + ) + .unwrap_err(); + assert!( + negative.contains("cooldown_redis_read_interval"), + "{negative}" + ); + assert!(text.contains("max_redis_batch_cache_size"), "{text}"); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/redis/store/rate_limits.rs b/litellm-rust/crates/python-bridge/src/redis/store/rate_limits.rs new file mode 100644 index 00000000000..ca95e1530f4 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/store/rate_limits.rs @@ -0,0 +1,577 @@ +use std::{sync::Arc, time::Duration}; + +use indexmap::IndexMap; +use litellm_redis_store::{ + Pending, + rate_limits::{ + AcquireReply, Applied, AtomicCounter, AtomicReply, Counter, Entry, Gauge, GuardedIncrement, + GuardedReply, Increment, Kind, Local, RateLimits, Scope, WindowCount, + }, +}; +use pyo3::{exceptions::PyValueError, prelude::*}; + +use super::{ + NativePending, PyNumber, Spelled, kind_error, redis_async, remote, + request::{NativeRequestRedis, NativeSend}, + ttl, +}; +use crate::redis::client::NativeRedisClient; + +/// The v3 rate limiter's keyspace. Memory reads and writes are synchronous like `InMemoryCache`; +/// every Redis call is a future. +#[pyclass(frozen, name = "NativeRateLimits")] +pub(crate) struct NativeRateLimits(Arc); + +/// An `int` or `float` counter, a `str` window start, or a `{slot_id: acquired_at}` registry. +#[derive(FromPyObject)] +enum PyLocal { + Number(PyNumber), + Text(String), + Slots(IndexMap), +} + +impl From for Local { + fn from(local: PyLocal) -> Self { + match local { + PyLocal::Number(number) => Self::Number(number.0), + PyLocal::Text(text) => Self::Text(text), + PyLocal::Slots(slots) => Self::Slots(slots.into_iter().collect()), + } + } +} + +/// What the memory tier holds, back as the Python value it was written as. +struct LocalValue(Local); + +impl<'py> IntoPyObject<'py> for LocalValue { + type Target = PyAny; + type Output = Bound<'py, PyAny>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> PyResult { + Ok(match self.0 { + Local::Number(number) => PyNumber(number).into_pyobject(py)?, + Local::Text(text) => text.into_pyobject(py)?.into_any(), + Local::Slots(slots) => slots + .into_iter() + .collect::>() + .into_pyobject(py)? + .into_any(), + }) + } +} + +/// `litellm.redis.stores.rate_limits.RateLimitScope`. +#[derive(FromPyObject)] +struct PyScope { + descriptor_key: String, + descriptor_value: String, +} + +impl From for Scope { + fn from(scope: PyScope) -> Self { + Self { + descriptor: scope.descriptor_key, + value: scope.descriptor_value, + } + } +} + +/// `RateLimitCounter`. +#[derive(FromPyObject)] +struct PyCounter { + scope: PyScope, + kind: Spelled, +} + +impl From for Counter { + fn from(counter: PyCounter) -> Self { + Self { + scope: counter.scope.into(), + kind: counter.kind.0, + } + } +} + +/// `RateLimitWindow`, `RateLimitCounter` or `RateLimitGauge`, told apart by the key suffix each names. +#[derive(FromPyObject)] +struct PyEntry { + suffix: String, + scope: PyScope, +} + +impl TryFrom for Entry { + type Error = PyErr; + + fn try_from(entry: PyEntry) -> PyResult { + Self::new(&entry.suffix, entry.scope.into()) + .map_err(|_| PyValueError::new_err(format!("unknown key suffix {:?}", entry.suffix))) + } +} + +/// `CounterIncrement`. +#[derive(FromPyObject)] +struct PyIncrement { + counter: PyCounter, + amount: PyNumber, + ttl: Option, +} + +impl From for Increment { + fn from(increment: PyIncrement) -> Self { + Self { + counter: increment.counter.into(), + amount: increment.amount.0, + ttl: increment.ttl.map(Duration::from_secs), + } + } +} + +/// `AtomicCounter`. +#[derive(FromPyObject)] +struct PyAtomicCounter { + counter: PyCounter, + limit: i64, + increment: i64, + ttl: i64, + window_size: i64, +} + +impl From for AtomicCounter { + fn from(counter: PyAtomicCounter) -> Self { + Self { + counter: counter.counter.into(), + limit: counter.limit, + increment: counter.increment, + ttl_seconds: counter.ttl, + window_size_seconds: counter.window_size, + } + } +} + +fn all>(records: Vec

) -> Vec { + records.into_iter().map(T::from).collect() +} + +#[pyclass(frozen, eq, get_all, name = "WindowCount")] +#[derive(PartialEq)] +pub(crate) struct PyWindowCount { + window_start: String, + count: i64, +} + +#[pymethods] +impl PyWindowCount { + #[new] + fn new(window_start: String, count: i64) -> Self { + Self { + window_start, + count, + } + } +} + +/// Per counter, the count after the increment and the start of the window it landed in. +#[pyclass(frozen, eq, get_all, name = "AtomicApplied")] +#[derive(PartialEq)] +pub(crate) struct PyAtomicApplied { + counters: Vec<(i64, i64)>, +} + +#[pymethods] +impl PyAtomicApplied { + #[new] + fn new(counters: Vec<(i64, i64)>) -> Self { + Self { counters } + } +} + +#[pyclass(frozen, eq, get_all, name = "AtomicOverLimit")] +#[derive(PartialEq)] +pub(crate) struct PyAtomicOverLimit { + index: usize, + current: i64, + limit: i64, +} + +#[pymethods] +impl PyAtomicOverLimit { + #[new] + fn new(index: usize, current: i64, limit: i64) -> Self { + Self { + index, + current, + limit, + } + } +} + +#[derive(IntoPyObject)] +enum PyAtomicReply { + Applied(PyAtomicApplied), + OverLimit(PyAtomicOverLimit), +} + +#[pyclass(frozen, eq, get_all, name = "GuardedReply")] +#[derive(PartialEq)] +pub(crate) struct PyGuardedReply { + applied: bool, + count: i64, +} + +#[pymethods] +impl PyGuardedReply { + #[new] + fn new(applied: bool, count: i64) -> Self { + Self { applied, count } + } +} + +#[pyclass(frozen, eq, get_all, name = "GaugesAcquired")] +#[derive(PartialEq)] +pub(crate) struct PyGaugesAcquired { + in_flight: Vec, +} + +#[pymethods] +impl PyGaugesAcquired { + #[new] + fn new(in_flight: Vec) -> Self { + Self { in_flight } + } +} + +#[pyclass(frozen, eq, get_all, name = "GaugeOverLimit")] +#[derive(PartialEq)] +pub(crate) struct PyGaugeOverLimit { + index: usize, + in_flight: i64, + limit: i64, +} + +#[pymethods] +impl PyGaugeOverLimit { + #[new] + fn new(index: usize, in_flight: i64, limit: i64) -> Self { + Self { + index, + in_flight, + limit, + } + } +} + +#[derive(IntoPyObject)] +enum PyAcquireReply { + Acquired(PyGaugesAcquired), + OverLimit(PyGaugeOverLimit), +} + +fn window_counts(counts: Vec) -> Vec { + counts + .into_iter() + .map( + |WindowCount { + window_start, + count, + }| PyWindowCount { + window_start, + count, + }, + ) + .collect() +} + +fn atomic(reply: AtomicReply) -> PyAtomicReply { + match reply { + AtomicReply::Applied(applied) => PyAtomicReply::Applied(PyAtomicApplied { + counters: applied + .into_iter() + .map( + |Applied { + count, + window_start, + }| (count, window_start), + ) + .collect(), + }), + AtomicReply::OverLimit { + index, + current, + limit, + } => PyAtomicReply::OverLimit(PyAtomicOverLimit { + index, + current, + limit, + }), + } +} + +fn guarded(GuardedReply { applied, count }: GuardedReply) -> PyGuardedReply { + PyGuardedReply { applied, count } +} + +fn acquire(reply: AcquireReply) -> PyAcquireReply { + match reply { + AcquireReply::Acquired(in_flight) => { + PyAcquireReply::Acquired(PyGaugesAcquired { in_flight }) + } + AcquireReply::OverLimit { + index, + in_flight, + limit, + } => PyAcquireReply::OverLimit(PyGaugeOverLimit { + index, + in_flight, + limit, + }), + } +} + +/// One script call declared on the batch the caller's `NativeSend` picked; `None` without Redis. +fn script( + call_type: &'static str, + declared: Option>, + reply: fn(T) -> R, +) -> Option +where + T: Send + 'static, + R: for<'a> IntoPyObject<'a> + Send + 'static, +{ + Some(super::pending(call_type, kind_error, declared?, reply)) +} + +#[pymethods] +impl NativeRateLimits { + #[new] + #[pyo3(signature = (client, backfill_ttl_seconds = None))] + fn new( + client: Option>, + backfill_ttl_seconds: Option, + ) -> PyResult { + Ok(Self(Arc::new(RateLimits::new( + client.map(|client| remote(&client)), + ttl(backfill_ttl_seconds)?, + )))) + } + + #[getter] + fn has_redis(&self) -> bool { + self.0.has_redis() + } + + fn slot_groups(&self, scopes: Vec, force: bool) -> Vec> { + self.0.slot_groups(&all(scopes), force) + } + + fn get_local(&self, entry: PyEntry) -> PyResult> { + Ok(self + .0 + .get_local(&entry.try_into()?) + .map_err(kind_error)? + .map(LocalValue)) + } + + #[pyo3(signature = (entry, value, ttl_seconds))] + fn set_local(&self, entry: PyEntry, value: PyLocal, ttl_seconds: Option) -> PyResult<()> { + self.0 + .set_local(&entry.try_into()?, value.into(), ttl(ttl_seconds)?) + .map_err(kind_error) + } + + /// `request` answers a key its pre-call reads already found absent without another `GET`. + #[pyo3(signature = (entry, *, backfill_ttl_seconds = None, request = None))] + fn get<'py>( + &self, + py: Python<'py>, + entry: PyEntry, + backfill_ttl_seconds: Option, + request: Option<&NativeRequestRedis>, + ) -> PyResult> { + let store = self.0.clone(); + let entry = Entry::try_from(entry)?; + let backfill_ttl = ttl(backfill_ttl_seconds)?; + let request = request.map(|request| request.0.clone()); + redis_async(py, "async_get_cache", kind_error, async move { + store + .get(request.as_ref(), &entry, backfill_ttl) + .await + .map(|local| local.map(LocalValue)) + }) + } + + fn sliding_window( + &self, + counters: Vec, + now: i64, + window_size: i64, + send: &NativeSend, + ) -> Option { + script( + "batch_rate_limiter_script", + self.0 + .sliding_window(send.send(), &all(counters), now, window_size), + window_counts, + ) + } + + fn read_counters(&self, counters: Vec, send: &NativeSend) -> Option { + script( + "batch_counter_read_script", + self.0.read_counters(send.send(), &all(counters)), + std::convert::identity, + ) + } + + fn check_and_increment( + &self, + counters: Vec, + send: &NativeSend, + ) -> Option { + script( + "check_and_increment_by_n_script", + self.0.check_and_increment(send.send(), &all(counters)), + atomic, + ) + } + + fn guarded_increment( + &self, + counter: PyCounter, + expected_window_start: String, + amount: PyNumber, + ttl_seconds: Option, + send: &NativeSend, + ) -> Option { + let increment = GuardedIncrement { + counter: counter.into(), + expected_window_start, + amount: amount.0, + ttl: ttl_seconds.map(Duration::from_secs), + }; + script( + "window_guarded_token_increment_script", + self.0.guarded_increment(send.send(), &increment), + guarded, + ) + } + + fn parallel_acquire( + &self, + gauges: Vec<(PyScope, i64)>, + slot_ttl_seconds: u64, + slot_id: &str, + send: &NativeSend, + ) -> Option { + let gauges: Vec = gauges + .into_iter() + .map(|(scope, limit)| Gauge { + scope: scope.into(), + limit, + }) + .collect(); + script( + "parallel_acquire_script", + self.0.parallel_acquire( + send.send(), + &gauges, + Duration::from_secs(slot_ttl_seconds), + slot_id, + ), + acquire, + ) + } + + fn parallel_release( + &self, + gauges: Vec, + slot_id: &str, + send: &NativeSend, + ) -> Option { + script( + "parallel_release_script", + self.0.parallel_release(send.send(), &all(gauges), slot_id), + std::convert::identity, + ) + } + + fn parallel_count( + &self, + gauges: Vec, + slot_ttl_seconds: u64, + send: &NativeSend, + ) -> Option { + script( + "parallel_count_script", + self.0.parallel_count( + send.send(), + &all(gauges), + Duration::from_secs(slot_ttl_seconds), + ), + std::convert::identity, + ) + } + + fn token_increment( + &self, + increments: Vec, + send: &NativeSend, + ) -> Option { + script( + "token_increment_script", + self.0.token_increment(send.send(), &all(increments)), + std::convert::identity, + ) + } + + /// `False` without Redis. + fn token_increment_post_call( + &self, + request: &NativeRequestRedis, + increments: Vec, + ) -> bool { + self.0 + .token_increment_post_call(&request.0, all(increments)) + } + + /// `False` without Redis. + fn parallel_release_post_call( + &self, + request: &NativeRequestRedis, + gauges: Vec, + slot_id: &str, + ) -> bool { + self.0 + .parallel_release_post_call(&request.0, &all(gauges), slot_id) + } + + fn increment<'py>( + &self, + py: Python<'py>, + increment: PyIncrement, + ) -> PyResult> { + let store = self.0.clone(); + let increment = Increment::from(increment); + redis_async(py, "async_increment", kind_error, async move { + Ok(store.increment(&increment).await.map(PyNumber)) + }) + } + + fn increment_many<'py>( + &self, + py: Python<'py>, + increments: Vec, + ) -> PyResult> { + let store = self.0.clone(); + let increments: Vec = all(increments); + redis_async(py, "async_increment_pipeline", kind_error, async move { + store.increment_many(&increments).await; + Ok(()) + }) + } + + fn increment_many_post_call(&self, request: &NativeRequestRedis, increments: Vec) { + self.0 + .increment_many_post_call(&request.0, &all(increments)); + } +} diff --git a/litellm-rust/crates/python-bridge/src/redis/store/request.rs b/litellm-rust/crates/python-bridge/src/redis/store/request.rs new file mode 100644 index 00000000000..3436b26a16c --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/store/request.rs @@ -0,0 +1,60 @@ +use litellm_redis_store::{RequestRedis, Send}; +use pyo3::prelude::*; + +use super::{call, duration}; +use crate::execution::run_async_value; + +/// One request's native Redis batches. Python builds one per request scope and passes it to each +/// request-aware store call. +#[pyclass(frozen, name = "NativeRequestRedis")] +pub(crate) struct NativeRequestRedis(pub(crate) RequestRedis); + +#[pymethods] +impl NativeRequestRedis { + #[new] + fn new(py: Python<'_>, post_call_deadline_seconds: f64) -> PyResult { + let runtime = litellm_host_python::runtime_handle()?; + let deadline = duration(post_call_deadline_seconds)?; + let deadline_tag = call(py, "post_call_redis_batch")?.tag(); + Ok(Self(crate::logger::capture(py).scope(|| { + RequestRedis::with_deadline_tag(runtime, deadline, Some(deadline_tag)) + }))) + } + + fn flush_pre<'py>(&self, py: Python<'py>) -> PyResult> { + let request = self.0.clone(); + let scope = call(py, "request_redis_batch")?; + run_async_value(py, async move { + scope.scope(request.flush_pre()).await; + Ok(()) + }) + } + + fn flush_post<'py>(&self, py: Python<'py>) -> PyResult> { + let request = self.0.clone(); + let scope = call(py, "post_call_redis_batch")?; + run_async_value(py, async move { + scope.scope(request.flush_post()).await; + Ok(()) + }) + } +} + +/// The Python face of `Send`: which batch a store op rides. Each variant carries exactly what it +/// needs, so there is no phase string and no phase without a request. +#[pyclass(frozen, name = "NativeSend")] +pub(crate) enum NativeSend { + Now(), + PreCall(Py), + PostCall(Py), +} + +impl NativeSend { + pub(crate) fn send(&self) -> Send<'_> { + match self { + Self::Now() => Send::Now, + Self::PreCall(request) => Send::PreCall(&request.get().0), + Self::PostCall(request) => Send::PostCall(&request.get().0), + } + } +} diff --git a/litellm-rust/crates/python-bridge/src/redis/store/router_cooldowns.rs b/litellm-rust/crates/python-bridge/src/redis/store/router_cooldowns.rs new file mode 100644 index 00000000000..5d695ca7090 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/store/router_cooldowns.rs @@ -0,0 +1,182 @@ +use std::{collections::HashMap, sync::Arc}; + +use litellm_redis_store::{ + Send, + router_cooldowns::{Cooldown, RouterCooldowns}, +}; +use pyo3::{prelude::*, types::PyTuple}; + +use super::{ + NativePending, PyData, PyNumber, duration, kind_error, pending, policy::router_tier_policies, + redis_async, redis_sync, remote, request::NativeSend, +}; +use crate::redis::client::NativeRedisClient; + +/// One router's cooldown store. The router's sync completion path calls `put`, `active_sync`, +/// `minimum` and `increment_allowed_fails`, so those block with the GIL released like redis-py; the +/// `async_` twins return awaitables for async callers. +#[pyclass(frozen, name = "NativeRouterCooldowns")] +pub(crate) struct NativeRouterCooldowns(Arc); + +/// `[(model_id, cooldown), ...]` the way `RedisCache` hands the router its batch read. +struct PyActive(Vec<(String, Cooldown)>); + +impl<'py> IntoPyObject<'py> for PyActive { + type Target = PyAny; + type Output = Bound<'py, PyAny>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> PyResult { + active_to_py(py, self.0)? + .into_pyobject(py) + .map(Bound::into_any) + } +} + +fn active_to_py( + py: Python<'_>, + active: Vec<(String, Cooldown)>, +) -> PyResult>> { + active + .into_iter() + .map(|(model_id, cooldown)| { + PyTuple::new( + py, + [ + model_id.into_pyobject(py)?.into_any(), + PyData(cooldown).into_pyobject(py)?, + ], + ) + }) + .collect() +} + +#[pymethods] +impl NativeRouterCooldowns { + #[new] + fn new(py: Python<'_>, client: Option>) -> PyResult { + let remote = client.map(|client| remote(&client)); + let policy = router_tier_policies(py)?.cooldowns; + Ok(Self(Arc::new(RouterCooldowns::new(remote, policy)))) + } + + /// Adds the Redis tier and keeps memory; `false` when one was already attached. + fn attach(&self, client: PyRef<'_, NativeRedisClient>) -> bool { + self.0.attach(remote(&client)) + } + + #[getter] + fn has_redis(&self) -> bool { + self.0.has_redis() + } + + fn put( + &self, + py: Python<'_>, + model_id: String, + cooldown: PyData, + ttl_seconds: f64, + ) -> PyResult<()> { + let store = self.0.clone(); + let ttl = duration(ttl_seconds)?; + redis_sync(py, "set_cache", kind_error, async move { + store.put_str(&model_id, &cooldown.0, ttl).await + }) + } + + fn increment_allowed_fails( + &self, + py: Python<'_>, + model_id: String, + suffix: Option, + ttl_seconds: f64, + ) -> PyResult { + let store = self.0.clone(); + let ttl = duration(ttl_seconds)?; + redis_sync(py, "increment_cache", kind_error, async move { + store + .increment_allowed_fails_int(&model_id, suffix.as_deref(), ttl) + .await + }) + } + + fn async_put<'py>( + &self, + py: Python<'py>, + model_id: String, + cooldown: PyData, + ttl_seconds: f64, + ) -> PyResult> { + let store = self.0.clone(); + let ttl = duration(ttl_seconds)?; + redis_async(py, "async_set_cache", kind_error, async move { + store.put(&model_id, &cooldown.0, ttl).await + }) + } + + fn async_increment_allowed_fails<'py>( + &self, + py: Python<'py>, + model_id: String, + suffix: Option, + ttl_seconds: f64, + ) -> PyResult> { + let store = self.0.clone(); + let ttl = duration(ttl_seconds)?; + redis_async(py, "async_increment", kind_error, async move { + store + .increment_allowed_fails(&model_id, suffix.as_deref(), ttl) + .await + }) + } + + /// The deployments still cooling down, read on the batch `send` picks. + fn active(&self, model_ids: Vec, send: &NativeSend) -> PyResult { + let active = self.0.active(send.send(), &model_ids).map_err(kind_error)?; + Ok(pending( + "async_batch_get_cache", + kind_error, + active, + PyActive, + )) + } + + /// `active` on its own with the GIL released, for the router's sync completion path. + fn active_sync<'py>( + &self, + py: Python<'py>, + model_ids: Vec, + ) -> PyResult>> { + let active = self.0.active(Send::Now, &model_ids).map_err(kind_error)?; + let active = redis_sync(py, "batch_get_cache", kind_error, active)?; + active_to_py(py, active) + } + + fn minimum(&self, py: Python<'_>, model_ids: Vec) -> PyResult> { + let store = self.0.clone(); + redis_sync(py, "batch_get_cache", kind_error, async move { + store + .minimum(&model_ids) + .await + .map(|minimum| minimum.map(PyNumber)) + }) + } + + fn delete_local(&self, model_id: &str) -> PyResult<()> { + self.0.delete_local(model_id).map_err(kind_error) + } + + fn flush_local_counters(&self) -> PyResult<()> { + self.0.flush_local_counters().map_err(kind_error) + } + + fn snapshot(&self) -> PyResult>> { + Ok(self + .0 + .snapshot() + .map_err(kind_error)? + .into_iter() + .map(|(key, cooldown)| (key, PyData(cooldown))) + .collect()) + } +} diff --git a/litellm-rust/crates/python-bridge/src/redis/store/router_usage.rs b/litellm-rust/crates/python-bridge/src/redis/store/router_usage.rs new file mode 100644 index 00000000000..48565028de3 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/store/router_usage.rs @@ -0,0 +1,182 @@ +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use litellm_redis_store::router_usage::{Counter, Kind, RouterUsage}; +use pyo3::prelude::*; + +use super::{ + PyNumber, Spelled, kind_error, policy::router_tier_policies, redis_async, redis_sync, remote, +}; +use crate::redis::client::NativeRedisClient; + +/// `litellm.redis.stores.router_usage.UsageCounter`. +#[derive(FromPyObject)] +struct PyCounter { + deployment_id: String, + model: Option, + minute: String, + kind: Spelled, +} + +impl From for Counter { + fn from(counter: PyCounter) -> Self { + Self { + deployment_id: counter.deployment_id, + model: counter.model, + minute: counter.minute, + kind: counter.kind.0, + } + } +} + +fn counters(counters: Vec) -> Vec { + counters.into_iter().map(Counter::from).collect() +} + +/// One router's global TPM/RPM counters. The `_sync` methods block with the GIL released, for the +/// Rust router's sync calls, which have no event loop to await on. +#[pyclass(frozen, name = "NativeRouterUsage")] +pub(crate) struct NativeRouterUsage(Arc); + +#[pymethods] +impl NativeRouterUsage { + #[new] + fn new(py: Python<'_>, client: Option>) -> PyResult { + let remote = client.map(|client| remote(&client)); + let policy = router_tier_policies(py)?.usage; + Ok(Self(Arc::new(RouterUsage::new(remote, policy)))) + } + + /// Adds the Redis tier and keeps memory; `false` when one was already attached. + fn attach(&self, client: PyRef<'_, NativeRedisClient>) -> bool { + self.0.attach(remote(&client)) + } + + #[getter] + fn has_redis(&self) -> bool { + self.0.has_redis() + } + + fn increment<'py>( + &self, + py: Python<'py>, + counter: PyCounter, + value: PyNumber, + ttl_seconds: u64, + ) -> PyResult> { + let store = self.0.clone(); + let counter = Counter::from(counter); + redis_async(py, "async_increment", kind_error, async move { + Ok(store + .increment(&counter, value.0, Duration::from_secs(ttl_seconds)) + .await + .map(PyNumber)) + }) + } + + /// `_increment_deployment`'s `async_increment_pipeline`, always on its own round trip. + fn increment_deployment<'py>( + &self, + py: Python<'py>, + deployment_id: String, + model: Option, + minute: String, + tokens: PyNumber, + requests: i64, + ) -> PyResult> { + let store = self.0.clone(); + redis_async(py, "async_increment_pipeline", kind_error, async move { + store + .increment_deployment( + &deployment_id, + model.as_deref(), + &minute, + tokens.0, + requests, + ) + .await + }) + } + + fn get_many<'py>( + &self, + py: Python<'py>, + counters: Vec, + ) -> PyResult> { + let store = self.0.clone(); + let counters = self::counters(counters); + redis_async(py, "async_batch_get_cache", kind_error, async move { + let read = store.get_many(&counters).await?; + Ok(read + .into_iter() + .map(|count| count.map(PyNumber)) + .collect::>()) + }) + } + + fn increment_sync( + &self, + py: Python<'_>, + counter: PyCounter, + value: PyNumber, + ttl_seconds: u64, + ) -> PyResult> { + let store = self.0.clone(); + let counter = Counter::from(counter); + redis_sync(py, "increment_cache", kind_error, async move { + Ok(store + .increment(&counter, value.0, Duration::from_secs(ttl_seconds)) + .await + .map(PyNumber)) + }) + } + + fn increment_deployment_sync( + &self, + py: Python<'_>, + deployment_id: String, + model: Option, + minute: String, + tokens: PyNumber, + requests: i64, + ) -> PyResult<()> { + let store = self.0.clone(); + redis_sync(py, "increment_pipeline", kind_error, async move { + store + .increment_deployment( + &deployment_id, + model.as_deref(), + &minute, + tokens.0, + requests, + ) + .await + }) + } + + fn get_many_sync( + &self, + py: Python<'_>, + counters: Vec, + ) -> PyResult>> { + let store = self.0.clone(); + let counters = self::counters(counters); + redis_sync(py, "batch_get_cache", kind_error, async move { + let read = store.get_many(&counters).await?; + Ok(read.into_iter().map(|count| count.map(PyNumber)).collect()) + }) + } + + fn flush_local_counters(&self) -> PyResult<()> { + self.0.flush_local_counters().map_err(kind_error) + } + + fn snapshot(&self) -> PyResult> { + Ok(self + .0 + .snapshot() + .map_err(kind_error)? + .into_iter() + .map(|(key, count)| (key, PyNumber(count))) + .collect()) + } +} diff --git a/litellm-rust/crates/python-bridge/src/redis/store/sensitive_route_pins.rs b/litellm-rust/crates/python-bridge/src/redis/store/sensitive_route_pins.rs new file mode 100644 index 00000000000..8b1b82945fb --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/store/sensitive_route_pins.rs @@ -0,0 +1,49 @@ +use std::sync::Arc; + +use litellm_redis_store::sensitive_route_pins::SensitiveRoutePins; +use pyo3::prelude::*; + +use super::{duration, kind_error, redis_async, remote}; +use crate::redis::client::NativeRedisClient; + +#[pyclass(frozen, name = "NativeSensitiveRoutePins")] +pub(crate) struct NativeSensitiveRoutePins(Arc); + +#[pymethods] +impl NativeSensitiveRoutePins { + #[new] + fn new(client: Option>) -> Self { + Self(Arc::new(SensitiveRoutePins::new( + client.map(|client| remote(&client)), + ))) + } + + fn get<'py>( + &self, + py: Python<'py>, + tenant: String, + session_id: String, + default_ttl_seconds: f64, + ) -> PyResult> { + let store = self.0.clone(); + let default_ttl = duration(default_ttl_seconds)?; + redis_async(py, "async_get_cache", kind_error, async move { + store.get(&tenant, &session_id, default_ttl).await + }) + } + + fn set<'py>( + &self, + py: Python<'py>, + tenant: String, + session_id: String, + model: String, + ttl_seconds: f64, + ) -> PyResult> { + let store = self.0.clone(); + let ttl = duration(ttl_seconds)?; + redis_async(py, "async_set_cache", kind_error, async move { + store.set(&tenant, &session_id, &model, ttl).await + }) + } +} diff --git a/litellm-rust/crates/python-bridge/src/redis/store/spend_counters.rs b/litellm-rust/crates/python-bridge/src/redis/store/spend_counters.rs new file mode 100644 index 00000000000..9da6a0e00e5 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/store/spend_counters.rs @@ -0,0 +1,277 @@ +use std::{collections::HashMap, sync::Arc}; + +use litellm_redis_store::{ + Number, + spend_counters::{LostSeed, PostCallSettlement, Reading, SpendCounter, SpendCounters}, +}; +use pyo3::prelude::*; + +use super::{ + NativePending, PyData, PyNumber, duration, kind_error, pending, redis_async, remote, + request::{NativeRequestRedis, NativeSend}, + ttl, unwrap_all, +}; +use crate::redis::client::NativeRedisClient; + +/// A counter's value and whether Redis answered it, a clean miss included. +#[pyclass(frozen, eq, get_all, name = "SpendRead")] +#[derive(PartialEq)] +pub(crate) struct PySpendRead { + value: Option, + from_redis: bool, +} + +#[pymethods] +impl PySpendRead { + #[new] + fn new(value: Option, from_redis: bool) -> Self { + Self { value, from_redis } + } +} + +/// Whether every increment one `increment_many_post_call` declared has its reply. +#[pyclass(frozen, name = "PostCallSettlement")] +pub(crate) struct PyPostCallSettlement(PostCallSettlement); + +#[pymethods] +impl PyPostCallSettlement { + #[getter] + fn settled(&self) -> bool { + self.0.is_settled() + } +} + +fn increments(increments: Vec<(PyData, PyNumber)>) -> Vec<(SpendCounter, Number)> { + increments + .into_iter() + .map(|(counter, amount)| (counter.0, amount.0)) + .collect() +} + +fn numbers(values: Option>>) -> Option>> { + values.map(|values| { + values + .into_iter() + .map(|value| value.map(PyNumber)) + .collect() + }) +} + +/// The proxy's spend counters, Redis first with this worker's memory copy behind it. +#[pyclass(frozen, name = "NativeSpendCounters")] +pub(crate) struct NativeSpendCounters(Arc); + +#[pymethods] +impl NativeSpendCounters { + #[new] + fn new(client: Option>) -> Self { + Self(Arc::new(SpendCounters::new( + client.map(|client| remote(&client)), + ))) + } + + /// Adds the Redis tier and keeps memory; `false` when one was already attached. + fn attach(&self, client: PyRef<'_, NativeRedisClient>) -> bool { + self.0.attach(remote(&client)) + } + + #[getter] + fn has_redis(&self) -> bool { + self.0.has_redis() + } + + #[getter] + fn redis_ttl_seconds(&self) -> Option { + self.0.redis_ttl().map(|ttl| ttl.as_secs_f64()) + } + + /// The least TTL a Redis write gives a committed spend twin. + fn set_committed_ttl(&self, ttl_seconds: f64) -> PyResult<()> { + self.0.set_committed_ttl(duration(ttl_seconds)?); + Ok(()) + } + + /// `("redis" | "memory", value)`: where the answer came from, a Redis miss included. + fn read<'py>( + &self, + py: Python<'py>, + counter: PyData, + ) -> PyResult> { + let store = self.0.clone(); + redis_async(py, "async_get_cache", kind_error, async move { + let reading = store.read(&counter.0).await?; + Ok(PySpendRead { + value: reading.value().map(PyNumber), + from_redis: matches!(reading, Reading::Redis(_)), + }) + }) + } + + /// One `MGET` on the batch `send` picks; `None` without Redis. It resolves to `None` when the + /// read failed, so each caller reads on its own. + fn read_many( + &self, + counters: Vec>, + send: &NativeSend, + ) -> Option { + let read = self.0.read_many(send.send(), &unwrap_all(counters))?; + Some(pending("async_batch_get_cache", kind_error, read, numbers)) + } + + fn increment<'py>( + &self, + py: Python<'py>, + counter: PyData, + amount: PyNumber, + ) -> PyResult> { + let store = self.0.clone(); + redis_async(py, "async_increment", kind_error, async move { + store.increment(&counter.0, amount.0).await.map(PyNumber) + }) + } + + fn increment_many<'py>( + &self, + py: Python<'py>, + increments: Vec<(PyData, PyNumber)>, + ) -> PyResult> { + let store = self.0.clone(); + let increments = self::increments(increments); + redis_async(py, "async_increment_pipeline", kind_error, async move { + let totals = store.increment_many(&increments).await?; + Ok(totals.into_iter().map(PyNumber).collect::>()) + }) + } + + /// Declares the increments on the request's post-call batch; `None` when there is no Redis. + fn increment_many_post_call( + &self, + request: &NativeRequestRedis, + increments: Vec<(PyData, PyNumber)>, + ) -> Option { + self.0 + .increment_many_post_call(&request.0, &self::increments(increments)) + .map(PyPostCallSettlement) + } + + fn set_max<'py>( + &self, + py: Python<'py>, + counter: PyData, + spend: PyNumber, + ) -> PyResult> { + let store = self.0.clone(); + redis_async(py, "async_set_max", kind_error, async move { + store + .set_max(&counter.0, spend.0) + .await + .map(|stored| stored.map(PyNumber)) + }) + } + + #[pyo3(signature = (counter, spend, *, increment_if_lost))] + fn seed<'py>( + &self, + py: Python<'py>, + counter: PyData, + spend: PyNumber, + increment_if_lost: bool, + ) -> PyResult> { + let store = self.0.clone(); + let lost = if increment_if_lost { + LostSeed::Increment + } else { + LostSeed::Answer + }; + redis_async(py, "async_set_cache", kind_error, async move { + store.seed(&counter.0, spend.0, lost).await.map(PyNumber) + }) + } + + #[pyo3(signature = (counter, spend, ttl_seconds = None))] + fn set<'py>( + &self, + py: Python<'py>, + counter: PyData, + spend: PyNumber, + ttl_seconds: Option, + ) -> PyResult> { + let store = self.0.clone(); + let ttl = ttl(ttl_seconds)?; + redis_async(py, "async_set_cache", kind_error, async move { + store.set(&counter.0, spend.0, ttl).await + }) + } + + fn delete<'py>( + &self, + py: Python<'py>, + counters: Vec>, + ) -> PyResult> { + let store = self.0.clone(); + let counters = unwrap_all(counters); + redis_async(py, "async_delete_cache", kind_error, async move { + store.delete(&counters).await + }) + } + + fn refresh_ttl<'py>( + &self, + py: Python<'py>, + counter: PyData, + ) -> PyResult> { + let store = self.0.clone(); + redis_async(py, "async_refresh_ttl", kind_error, async move { + store.refresh_ttl(&counter.0).await + }) + } + + fn get_local(&self, counter: PyData) -> PyResult> { + Ok(self + .0 + .get_local(&counter.0) + .map_err(kind_error)? + .map(PyNumber)) + } + + #[pyo3(signature = (counter, spend, ttl_seconds = None))] + fn set_local( + &self, + counter: PyData, + spend: PyNumber, + ttl_seconds: Option, + ) -> PyResult<()> { + self.0 + .set_local(&counter.0, spend.0, ttl(ttl_seconds)?) + .map_err(kind_error) + } + + fn delete_local(&self, counter: PyData) -> PyResult<()> { + self.0.delete_local(&counter.0).map_err(kind_error) + } + + fn increment_local( + &self, + counter: PyData, + amount: PyNumber, + ) -> PyResult { + self.0 + .increment_local(&counter.0, amount.0) + .map(PyNumber) + .map_err(kind_error) + } + + fn flush_local(&self) -> PyResult<()> { + self.0.flush_local().map_err(kind_error) + } + + fn snapshot(&self) -> PyResult> { + Ok(self + .0 + .snapshot() + .map_err(kind_error)? + .into_iter() + .map(|(key, spend)| (key, PyNumber(spend))) + .collect()) + } +} diff --git a/litellm-rust/crates/python-bridge/src/redis/store/value.rs b/litellm-rust/crates/python-bridge/src/redis/store/value.rs new file mode 100644 index 00000000000..c86c9e9bb20 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/redis/store/value.rs @@ -0,0 +1,125 @@ +use litellm_python_compat::{BigInt, MAX_DEPTH, Value}; +use pyo3::{ + exceptions::{PyTypeError, PyValueError}, + prelude::*, + types::{ + PyBool, PyBytes, PyComplex, PyDict, PyFloat, PyFrozenSet, PyInt, PyList, PySet, PyString, + PyTuple, + }, +}; + +/// A plain Python value (what `json.loads` or `ast.literal_eval` returns) across the bridge. +/// Anything else, a live object included, is refused with `TypeError`. +pub(crate) struct PyValue(pub(crate) Value); + +impl<'a, 'py> FromPyObject<'a, 'py> for PyValue { + type Error = PyErr; + + fn extract(value: Borrowed<'a, 'py, PyAny>) -> PyResult { + to_value(&value, 0).map(Self) + } +} + +impl<'py> IntoPyObject<'py> for PyValue { + type Target = PyAny; + type Output = Bound<'py, PyAny>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> PyResult { + to_python(py, self.0) + } +} + +fn to_value(value: &Bound<'_, PyAny>, depth: usize) -> PyResult { + if depth > MAX_DEPTH { + return Err(PyValueError::new_err("value is nested too deeply")); + } + let nested = |items: Bound<'_, PyAny>| -> PyResult> { + items + .try_iter()? + .map(|item| to_value(&item?, depth + 1)) + .collect() + }; + if value.is_none() { + return Ok(Value::None); + } + if let Ok(flag) = value.cast::() { + return Ok(Value::Bool(flag.is_true())); + } + if value.is_instance_of::() { + return Ok(Value::Int(match value.extract::() { + Ok(small) => small.into(), + Err(_) => value + .str()? + .to_str()? + .parse::() + .map_err(|error| PyValueError::new_err(error.to_string()))?, + })); + } + if let Ok(number) = value.cast::() { + return Ok(Value::Float(number.value())); + } + if let Ok(text) = value.cast::() { + return Ok(Value::Str(text.to_str()?.to_owned())); + } + if let Ok(bytes) = value.cast::() { + return Ok(Value::Bytes(bytes.as_bytes().to_vec())); + } + if let Ok(complex) = value.cast::() { + return Ok(Value::Complex { + re: complex.real(), + im: complex.imag(), + }); + } + if let Ok(entries) = value.cast::() { + return entries + .iter() + .map(|(key, item)| Ok((to_value(&key, depth + 1)?, to_value(&item, depth + 1)?))) + .collect::>() + .map(Value::Dict); + } + if value.is_instance_of::() { + return nested(value.clone()).map(Value::List); + } + if value.is_instance_of::() { + return nested(value.clone()).map(Value::Tuple); + } + if value.is_instance_of::() || value.is_instance_of::() { + return nested(value.clone()).map(Value::Set); + } + Err(PyTypeError::new_err(format!( + "{} is not a plain data value", + value.get_type().name()? + ))) +} + +fn to_python(py: Python<'_>, value: Value) -> PyResult> { + let all = |items: Vec| -> PyResult>> { + items.into_iter().map(|item| to_python(py, item)).collect() + }; + Ok(match value { + Value::None => py.None().into_bound(py), + Value::Bool(flag) => PyBool::new(py, flag).to_owned().into_any(), + Value::Int(number) => match i64::try_from(&number) { + Ok(small) => small.into_pyobject(py)?.into_any(), + Err(_) => py + .get_type::() + .call1((number.to_string(),))? + .into_any(), + }, + Value::Float(number) => number.into_pyobject(py)?.into_any(), + Value::Complex { re, im } => PyComplex::from_doubles(py, re, im).into_any(), + Value::Str(text) => text.into_pyobject(py)?.into_any(), + Value::Bytes(bytes) => PyBytes::new(py, &bytes).into_any(), + Value::Tuple(items) => PyTuple::new(py, all(items)?)?.into_any(), + Value::List(items) => PyList::new(py, all(items)?)?.into_any(), + Value::Set(items) => PySet::new(py, all(items)?)?.into_any(), + Value::Dict(entries) => { + let dict = PyDict::new(py); + for (key, item) in entries { + dict.set_item(to_python(py, key)?, to_python(py, item)?)?; + } + dict.into_any() + } + }) +} diff --git a/litellm-rust/crates/python-bridge/src/router/config.rs b/litellm-rust/crates/python-bridge/src/router/config.rs new file mode 100644 index 00000000000..5e099b8841c --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/config.rs @@ -0,0 +1,414 @@ +//! The documents `RouterCoreHandle` reads: the router's static and dynamic config, a call's +//! request and an `apply` change set. A config key router-core does not read declines the +//! handle, so Python keeps the router on its own core. + +use std::collections::BTreeMap; + +use litellm_router_core::select::PassThrough; +use litellm_router_core::{ + ChangeSet, ConfigError, DeploymentSpec, OutcomeCallbacks, RouterConfig, RouterDynamicConfig, + RouterRequest, + config::{Fallbacks, RetryPolicy}, + host::UncountedRetries, + mock::MockFailure, + registry::IndexOverrides, +}; +use serde::{Deserialize, de::DeserializeOwned}; +use serde_json::{Map, Value}; + +/// The `RouterConfig` keys router-core reads. +const STATIC_KEYS: &[&str] = &[ + "model_list", + "routing_strategy", + "routing_groups", + "routing_group_members", + "plugins", + "cooldown_time", + "disable_cooldowns", + "allowed_fails_policy", + "retry_policy", + "model_group_retry_policy", + "retry_after", + "timeout", + "request_timeout", + "drop_params", + "enable_pre_call_checks", + "routing_plugins", + "callback_filters", + "enable_health_check_routing", + "health_check_staleness_threshold", + "background_health_check_model_groups", + "enable_weighted_failover", + "fallback_on_unsupported_params", + "provider_default_deployment_ids", +]; + +/// The `RouterDynamicConfig` keys router-core reads. +const DYNAMIC_KEYS: &[&str] = &[ + "version", + "model_alias_map", + "model_group_alias", + "fallbacks", + "default_fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "num_retries", + "max_fallbacks", + "allowed_fails", + "allowed_fails_is_default", + "fallback_keys", + "default_deployment", + "default_deployment_raises", +]; + +/// Router arguments that do not change routing, accepted in either config and dropped. +const IGNORED_KEYS: &[&str] = &[ + "set_verbose", + "debug_level", + "alerting_config", + "client_ttl", +]; + +/// Fallback lists Python treats as unset when empty (`context_window_fallbacks or None`). +const EMPTY_MEANS_UNSET: &[&str] = &["context_window_fallbacks", "content_policy_fallbacks"]; + +/// Which document a failure is about. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)] +pub(crate) enum Document { + #[strum(serialize = "router config")] + Static, + #[strum(serialize = "dynamic router config")] + Dynamic, + #[strum(serialize = "router request")] + Request, + #[strum(serialize = "change set")] + Changes, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum Error { + #[error("the {document} must be a JSON object")] + NotObject { document: Document }, + #[error("router-core does not read `{key}` in the {document}")] + UnknownKey { document: Document, key: String }, + #[error("the {document} does not have the shape router-core reads")] + Shape { + document: Document, + #[source] + source: serde_json::Error, + }, + #[error("the request names no model")] + MissingModel, + #[error("router-core rejected the router config")] + Core(#[from] ConfigError), +} + +impl Error { + /// Whether router-core cannot serve this config, so Python keeps its own core. + pub(crate) fn declined(&self) -> bool { + match self { + Self::UnknownKey { .. } | Self::Core(_) => true, + Self::Shape { document, .. } => { + matches!(document, Document::Static | Document::Dynamic) + } + Self::NotObject { .. } | Self::MissingModel => false, + } + } +} + +fn object(document: Document, value: Value) -> Result, Error> { + match value { + Value::Object(object) => Ok(object), + _ => Err(Error::NotObject { document }), + } +} + +fn shaped(document: Document, object: Map) -> Result { + serde_json::from_value(Value::Object(object)) + .map_err(|source| Error::Shape { document, source }) +} + +/// Drop unset and ignored keys, then reject any key outside `known`. +fn routing_keys( + document: Document, + value: Value, + known: &[&str], +) -> Result, Error> { + let mut object = object(document, value)?; + object.retain(|key, value| { + let empty_unset = EMPTY_MEANS_UNSET.contains(&key.as_str()) + && value.as_array().is_some_and(Vec::is_empty); + !(value.is_null() || empty_unset || IGNORED_KEYS.contains(&key.as_str())) + }); + if let Some(key) = object.keys().find(|key| !known.contains(&key.as_str())) { + return Err(Error::UnknownKey { + document, + key: key.clone(), + }); + } + Ok(object) +} + +pub(crate) fn parse_router_config(value: Value) -> Result { + let object = routing_keys(Document::Static, value, STATIC_KEYS)?; + shaped(Document::Static, object) +} + +pub(crate) fn parse_dynamic_config(value: Value) -> Result { + let object = routing_keys(Document::Dynamic, value, DYNAMIC_KEYS)?; + shaped(Document::Dynamic, object) +} + +/// A call's request as Python spells it. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RequestSpec { + model: Option, + team_id: Option, + listing_team_id: Option, + proxy_admin: Option, + allowed_models: Option>, + admin_model: Option, + specific_deployment: Option, + num_retries: Option, + uncounted_retries: Option, + disable_fallbacks: Option, + router_weights: Option, + pinned_deployment_id: Option, + routing_strategy: Option, + filter_tags: Option, + /// `listed` or `healthy`: which pass-through pick the request makes. + pass_through: Option, + target_order: Option, + excluded_ids: Option>, + retry_skipped_ids: Option>, + pre_routes: Option, + has_image: Option, + endpoint: Option, + web_search: Option, + route_handler: Option, + decisions_media: Option, + batch_create: Option, + batch_upload: Option, + embedding_batch: Option, + wildcard_request: Option, + unrouted: Option, + passthrough_on_no_deployment: Option, + skip_weighted_failover: Option, + same_group_fallbacks_only: Option, + model_group_retry_policy: Option>, + retry_policy_raises: Option>, + #[serde(default, deserialize_with = "present")] + fallbacks: Option>, + #[serde(default, deserialize_with = "present")] + context_window_fallbacks: Option>, + #[serde(default, deserialize_with = "present")] + content_policy_fallbacks: Option>, + max_fallbacks: Option, + mock_failure: Option, + cooldown_time: Option, + health_check_probe: Option, + outcome_callbacks: Option, + body: Option, +} + +/// A key the request passed, even as `None`: `Some(None)`. +fn present<'de, D, T>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, + T: Deserialize<'de>, +{ + Option::::deserialize(deserializer).map(Some) +} + +/// Read a request; `model` replaces the request's own when given. +pub(crate) fn parse_request(value: Value, model: Option<&str>) -> Result { + let spec: RequestSpec = shaped(Document::Request, object(Document::Request, value)?)?; + let model = model + .map(str::to_owned) + .or(spec.model) + .ok_or(Error::MissingModel)?; + Ok(RouterRequest { + model, + body: spec.body.unwrap_or_default(), + team_id: spec.team_id, + listing_team_id: spec.listing_team_id, + proxy_admin: spec.proxy_admin.unwrap_or(false), + allowed_models: spec.allowed_models.unwrap_or_default(), + admin_model: spec.admin_model, + specific_deployment: spec.specific_deployment.unwrap_or(false), + num_retries: spec.num_retries, + uncounted_retries: spec.uncounted_retries, + disable_fallbacks: spec.disable_fallbacks.unwrap_or(false), + router_weights: spec.router_weights, + pinned_deployment_id: spec.pinned_deployment_id, + routing_strategy: spec.routing_strategy, + filter_tags: spec.filter_tags.unwrap_or(false), + pass_through: match spec.pass_through.as_deref() { + None => None, + Some("healthy") => Some(PassThrough::Healthy), + Some(_) => Some(PassThrough::Listed), + }, + target_order: spec.target_order, + excluded_ids: spec.excluded_ids.unwrap_or_default(), + retry_skipped_ids: spec.retry_skipped_ids.unwrap_or_default(), + pre_routes: spec.pre_routes.unwrap_or(false), + has_image: spec.has_image.unwrap_or(false), + endpoint: spec.endpoint, + web_search: spec.web_search.unwrap_or(false), + route_handler: spec.route_handler, + decisions_media: spec.decisions_media.unwrap_or(false), + batch_create: spec.batch_create.unwrap_or(false), + batch_upload: spec.batch_upload.unwrap_or(false), + embedding_batch: spec.embedding_batch.unwrap_or(false), + wildcard_request: spec.wildcard_request, + unrouted: spec.unrouted.unwrap_or(false), + passthrough_on_no_deployment: spec.passthrough_on_no_deployment.unwrap_or(false), + skip_weighted_failover: spec.skip_weighted_failover.unwrap_or(false), + same_group_fallbacks_only: spec.same_group_fallbacks_only.unwrap_or(false), + model_group_retry_policy: spec.model_group_retry_policy, + retry_policy_raises: spec.retry_policy_raises.unwrap_or_default(), + fallbacks: spec.fallbacks, + context_window_fallbacks: spec.context_window_fallbacks, + content_policy_fallbacks: spec.content_policy_fallbacks, + max_fallbacks: spec.max_fallbacks, + mock_failure: spec.mock_failure, + cooldown_time: spec.cooldown_time, + health_check_probe: spec.health_check_probe.unwrap_or(false), + outcome_callbacks: spec.outcome_callbacks.unwrap_or_default(), + payload: None, + }) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ChangesSpec { + /// Every deployment, replacing the current list before the upserts and deletes. + #[serde(default)] + replace: Option>, + /// Deployments added at the end, beside any that share their id. + #[serde(default)] + append: Vec, + #[serde(default)] + upsert: Vec, + #[serde(default)] + delete: Vec, + /// A new dynamic document (the same keys `RouterCoreHandle(static, dynamic)` takes). + #[serde(default)] + settings: Option, + /// The registry indexes the host wrote (`IndexOverrides`); `{}` drops them. + #[serde(default)] + index: Option, + /// The deployment at the id map's position for this id, removed with every index shifted. + #[serde(default)] + remove: Option, + /// A write to the indexes the host hands out (`HostIndexes`), field by field. + #[serde(default)] + host_index: Option, +} + +pub(crate) fn parse_changes(value: Value) -> Result { + let spec: ChangesSpec = shaped(Document::Changes, object(Document::Changes, value)?)?; + Ok(ChangeSet { + replace: spec.replace, + appends: spec.append, + upserts: spec.upsert, + deletes: spec.delete, + settings: spec.settings.map(parse_dynamic_config).transpose()?, + index: spec.index, + remove: spec.remove, + host_index: spec.host_index, + }) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::unknown_key(json!({"routing_strategy_args": {"ttl": 1}}), "routing_strategy_args")] + #[case::dynamic_key_in_static(json!({"num_retries": 2}), "num_retries")] + fn an_unread_static_key_declines_naming_the_key(#[case] json: Value, #[case] key: &str) { + let error = parse_router_config(json).unwrap_err(); + assert!(error.declined()); + assert!(matches!(&error, Error::UnknownKey { key: named, .. } if named == key)); + } + + #[rstest] + #[case::null_unknown(json!({"routing_strategy_args": null}))] + #[case::ignored(json!({"set_verbose": true, "debug_level": "INFO"}))] + #[case::pre_call_checks_off(json!({"enable_pre_call_checks": false}))] + #[case::pre_call_checks_on(json!({"enable_pre_call_checks": true}))] + fn unset_and_ignored_keys_are_accepted(#[case] json: Value) { + parse_router_config(json).unwrap(); + } + + #[rstest] + #[case::not_object(json!([]), false)] + #[case::bad_shape(json!({"retry_after": "soon"}), true)] + fn a_config_that_is_not_an_object_is_a_value_error_and_a_wrong_shape_declines( + #[case] json: Value, + #[case] declined: bool, + ) { + assert_eq!(parse_router_config(json).unwrap_err().declined(), declined); + } + + #[rstest] + fn python_router_defaults_read_as_unset() { + let dynamic = parse_dynamic_config(json!({ + "num_retries": null, "fallbacks": [], "context_window_fallbacks": [], + "content_policy_fallbacks": [], "model_group_alias": {"gpt": "gpt-4"}, + })) + .unwrap(); + assert_eq!(dynamic.num_retries, None); + assert!(dynamic.context_window_fallbacks.is_none()); + assert!(dynamic.content_policy_fallbacks.is_none()); + assert_eq!(dynamic.model_alias_map["gpt"], "gpt-4"); + } + + #[rstest] + #[case::own_model(json!({"model": "gpt", "num_retries": 1}), None, "gpt")] + #[case::override_model(json!({"model": "gpt"}), Some("other"), "other")] + fn a_request_reads_its_model( + #[case] json: Value, + #[case] model: Option<&str>, + #[case] expected: &str, + ) { + assert_eq!(parse_request(json, model).unwrap().model, expected); + } + + #[rstest] + #[case::negative("negative", UncountedRetries::Negative)] + #[case::not_positive("not_positive", UncountedRetries::NotPositive)] + #[case::raises("raises", UncountedRetries::Raises)] + fn a_request_reads_a_num_retries_that_is_no_count( + #[case] spelled: &str, + #[case] expected: UncountedRetries, + ) { + let request = + parse_request(json!({"model": "gpt", "uncounted_retries": spelled}), None).unwrap(); + assert_eq!(request.uncounted_retries, Some(expected)); + assert_eq!(request.num_retries, None); + } + + #[rstest] + #[case::unknown_field(json!({"model": "gpt", "temperature": 1}))] + #[case::no_model(json!({"team_id": "t"}))] + fn a_bad_request_is_a_value_error(#[case] json: Value) { + assert!(!parse_request(json, None).unwrap_err().declined()); + } + + #[rstest] + fn a_change_set_reads_upserts_and_deletes() { + let changes = parse_changes(json!({ + "upsert": [{"model_name": "m", "litellm_params": {"model": "openai/x"}}], + "delete": ["old"], + })) + .unwrap(); + assert_eq!(changes.upserts.len(), 1); + assert_eq!(changes.deletes, ["old"]); + } +} diff --git a/litellm-rust/crates/python-bridge/src/router/host.rs b/litellm-rust/crates/python-bridge/src/router/host.rs new file mode 100644 index 00000000000..04d19e70edc --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/host.rs @@ -0,0 +1,1699 @@ +//! The host behind `RouterCoreHandle`. Rust never awaits Python: each host operation the call +//! loop needs becomes a [`HostCall`] parked in the call's [`Exchange`], and the call future stays +//! pending until Python runs it and hands the [`HostCallResponse`] back through [`Driver::resume`]. +//! +//! The driver polls the call future with a no-op waker while the call's exchange is bound to the +//! current thread, so a host future finds its exchange without the router passing it through. +//! +//! The routing cache is a host operation too: [`StepCache`] parks every [`RouterCache`] op as a +//! `cache` step, and Python runs it against the router's own `DualCache` (memory plus Redis), so +//! Python and Rust read and write the same cooldowns, fail counters and usage. + +use std::{ + cell::RefCell, + future::{Future, poll_fn}, + pin::Pin, + sync::{Arc, Mutex, MutexGuard, PoisonError}, + task::{Context, Poll, Waker}, + time::Duration, +}; + +use async_trait::async_trait; +use litellm_router_core::{ + Attempt, AttemptError, Blocked, BuildAsk, BuildPath, Built, CallbackFiltered, Deployment, + Error, Host, Output, Payload, PreRoute, PreRouted, Provided, Randomness, RouterCache, + RouterEvent, RouterRequest, RouterResponse, RoutingContext, SecretRef, SelectStage, + ThreadRandomness, build::BuiltDeployment, host::ModelInfoLookup, model_info::MergedModelInfo, + precall::PreCallFacts, +}; +use pyo3::{PyResult, Python, types::PyAnyMethods}; +use serde_json::{Map, Value}; + +/// The `kind` of a step dict. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, strum::IntoStaticStr)] +pub(crate) enum StepKind { + #[strum(serialize = "attempt")] + Attempt, + #[strum(serialize = "sleep")] + Sleep, + #[strum(serialize = "allow_fallback")] + AllowFallback, + #[strum(serialize = "refuse_fallbacks")] + RefuseFallbacks, + #[strum(serialize = "select")] + Select, + #[strum(serialize = "pre_route")] + PreRoute, + #[strum(serialize = "cache")] + Cache, + #[strum(serialize = "secrets")] + Secrets, + #[strum(serialize = "validate")] + Validate, + #[strum(serialize = "provider")] + Provider, + #[strum(serialize = "added")] + Added, + #[strum(serialize = "model_info")] + ModelInfo, + #[strum(serialize = "done")] + Done, + #[strum(serialize = "failed")] + Failed, +} + +/// Which selection stage Python is asked to run. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum Stage { + /// Pre-call checks and sub-routers: return the deployments that remain. + Plugins, + /// A strategy router-core does not implement: return the candidates with the pick first. + /// When candidates share an id, `records` holds each candidate's deployment, in order, and the + /// host answers with the pick's position, so a pick splits over every entry as Python's does. + Strategy { + strategy: String, + records: Vec, + }, + /// Tag routing: return the deployments that remain. + Tags, + /// `enable_pre_call_checks`: return the facts the filter decides by. + PreCall, + /// `_deployment_declares_no_vision`: return the candidates that declare no vision support. + TextOnly, + /// `_deployment_serves_route`: return the candidates that cannot serve `handler`'s call. + RouteIncapable { handler: String }, + /// `_decisions_deployment_refuses_media`: return the candidates that refuse the call's media. + MediaRefusing, + /// `deployments_admitting_embedding_batch`: return the candidates the embedding pick keeps. + EmbeddingBatch, + /// The override strategy's pre-call check on a deployment named by id. + OverrideCheck(String), + /// The routing plugins: run them when `run`, then return the candidate models they left. + RoutingPlugins { run: bool }, + /// The host's `CustomLogger` deployment filters: return the ids that remain, or a failure. + /// They read the ids this request already failed on, as Python's request kwargs carry them. + CallbackFilters { + excluded_ids: Vec, + skipped_ids: Vec, + target_order: Option, + }, + /// `f"{provider}/{model}"` for a name no wildcard pattern matches as spelled: return it as + /// the one entry of `models`. + ProviderRequest, +} + +/// The `op` of a `cache` step: one per [`RouterCache`] method. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, strum::IntoStaticStr)] +pub(crate) enum CacheOp { + /// `async_get_cache(key)`: answered with the stored JSON value, or `None`. + #[strum(serialize = "get")] + Get, + /// `async_set_cache(key, value, ttl)`: answered with `None`. + #[strum(serialize = "set")] + Set, + /// `async_increment(key, value, ttl)`: answered with the new number. The TTL applies when + /// the key is created. + #[strum(serialize = "increment")] + Increment, + /// `async_increment_usage(keys, value, ttl)`: `value` is one amount per key, added as + /// `_increment_deployment_usage` adds them. Answered with `None`. + #[strum(serialize = "increment_usage")] + IncrementUsage, + /// `async_batch_get_cache(keys)`: answered with one JSON value or `None` per key, in order. + #[strum(serialize = "batch_get")] + BatchGet, + /// `async_delete_cache(key)`: answered with `None`. + #[strum(serialize = "delete")] + Delete, +} + +/// Python's outcome of a `cache` step. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum CacheOutcome { + /// `get`: the stored value (`None` when missing). + Value(Option), + /// `batch_get`: one entry per key. + Values(Vec>), + /// `increment`: the counter after the increment. + Number(f64), + /// `set`, `delete` and `increment_usage`. + Stored, +} + +impl CacheOutcome { + /// The op this outcome answers (`Stored` answers both `set` and `delete`). + fn answers(&self, op: CacheOp) -> bool { + matches!( + (self, op), + (Self::Value(_), CacheOp::Get) + | (Self::Values(_), CacheOp::BatchGet) + | (Self::Number(_), CacheOp::Increment) + | ( + Self::Stored, + CacheOp::Set | CacheOp::Delete | CacheOp::IncrementUsage + ) + ) + } +} + +/// A fallback chain's dict targets, which can carry an api key. +#[derive(Clone, PartialEq, veil::Redact)] +pub(crate) struct DictTargets(#[redact(with = "[REDACTED]")] pub(crate) Vec>); + +/// What a call is blocked on, for Python to run. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum HostCall { + Attempt { + deployment_id: String, + /// The picked deployment as configured: `model_name`, `litellm_params`, `model_info` + /// (with its `id`). The host builds the request from it; it never looks the deployment up + /// in a Python router. + deployment: Value, + /// The name the attempt resolved: the requested name or the fallback target. + model: String, + model_group: String, + /// The group a pre-routing hook selected for `model`, when it differs. + routed: Option, + /// The `order` level of a same-group order fallback. + target_order: Option, + /// Every deployment `model` resolved to for this caller, the picked one included. + candidate_ids: Vec, + retry: u32, + fallback_depth: u32, + fallback_level: u32, + timeout: Option, + /// Request params the fallback walk's dict targets merged in. + params: Option>, + /// The walk level whose request metadata the attempt carries. + scope: u32, + max_retries: u32, + model_group_size: usize, + }, + Sleep { + seconds: f64, + }, + AllowFallback { + target: String, + }, + RefuseFallbacks { + /// The walk level whose chain the targets are. + level: u32, + targets: DictTargets, + }, + Select { + stage: Stage, + model_group: String, + /// The name the request resolved from, which strategies key their state by. + name: String, + /// Whose selector a strategy reads: `default`, `group` (with `group`) or `override`. + scope: &'static str, + group: Option, + /// The deployments the name resolved to, before health filters (the tag stage reads them). + pool: Vec, + candidates: Vec, + }, + /// Pre-routing before a pick resolves `model`: answered with the model group to resolve + /// instead (a string), a custom strategy's deployment document (a mapping), or `None`. + PreRoute { + model: String, + fallback_depth: u32, + retry: u32, + }, + /// One [`RouterCache`] op against the host's cache. `key` is set for every op but + /// `batch_get`, which sets `keys`; `value` is set for `set` (the JSON to store) and + /// `increment` (the amount); `ttl` is in seconds. + Cache { + op: CacheOp, + key: Option, + keys: Option>, + value: Option, + ttl: Option, + /// The op reads or writes this worker's in-memory tier only (`local_only=True`). + local: bool, + }, + /// The deployment build's `os.environ/` references: answered with one value per ref. + Secrets { + index: usize, + refs: Vec, + }, + /// The deployment build's `validate`, `provider` and `added` steps for one deployment. + Build { + kind: StepKind, + path: BuildPath, + index: usize, + entry: Value, + declared_id: Option, + duplicate_id: bool, + /// The provider the `provider` step resolved (`added` only). + provider: Option, + }, + /// One lookup of a model-info query ([`ModelInfoLookup`], as its `lookup` field and + /// arguments): answered as the lookup documents. + ModelInfo { + lookup: Value, + }, +} + +impl HostCall { + pub(crate) fn kind(&self) -> StepKind { + match self { + Self::Attempt { .. } => StepKind::Attempt, + Self::Sleep { .. } => StepKind::Sleep, + Self::AllowFallback { .. } => StepKind::AllowFallback, + Self::RefuseFallbacks { .. } => StepKind::RefuseFallbacks, + Self::Select { .. } => StepKind::Select, + Self::PreRoute { .. } => StepKind::PreRoute, + Self::Cache { .. } => StepKind::Cache, + Self::Secrets { .. } => StepKind::Secrets, + Self::Build { kind, .. } => *kind, + Self::ModelInfo { .. } => StepKind::ModelInfo, + } + } +} + +/// Python's response to a [`HostCall`]. +#[derive(Debug)] +pub(crate) enum HostCallResponse { + /// The attempt's total tokens, or its failure. + Attempt(Result), + /// A successful attempt whose response the provider blocked, or a stream whose outcome the + /// host reports when it ends (`deferred`), with its tokens. + Served { + total_tokens: u64, + blocked: Option, + deferred: bool, + tracks_usage: Option, + }, + Slept, + AllowFallback(bool), + /// Whether the chain a `refuse_fallbacks` step named is refused. + RefuseFallbacks(bool), + /// The candidate ids that remain, pick first. + Select(Vec), + /// A strategy's pick among candidates sharing an id: positions in the candidates, pick first. + Positions(Vec), + /// The model group to resolve instead, a custom strategy's deployment, or `None`. + PreRoute(Option), + /// The facts a `pre_call` select step asked for. + PreCall(PreCallFacts), + /// The candidate models a `routing_plugins` select step reported. + Models(Option>), + /// The ids a `callback_filters` select step left with the one a filter prefers, or the error + /// a filter raised. + Filtered(Result<(Vec, Option), AttemptError>), + /// What a pick's select or pre-route step, or a fallback check, raised on the host instead of + /// answering: the walk takes it as the try's failure. + Raised(AttemptError), + Cache(CacheOutcome), + /// One value per ref of a `secrets` step. + Secrets(Vec), + Validated(Built), + Provided(Provided), + Added(Built), + /// A `model_info` lookup's response, as the host spelled it. + ModelInfo(Value), +} + +impl HostCallResponse { + fn kind(&self) -> StepKind { + match self { + Self::Attempt(_) | Self::Served { .. } => StepKind::Attempt, + // Answers whichever step raised it (`check`). + Self::Raised(_) => StepKind::Select, + Self::Slept => StepKind::Sleep, + Self::AllowFallback(_) => StepKind::AllowFallback, + Self::RefuseFallbacks(_) => StepKind::RefuseFallbacks, + Self::Select(_) + | Self::Positions(_) + | Self::PreCall(_) + | Self::Models(_) + | Self::Filtered(_) => StepKind::Select, + Self::PreRoute(_) => StepKind::PreRoute, + Self::Cache(_) => StepKind::Cache, + Self::Secrets(_) => StepKind::Secrets, + Self::Validated(_) => StepKind::Validate, + Self::Provided(_) => StepKind::Provider, + Self::Added(_) => StepKind::Added, + Self::ModelInfo(_) => StepKind::ModelInfo, + } + } + + fn check(&self, call: &HostCall) -> Result<(), DriveError> { + if matches!(self, Self::Raised(_)) + && matches!( + call.kind(), + StepKind::Select | StepKind::PreRoute | StepKind::AllowFallback + ) + { + return Ok(()); + } + if self.kind() != call.kind() { + return Err(DriveError::WrongAnswer { + expected: call.kind(), + got: self.kind(), + }); + } + if let ( + Self::Select(ids) | Self::Filtered(Ok((ids, _))), + HostCall::Select { candidates, .. }, + ) = (self, call) + && let Some(id) = { + let known: std::collections::HashSet<&str> = + candidates.iter().map(String::as_str).collect(); + ids.iter().find(|id| !known.contains(id.as_str())) + } + { + return Err(DriveError::UnknownCandidate { id: id.clone() }); + } + if let (Self::Positions(positions), HostCall::Select { candidates, .. }) = (self, call) + && let Some(position) = positions.iter().find(|p| **p >= candidates.len()) + { + return Err(DriveError::UnknownCandidate { + id: format!("#{position}"), + }); + } + if let (Self::Secrets(values), HostCall::Secrets { refs, .. }) = (self, call) + && values.len() != refs.len() + { + return Err(DriveError::SecretsLength { + expected: refs.len(), + got: values.len(), + }); + } + if let (Self::Cache(outcome), HostCall::Cache { op, keys, .. }) = (self, call) { + if !outcome.answers(*op) { + return Err(DriveError::WrongCacheOutcome { op: *op }); + } + if let (CacheOutcome::Values(values), Some(keys)) = (outcome, keys) + && values.len() != keys.len() + { + return Err(DriveError::BatchLength { + expected: keys.len(), + got: values.len(), + }); + } + } + Ok(()) + } +} + +/// How a driven future ended well. +#[derive(Debug)] +pub(crate) enum Finish { + /// `start`: the call's response. + Call(RouterResponse), + /// `select`: the deployment picked and the model group it serves. + Selected { + deployment_id: String, + model_group: String, + /// The deployment as served: a priority group's member carries its priority as `order`. + deployment: Value, + }, + /// `healthy`: the deployments a pick would choose among, as served. + Healthy { + /// A deployment named by its id, which the listing returns alone rather than as a list. + single: bool, + deployment_ids: Vec, + deployments: Vec, + }, + /// `report_failure`: the cooldown applied, if the deployment was cooled down. + Reported { cooldown: Option }, + /// `report_success` / `count_success`: the success counter's key, if one was counted. + Counted { key: Option }, + /// `build`: the deployments it installed, in order. + Built(Vec), + /// `router_model_info`: the merged model info (`None` when the lookup answered `None`). + RouterModelInfo(Option), + /// `deployment_model_info`: the merged model info, if any. + DeploymentModelInfo(Option>), + /// A model-info query that ends with one JSON answer (`model_group_info`, + /// `configured_token_limits`, `model_listing_info`). + Value(Value), +} + +/// One step of a call: what it is blocked on, or how it ended. +#[derive(Debug)] +pub(crate) enum Step { + HostCall(HostCall), + Done(Finish), + Failed(Error), +} + +/// A misuse of the step protocol by the caller. +#[derive(Debug, thiserror::Error)] +pub(crate) enum DriveError { + #[error("the router call already finished")] + Finished, + #[error("the router call has not made a host call yet, so the outcome must be None")] + UnexpectedResponse, + #[error("the router call is waiting for an outcome of its `{expected}` step")] + MissingResponse { expected: StepKind }, + #[error("the router call is waiting for an outcome of its `{expected}` step, not `{got}`")] + WrongAnswer { expected: StepKind, got: StepKind }, + #[error("deployment `{id}` was not one of the select step's candidates")] + UnknownCandidate { id: String }, + #[error("the outcome does not answer a `{op}` cache step")] + WrongCacheOutcome { op: CacheOp }, + #[error("a `batch_get` cache step for {expected} keys was answered with {got} values")] + BatchLength { expected: usize, got: usize }, + #[error("a `secrets` step for {expected} references was answered with {got} values")] + SecretsLength { expected: usize, got: usize }, + #[error("the router call is pending on something other than a step")] + Stalled, +} + +impl DriveError { + /// Whether the caller passed a bad outcome, rather than driving a call that cannot go on. + pub(crate) fn is_request(&self) -> bool { + match self { + Self::UnexpectedResponse + | Self::MissingResponse { .. } + | Self::WrongAnswer { .. } + | Self::UnknownCandidate { .. } + | Self::WrongCacheOutcome { .. } + | Self::BatchLength { .. } + | Self::SecretsLength { .. } => true, + Self::Finished | Self::Stalled => false, + } + } +} + +/// The host call a router call is blocked on and the response that releases it. +#[derive(Default)] +struct Exchange { + call: Option, + response: Option, + /// Events the call reported since the last step, handed to Python with the next one. + events: Vec, +} + +type Shared = Arc>; + +fn lock(exchange: &Mutex) -> MutexGuard<'_, Exchange> { + exchange.lock().unwrap_or_else(PoisonError::into_inner) +} + +thread_local! { + static CURRENT: RefCell> = const { RefCell::new(None) }; +} + +/// Binds an exchange to this thread for one poll and restores the previous one after. +struct Bound(Option); + +impl Bound { + fn new(exchange: &Shared) -> Self { + Self(CURRENT.replace(Some(exchange.clone()))) + } +} + +impl Drop for Bound { + fn drop(&mut self) { + CURRENT.set(self.0.take()); + } +} + +/// Park the host call `call` in the bound exchange and wait for its response. +/// `response`, or the failure a step answered with instead ([`HostCallResponse::Raised`]). +fn answered(response: HostCallResponse) -> Result { + match response { + HostCallResponse::Raised(failure) => Err(failure), + response => Ok(response), + } +} + +async fn call_host(call: HostCall) -> HostCallResponse { + let mut call = Some(call); + poll_fn(move |_| { + CURRENT.with_borrow(|current| { + let Some(exchange) = current else { + return Poll::Pending; + }; + let mut exchange = lock(exchange); + if let Some(call) = call.take() { + exchange.call = Some(call); + return Poll::Pending; + } + exchange.response.take().map_or(Poll::Pending, Poll::Ready) + }) + }) + .await +} + +const CHECKED: &str = "the driver checks each response against its host call"; + +fn ttl_seconds(ttl: Option) -> Option { + ttl.map(|ttl| ttl.as_secs_f64()) +} + +/// An increment amount as Python's counters store it: an int when it is whole. +fn amount_value(amount: f64) -> Value { + if amount.fract() == 0.0 && amount.abs() < i64::MAX as f64 { + Value::from(amount as i64) + } else { + Value::from(amount) + } +} + +/// The host's cache: every op is a `cache` step Python runs against the router's `DualCache`. +pub(crate) struct StepCache; + +async fn cache_step( + op: CacheOp, + key: Option<&str>, + keys: Option<&[String]>, + value: Option, + ttl: Option, +) -> CacheOutcome { + call_cache(op, key, keys, value, ttl, false).await +} + +async fn call_cache( + op: CacheOp, + key: Option<&str>, + keys: Option<&[String]>, + value: Option, + ttl: Option, + local: bool, +) -> CacheOutcome { + let call = HostCall::Cache { + op, + key: key.map(str::to_owned), + keys: keys.map(<[String]>::to_vec), + value, + ttl: ttl_seconds(ttl), + local, + }; + let HostCallResponse::Cache(outcome) = call_host(call).await else { + unreachable!("{CHECKED}"); + }; + outcome +} + +#[async_trait] +impl RouterCache for StepCache { + async fn async_get_cache(&self, key: &str) -> Result, litellm_cache::Error> { + let CacheOutcome::Value(value) = + cache_step(CacheOp::Get, Some(key), None, None, None).await + else { + unreachable!("{CHECKED}"); + }; + Ok(value) + } + + async fn async_set_cache( + &self, + key: &str, + value: Value, + ttl: Option, + ) -> Result<(), litellm_cache::Error> { + cache_step(CacheOp::Set, Some(key), None, Some(value), ttl).await; + Ok(()) + } + + async fn async_increment( + &self, + key: &str, + amount: f64, + ttl: Option, + ) -> Result { + let value = Some(amount_value(amount)); + let CacheOutcome::Number(next) = + cache_step(CacheOp::Increment, Some(key), None, value, ttl).await + else { + unreachable!("{CHECKED}"); + }; + Ok(next) + } + + async fn async_batch_get_cache( + &self, + keys: &[String], + ) -> Result>, litellm_cache::Error> { + let CacheOutcome::Values(values) = + cache_step(CacheOp::BatchGet, None, Some(keys), None, None).await + else { + unreachable!("{CHECKED}"); + }; + Ok(values) + } + + async fn async_delete_cache(&self, key: &str) -> Result<(), litellm_cache::Error> { + cache_step(CacheOp::Delete, Some(key), None, None, None).await; + Ok(()) + } + + async fn async_increment_usage( + &self, + increments: &[(String, f64)], + ttl: Option, + ) -> Result<(), litellm_cache::Error> { + let keys: Vec = increments.iter().map(|(key, _)| key.clone()).collect(); + let amounts = increments + .iter() + .map(|(_, amount)| amount_value(*amount)) + .collect(); + let value = Some(Value::Array(amounts)); + cache_step(CacheOp::IncrementUsage, None, Some(&keys), value, ttl).await; + Ok(()) + } + + async fn async_get_local(&self, key: &str) -> Result, litellm_cache::Error> { + let CacheOutcome::Value(value) = + call_cache(CacheOp::Get, Some(key), None, None, None, true).await + else { + unreachable!("{CHECKED}"); + }; + Ok(value) + } + + async fn async_increment_local( + &self, + key: &str, + amount: f64, + ttl: Option, + ) -> Result { + let value = Some(amount_value(amount)); + let CacheOutcome::Number(next) = + call_cache(CacheOp::Increment, Some(key), None, value, ttl, true).await + else { + unreachable!("{CHECKED}"); + }; + Ok(next) + } +} + +/// A deployment as Python spells a `model_list` entry, with its id in `model_info`. +/// Whether two candidates share an id, which a pick answered by id can't tell apart. +fn shares_ids(candidates: &[Arc]) -> bool { + let mut seen = std::collections::HashSet::new(); + !candidates.iter().all(|d| seen.insert(d.id.as_str())) +} + +/// The candidates a host answer names, in its order. An id names every candidate that carries +/// it, so deployments sharing an id stay together through a filter, as Python's list keeps them. +fn answered_by_id(candidates: &[Arc], ids: &[String]) -> Vec> { + let mut positions: std::collections::HashMap<&str, Vec> = + std::collections::HashMap::new(); + for (position, deployment) in candidates.iter().enumerate() { + positions + .entry(deployment.id.as_str()) + .or_default() + .push(position); + } + let mut kept = Vec::new(); + for id in ids { + if let Some(found) = positions.remove(id.as_str()) { + kept.extend( + found + .into_iter() + .map(|position| candidates[position].clone()), + ); + } + } + kept +} + +pub(crate) fn deployment_record(deployment: &Deployment) -> Value { + deployment.record() +} + +const RANDOMNESS_MODULE: &str = "litellm.rust_bridge.host.randomness"; + +/// The process's `random` module: a Python router draws its shuffle and retry jitter there, so +/// a seeded process picks the same deployments on either core. The bare module functions share +/// one generator, so these calls interleave with every other draw the process makes. +pub(crate) struct PythonRandomness; + +impl Randomness for PythonRandomness { + fn next_f64(&self) -> f64 { + Python::attach(|py| -> PyResult { + py.import(RANDOMNESS_MODULE)? + .call_method0("next_float")? + .extract() + }) + .unwrap_or_else(|_| ThreadRandomness.next_f64()) + } + + fn index_below(&self, len: usize) -> usize { + // `random.choice` is `seq[self._randbelow(len(seq))]`, and so is `randrange(len)`. + Python::attach(|py| -> PyResult { + py.import(RANDOMNESS_MODULE)? + .call_method1("index_below", (len,))? + .extract() + }) + .unwrap_or_else(|_| ThreadRandomness.index_below(len)) + } +} + +/// The [`Host`] every `RouterCoreHandle` router runs on. +pub(crate) struct StepHost; + +#[async_trait] +impl Host for StepHost { + async fn invoke(&self, attempt: &Attempt) -> Result { + let call = HostCall::Attempt { + deployment_id: attempt.deployment.id.clone(), + deployment: deployment_record(&attempt.deployment), + model: attempt.name.clone(), + model_group: attempt.model_group.clone(), + routed: attempt.routed.clone(), + target_order: attempt.target_order, + candidate_ids: attempt.candidate_ids.clone(), + retry: attempt.retry, + fallback_depth: attempt.fallback_depth, + fallback_level: attempt.fallback_level, + timeout: attempt.timeout.map(|timeout| timeout.as_secs_f64()), + params: attempt + .params + .as_ref() + .map(|params| Box::new(Value::Object(Map::clone(params)))), + scope: attempt.scope, + max_retries: attempt.max_retries, + model_group_size: attempt.model_group_size, + }; + let (result, blocked, deferred, tracks_usage) = match call_host(call).await { + HostCallResponse::Attempt(result) => (result, None, false, None), + HostCallResponse::Served { + total_tokens, + blocked, + deferred, + tracks_usage, + } => (Ok(total_tokens), blocked, deferred, tracks_usage), + _ => unreachable!("{CHECKED}"), + }; + result.map(|total_tokens| Output { + value: Payload(Arc::new(())), + total_tokens, + blocked, + deferred, + tracks_usage, + }) + } + + async fn select( + &self, + stage: &SelectStage, + ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result>, Error> { + let (scope, group) = ctx.scope.wire(); + let call = HostCall::Select { + stage: match stage { + SelectStage::Plugins => Stage::Plugins, + SelectStage::Strategy(strategy) => Stage::Strategy { + strategy: strategy.to_string(), + records: if shares_ids(&candidates) { + candidates.iter().map(|d| deployment_record(d)).collect() + } else { + Vec::new() + }, + }, + SelectStage::Tags => Stage::Tags, + SelectStage::TextOnly => Stage::TextOnly, + SelectStage::RouteIncapable { handler } => Stage::RouteIncapable { + handler: handler.clone(), + }, + SelectStage::MediaRefusing => Stage::MediaRefusing, + SelectStage::EmbeddingBatch => Stage::EmbeddingBatch, + SelectStage::OverrideCheck(strategy) => Stage::OverrideCheck(strategy.to_string()), + }, + pool: ctx.pool.iter().map(|d| d.id.clone()).collect(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: candidates.iter().map(|d| d.id.clone()).collect(), + }; + match answered(call_host(call).await)? { + HostCallResponse::Select(ids) => Ok(answered_by_id(&candidates, &ids)), + HostCallResponse::Positions(positions) => Ok(positions + .iter() + .map(|position| candidates[*position].clone()) + .collect()), + _ => unreachable!("{CHECKED}"), + } + } + + async fn pre_route(&self, pre: &PreRoute<'_>) -> Result, Error> { + let call = HostCall::PreRoute { + model: pre.name.to_owned(), + fallback_depth: pre.fallback_depth, + retry: pre.retry, + }; + let HostCallResponse::PreRoute(routed) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(routed) + } + + async fn routing_plugin_models( + &self, + ctx: &RoutingContext<'_>, + run: bool, + ) -> Result>, Error> { + let (scope, group) = ctx.scope.wire(); + let call = HostCall::Select { + stage: Stage::RoutingPlugins { run }, + pool: Vec::new(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: Vec::new(), + }; + let HostCallResponse::Models(models) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(models) + } + + async fn provider_request(&self, ctx: &RoutingContext<'_>) -> Result, Error> { + let (scope, group) = ctx.scope.wire(); + let call = HostCall::Select { + stage: Stage::ProviderRequest, + pool: Vec::new(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: Vec::new(), + }; + let HostCallResponse::Models(models) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(models.into_iter().flatten().next()) + } + + async fn pre_call_facts( + &self, + ctx: &RoutingContext<'_>, + candidates: &[Arc], + ) -> Result { + let (scope, group) = ctx.scope.wire(); + let call = HostCall::Select { + stage: Stage::PreCall, + pool: ctx.pool.iter().map(|d| d.id.clone()).collect(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: candidates.iter().map(|d| d.id.clone()).collect(), + }; + let HostCallResponse::PreCall(facts) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(facts) + } + + async fn callback_filters( + &self, + ctx: &RoutingContext<'_>, + candidates: Vec>, + ) -> Result { + let (scope, group) = ctx.scope.wire(); + let joined = |own: &[String], walked: &[String]| [own, walked].concat(); + let call = HostCall::Select { + stage: Stage::CallbackFilters { + excluded_ids: joined(&ctx.request.excluded_ids, ctx.excluded_ids), + skipped_ids: joined(&ctx.request.retry_skipped_ids, ctx.skipped_ids), + target_order: ctx.target_order, + }, + pool: Vec::new(), + model_group: ctx.model_group.to_owned(), + name: ctx.name.to_owned(), + scope, + group: group.map(str::to_owned), + candidates: candidates.iter().map(|d| d.id.clone()).collect(), + }; + let HostCallResponse::Filtered(result) = call_host(call).await else { + unreachable!("{CHECKED}"); + }; + let (ids, preferred) = result.map_err(Error::Attempt)?; + Ok(CallbackFiltered { + kept: answered_by_id(&candidates, &ids), + preferred, + }) + } + + async fn allow_fallback( + &self, + _request: &RouterRequest, + target: &str, + ) -> Result { + let call = HostCall::AllowFallback { + target: target.to_owned(), + }; + let HostCallResponse::AllowFallback(allowed) = answered(call_host(call).await)? else { + unreachable!("{CHECKED}"); + }; + Ok(allowed) + } + + async fn refuses_fallbacks(&self, level: u32, targets: &[Map]) -> bool { + let call = HostCall::RefuseFallbacks { + level, + targets: DictTargets(targets.to_vec()), + }; + let HostCallResponse::RefuseFallbacks(refused) = call_host(call).await else { + unreachable!("{CHECKED}"); + }; + refused + } + + /// A zero sleep is asked too: Python's retry loop awaits `asyncio.sleep(0)` between sibling + /// retries, which yields to the event loop. + async fn sleep(&self, duration: Duration) { + let call = HostCall::Sleep { + seconds: duration.as_secs_f64(), + }; + let HostCallResponse::Slept = call_host(call).await else { + unreachable!("{CHECKED}"); + }; + } + + /// Kept for Python only when it acts on it: an exhausted retry loop is stamped on the + /// error Python raises, a fallback hop tells it which target a failed walk ended on, and a + /// cooldown or a pick is reported to the host's Prometheus and service loggers (a pick also + /// joins the router cache's per-deployment count, a failed walk the router's error log). + fn observe(&self, event: &RouterEvent) { + if !matches!( + event, + RouterEvent::RetriesExhausted { .. } + | RouterEvent::Retry { .. } + | RouterEvent::Fallback { .. } + | RouterEvent::FallingBack { .. } + | RouterEvent::OrderChainFailed { .. } + | RouterEvent::FallbacksFailed { .. } + | RouterEvent::LevelFailed { .. } + | RouterEvent::PickFailed { .. } + | RouterEvent::RetriesOpened { .. } + | RouterEvent::RetryBudget { .. } + | RouterEvent::MockFailed { .. } + | RouterEvent::RetryLogged { .. } + | RouterEvent::RetryPolicyRaised { .. } + | RouterEvent::RetryCountRaised { .. } + | RouterEvent::FallbackDone { .. } + | RouterEvent::Blocked { .. } + | RouterEvent::CooldownSet { .. } + | RouterEvent::Selected { .. } + | RouterEvent::PickStarted { .. } + ) { + return; + } + CURRENT.with_borrow(|current| { + if let Some(exchange) = current { + lock(exchange).events.push(event.clone()); + } + }); + } + + fn cache(&self) -> Arc { + Arc::new(StepCache) + } + + async fn secrets(&self, index: usize, refs: &[SecretRef]) -> Result, Error> { + let HostCallResponse::Secrets(values) = call_host(HostCall::Secrets { + index, + refs: refs.to_vec(), + }) + .await + else { + unreachable!("{CHECKED}"); + }; + Ok(values) + } + + async fn validate(&self, build: &BuildAsk<'_>) -> Result { + let HostCallResponse::Validated(built) = + call_host(build_host_call(StepKind::Validate, build, None)).await + else { + unreachable!("{CHECKED}"); + }; + Ok(built) + } + + async fn provider(&self, build: &BuildAsk<'_>) -> Result { + let HostCallResponse::Provided(provided) = + call_host(build_host_call(StepKind::Provider, build, None)).await + else { + unreachable!("{CHECKED}"); + }; + Ok(provided) + } + + async fn added(&self, build: &BuildAsk<'_>, provider: Option<&str>) -> Result { + let HostCallResponse::Added(built) = + call_host(build_host_call(StepKind::Added, build, provider)).await + else { + unreachable!("{CHECKED}"); + }; + Ok(built) + } + + async fn model_info(&self, lookup: &ModelInfoLookup) -> Result { + let lookup = serde_json::to_value(lookup).unwrap_or(Value::Null); + let HostCallResponse::ModelInfo(response) = call_host(HostCall::ModelInfo { lookup }).await + else { + unreachable!("{CHECKED}"); + }; + Ok(response) + } +} + +fn build_host_call(kind: StepKind, build: &BuildAsk<'_>, provider: Option<&str>) -> HostCall { + HostCall::Build { + kind, + path: build.path, + index: build.index, + entry: Value::Object(build.entry.clone()), + declared_id: build.declared_id.map(str::to_owned), + duplicate_id: build.duplicate_id, + provider: provider.map(str::to_owned), + } +} + +type CallFuture = Pin> + Send>>; + +/// Poll `future` once with `exchange` bound. +fn poll_bound(future: Pin<&mut F>, exchange: &Shared) -> Poll { + let _bound = Bound::new(exchange); + future.poll(&mut Context::from_waker(Waker::noop())) +} + +/// Drives one router call a step at a time. +pub(crate) struct Driver { + future: Option, + exchange: Shared, + pending_call: Option, +} + +impl Driver { + pub(crate) fn new(call: impl Future> + Send + 'static) -> Self { + Self { + future: Some(Box::pin(call)), + exchange: Shared::default(), + pending_call: None, + } + } + + /// The host call the router call is blocked on, if it made one. + pub(crate) fn pending_call(&self) -> Option<&HostCall> { + self.pending_call.as_ref() + } + + /// The events reported since the last call. + pub(crate) fn take_events(&self) -> Vec { + std::mem::take(&mut lock(&self.exchange).events) + } + + /// Hand back the response to the pending host call (`None` on the first call) and run the call to + /// its next step or its end. + pub(crate) fn resume( + &mut self, + response: Option, + ) -> Result { + let Some(future) = self.future.as_mut() else { + return Err(DriveError::Finished); + }; + match (&self.pending_call, response) { + (None, None) => {} + (None, Some(_)) => return Err(DriveError::UnexpectedResponse), + (Some(call), None) => { + return Err(DriveError::MissingResponse { + expected: call.kind(), + }); + } + (Some(call), Some(response)) => { + response.check(call)?; + lock(&self.exchange).response = Some(response); + } + } + self.pending_call = None; + match poll_bound(future.as_mut(), &self.exchange) { + Poll::Ready(result) => { + self.future = None; + Ok(match result { + Ok(finish) => Step::Done(finish), + Err(error) => Step::Failed(error), + }) + } + Poll::Pending => { + let Some(call) = lock(&self.exchange).call.take() else { + self.future = None; + return Err(DriveError::Stalled); + }; + self.pending_call = Some(call.clone()); + Ok(Step::HostCall(call)) + } + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::HashMap, + time::{SystemTime, UNIX_EPOCH}, + }; + + use litellm_router_core::{ + FailureKind, Op, Router, RouterConfig, RouterDynamicConfig, SeededRandomness, + }; + use rstest::{fixture, rstest}; + use serde_json::{Value, json}; + + use super::*; + + fn router(config: Value, dynamic: Value) -> Router { + let config: RouterConfig = serde_json::from_value(config).unwrap(); + let dynamic: RouterDynamicConfig = serde_json::from_value(dynamic).unwrap(); + Router::builder(config, dynamic, Arc::new(StepHost)) + .randomness(Arc::new(SeededRandomness::new(7))) + .build() + .unwrap() + } + + fn deployment(group: &str, id: &str) -> Value { + json!({"model_name": group, "litellm_params": {"model": format!("openai/{id}")}, "model_info": {"id": id}}) + } + + fn start(router: &Router, model: &str) -> Driver { + let router = router.clone(); + let request = RouterRequest { + model: model.to_owned(), + ..RouterRequest::default() + }; + Driver::new(async move { + router + .call(Op::from("acompletion"), request) + .await + .map(Finish::Call) + }) + } + + fn rate_limited() -> HostCallResponse { + HostCallResponse::Attempt(Err(AttemptError::new(FailureKind::RateLimit, Some(429)))) + } + + /// A dict-backed stand-in for Python's `DualCache` that answers `cache` steps and keeps + /// every one it answered. + #[derive(Default)] + struct Store { + entries: HashMap, + calls: Vec, + } + + impl Store { + fn respond(&mut self, call: &HostCall) -> HostCallResponse { + let HostCall::Cache { + op, + key, + keys, + value, + .. + } = call + else { + panic!("not a cache step: {call:?}"); + }; + self.calls.push(call.clone()); + let key = key.clone().unwrap_or_default(); + HostCallResponse::Cache(match op { + CacheOp::Get => CacheOutcome::Value(self.entries.get(&key).cloned()), + CacheOp::Set => { + self.entries.insert(key, value.clone().unwrap()); + CacheOutcome::Stored + } + CacheOp::Increment => { + let current = self.entries.get(&key).and_then(Value::as_f64); + let next = + current.unwrap_or(0.0) + value.as_ref().and_then(Value::as_f64).unwrap(); + self.entries.insert(key, amount_value(next)); + CacheOutcome::Number(next) + } + CacheOp::IncrementUsage => { + let amounts = value.as_ref().and_then(Value::as_array).unwrap(); + for (key, amount) in keys.iter().flatten().zip(amounts) { + let current = self.entries.get(key).and_then(Value::as_f64); + let next = current.unwrap_or(0.0) + amount.as_f64().unwrap(); + self.entries.insert(key.clone(), amount_value(next)); + } + CacheOutcome::Stored + } + CacheOp::BatchGet => CacheOutcome::Values( + keys.iter() + .flatten() + .map(|key| self.entries.get(key).cloned()) + .collect(), + ), + CacheOp::Delete => { + self.entries.remove(&key); + CacheOutcome::Stored + } + }) + } + + /// Resume with `response`, answering every cache step on the way, up to the next other + /// step. + fn resume(&mut self, driver: &mut Driver, response: Option) -> Step { + let mut step = driver.resume(response).unwrap(); + while let Step::HostCall(call @ HostCall::Cache { .. }) = &step { + let response = self.respond(call); + step = driver.resume(Some(response)).unwrap(); + } + step + } + } + + #[fixture] + fn pair() -> Router { + router( + json!({"model_list": [deployment("primary", "a1"), deployment("primary", "a2")]}), + json!({}), + ) + } + + #[rstest] + fn a_first_attempt_success_is_done(pair: Router) { + let mut store = Store::default(); + let mut driver = start(&pair, "primary"); + let Step::HostCall(HostCall::Attempt { + deployment_id, + model_group, + retry: 0, + fallback_depth: 0, + .. + }) = store.resume(&mut driver, None) + else { + panic!("the call should start with an attempt"); + }; + assert_eq!(model_group, "primary"); + let Step::Done(Finish::Call(response)) = + store.resume(&mut driver, Some(HostCallResponse::Attempt(Ok(12)))) + else { + panic!("a successful attempt should finish the call"); + }; + assert_eq!(response.deployment_id, deployment_id); + assert_eq!(response.output.total_tokens, 12); + assert_eq!( + (response.attempted_retries, response.attempted_fallbacks), + (0, 0) + ); + assert!(matches!(driver.resume(None), Err(DriveError::Finished))); + } + + #[rstest] + fn cooldown_reads_are_cache_steps(pair: Router) { + let mut driver = start(&pair, "primary"); + let step = driver.resume(None).unwrap(); + let Step::HostCall(HostCall::Cache { + op: CacheOp::BatchGet, + key: None, + keys: Some(keys), + value: None, + ttl: None, + local: false, + }) = step + else { + panic!("the call should read cooldowns first, got {step:?}"); + }; + assert_eq!(keys, ["deployment:a1:cooldown", "deployment:a2:cooldown"]); + } + + #[rstest] + #[case::lone_deployment_backs_off(vec![deployment("primary", "a1")], true)] + #[case::sibling_retries_at_once( + vec![deployment("primary", "a1"), deployment("primary", "a2")], + false + )] + fn a_rate_limit_retries_after_the_backoff( + #[case] model_list: Vec, + #[case] backs_off: bool, + ) { + let router = router(json!({"model_list": model_list}), json!({"num_retries": 1})); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + assert_eq!( + store.resume(&mut driver, None).unwrap_kind(), + StepKind::Attempt + ); + let step = store.resume(&mut driver, Some(rate_limited())); + let Step::HostCall(HostCall::Sleep { seconds }) = step else { + panic!("a retry should sleep first, got {step:?}"); + }; + assert_eq!(seconds > 0.0, backs_off, "slept {seconds}s"); + let step = store.resume(&mut driver, Some(HostCallResponse::Slept)); + assert!( + matches!(step, Step::HostCall(HostCall::Attempt { retry: 1, .. })), + "expected the retry, got {step:?}" + ); + let Step::Done(Finish::Call(response)) = + store.resume(&mut driver, Some(HostCallResponse::Attempt(Ok(1)))) + else { + panic!("the retry should finish the call"); + }; + assert_eq!(response.attempted_retries, 1); + } + + #[rstest] + fn a_cooldown_python_wrote_is_read_back_by_the_next_call() { + let pair_without_retries = router( + json!({"model_list": [deployment("primary", "a1"), deployment("primary", "a2")]}), + json!({"num_retries": 0}), + ); + let mut store = Store::default(); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs_f64(); + store.entries.insert( + "deployment:a1:cooldown".into(), + json!({"exception_received": "slow down", "status_code": "429", "timestamp": now, "cooldown_time": 60}), + ); + for _ in 0..8 { + let mut call = start(&pair_without_retries, "primary"); + let Step::HostCall(HostCall::Attempt { deployment_id, .. }) = + store.resume(&mut call, None) + else { + panic!("the call should attempt the healthy deployment"); + }; + assert_eq!(deployment_id, "a2", "the cooled-down deployment was picked"); + } + } + + /// The host only runs the attempt; the core records its outcome (cooldowns, fail counters, + /// usage) through the host's cache. + #[rstest] + #[case::failure(rate_limited())] + #[case::success(HostCallResponse::Attempt(Ok(7)))] + fn attempt_outcomes_are_recorded_by_the_core(#[case] outcome: HostCallResponse) { + let tracked = |id: &str| { + json!({ + "model_name": "primary", + "litellm_params": {"model": format!("openai/{id}"), "tpm": 100, "cooldown_time": 7}, + "model_info": {"id": id, "allowed_fails": 3}, + }) + }; + let router = router( + json!({"model_list": [tracked("a1"), tracked("a2")]}), + json!({"num_retries": 0}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + store.resume(&mut driver, None); + let step = store.resume(&mut driver, Some(outcome)); + assert!( + matches!(step, Step::Done(_) | Step::Failed(_)), + "got {step:?}" + ); + let writes: Vec<&HostCall> = store + .calls + .iter() + .filter(|call| { + matches!(call, HostCall::Cache { op, .. } if !matches!(op, CacheOp::Get | CacheOp::BatchGet)) + }) + .collect(); + assert!(!writes.is_empty(), "the core wrote no outcome"); + } + + #[rstest] + fn a_failed_group_falls_back_once_python_allows_it() { + let router = router( + json!({"model_list": [deployment("primary", "a1"), deployment("backup", "b1")]}), + json!({"fallbacks": [{"primary": ["backup"]}]}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + assert_eq!( + store.resume(&mut driver, None).unwrap_kind(), + StepKind::Attempt + ); + let bad_request = + HostCallResponse::Attempt(Err(AttemptError::new(FailureKind::BadRequest, Some(400)))); + let step = store.resume(&mut driver, Some(bad_request)); + assert!( + matches!(&step, Step::HostCall(HostCall::AllowFallback { target }) if target == "backup"), + "expected the fallback check, got {step:?}" + ); + let step = store.resume(&mut driver, Some(HostCallResponse::AllowFallback(true))); + assert!( + matches!(&step, Step::HostCall(HostCall::Attempt { deployment_id, fallback_depth: 1, .. }) if deployment_id == "b1"), + "expected the fallback attempt, got {step:?}" + ); + let Step::Done(Finish::Call(response)) = + store.resume(&mut driver, Some(HostCallResponse::Attempt(Ok(3)))) + else { + panic!("the fallback should finish the call"); + }; + assert_eq!(response.model_group, "backup"); + assert_eq!(response.attempted_fallbacks, 1); + } + + #[rstest] + fn a_refused_fallback_fails_with_the_attempt_error() { + let router = router( + json!({"model_list": [deployment("primary", "a1"), deployment("backup", "b1")]}), + json!({"fallbacks": [{"primary": ["backup"]}]}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + store.resume(&mut driver, None); + let bad_request = + HostCallResponse::Attempt(Err(AttemptError::new(FailureKind::BadRequest, Some(400)))); + store.resume(&mut driver, Some(bad_request)); + let Step::Failed(error) = + store.resume(&mut driver, Some(HostCallResponse::AllowFallback(false))) + else { + panic!("a refused fallback should fail the call"); + }; + assert_eq!( + error.attempt().map(|a| a.kind), + Some(FailureKind::BadRequest) + ); + } + + #[rstest] + #[case::none_while_waiting(None, StepKind::Attempt)] + #[case::wrong_kind(Some(HostCallResponse::Slept), StepKind::Attempt)] + #[case::cache_outcome(Some(HostCallResponse::Cache(CacheOutcome::Stored)), StepKind::Attempt)] + fn a_mismatched_outcome_is_rejected_and_the_step_kept( + pair: Router, + #[case] response: Option, + #[case] expected: StepKind, + ) { + let mut store = Store::default(); + let mut driver = start(&pair, "primary"); + store.resume(&mut driver, None); + let error = driver.resume(response).unwrap_err(); + assert!(error.is_request(), "{error}"); + assert_eq!(driver.pending_call().map(HostCall::kind), Some(expected)); + assert!(matches!( + store.resume(&mut driver, Some(HostCallResponse::Attempt(Ok(1)))), + Step::Done(_) + )); + } + + #[rstest] + #[case::wrong_op(HostCallResponse::Cache(CacheOutcome::Stored))] + #[case::number_for_batch(HostCallResponse::Cache(CacheOutcome::Number(1.0)))] + #[case::short_batch(HostCallResponse::Cache(CacheOutcome::Values(vec![None])))] + #[case::not_a_cache_outcome(HostCallResponse::Slept)] + fn a_mismatched_cache_outcome_is_rejected_and_the_step_kept( + pair: Router, + #[case] response: HostCallResponse, + ) { + let mut store = Store::default(); + let mut driver = start(&pair, "primary"); + let step = driver.resume(None).unwrap(); + let Step::HostCall( + call @ HostCall::Cache { + op: CacheOp::BatchGet, + .. + }, + ) = &step + else { + panic!("expected the cooldown read, got {step:?}"); + }; + let error = driver.resume(Some(response)).unwrap_err(); + assert!(error.is_request(), "{error}"); + assert_eq!(driver.pending_call(), Some(call)); + let response = store.respond(call); + assert_eq!( + store.resume(&mut driver, Some(response)).unwrap_kind(), + StepKind::Attempt + ); + } + + #[rstest] + fn a_strategy_python_runs_asks_for_a_selection_and_keeps_the_pick() { + let router = router( + json!({ + "model_list": [deployment("primary", "a1"), deployment("primary", "a2")], + "routing_strategy": "least-busy", + }), + json!({}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + let step = store.resume(&mut driver, None); + let Step::HostCall(HostCall::Select { + stage: Stage::Strategy { strategy, records }, + candidates, + .. + }) = step + else { + panic!("a host strategy should ask for a selection, got {step:?}"); + }; + assert_eq!(strategy, "least-busy"); + assert_eq!(candidates, ["a1", "a2"]); + assert!(records.is_empty(), "distinct ids send no records"); + let step = store.resume( + &mut driver, + Some(HostCallResponse::Select(vec!["a2".to_owned()])), + ); + assert!( + matches!(&step, Step::HostCall(HostCall::Attempt { deployment_id, .. }) if deployment_id == "a2"), + "expected the picked deployment, got {step:?}" + ); + } + + #[rstest] + fn an_answer_by_id_keeps_its_order_and_every_candidate_sharing_an_id() { + let candidates: Vec> = ["a", "b", "a", "c"] + .into_iter() + .map(|id| { + let spec: litellm_router_core::DeploymentSpec = + serde_json::from_value(deployment("g", id)).unwrap(); + Arc::new(Deployment::from_spec(spec).unwrap()) + }) + .collect(); + let ids = ["c", "a", "x", "a"].map(String::from); + + let kept = answered_by_id(&candidates, &ids); + + let positions: Vec = kept + .iter() + .map(|d| candidates.iter().position(|c| Arc::ptr_eq(c, d)).unwrap()) + .collect(); + assert_eq!(positions, [3, 0, 2]); + } + #[rstest] + fn a_pick_among_deployments_sharing_an_id_is_answered_by_position() { + let mut second = deployment("primary", "a1"); + second["litellm_params"]["api_base"] = json!("https://second.example"); + let router = router( + json!({ + "model_list": [deployment("primary", "a1"), second], + "routing_strategy": "least-busy", + }), + json!({}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + let step = store.resume(&mut driver, None); + let Step::HostCall(HostCall::Select { + stage: Stage::Strategy { records, .. }, + .. + }) = step + else { + panic!("a host strategy should ask for a selection, got {step:?}"); + }; + assert_eq!( + records.len(), + 2, + "each candidate sharing an id sends its record" + ); + let step = store.resume(&mut driver, Some(HostCallResponse::Positions(vec![1]))); + assert!( + matches!(&step, Step::HostCall(HostCall::Attempt { deployment, .. }) + if deployment["litellm_params"]["api_base"] == "https://second.example"), + "expected the second entry, got {step:?}" + ); + } + + #[rstest] + fn a_selection_is_driven_through_cache_steps(pair: Router) { + let request = RouterRequest { + model: "primary".to_owned(), + ..RouterRequest::default() + }; + let mut store = Store::default(); + let mut driver = Driver::new(async move { + pair.select(&Op::from("acompletion"), &request) + .await + .map(|deployment| Finish::Selected { + deployment_id: deployment.id.clone(), + model_group: deployment.model_name.clone(), + deployment: deployment_record(&deployment), + }) + }); + let Step::Done(Finish::Selected { + deployment_id, + model_group, + .. + }) = store.resume(&mut driver, None) + else { + panic!("a selection should finish with the pick"); + }; + assert!(["a1", "a2"].contains(&deployment_id.as_str())); + assert_eq!(model_group, "primary"); + assert!(matches!( + store.calls.as_slice(), + [HostCall::Cache { + op: CacheOp::BatchGet, + .. + }] + )); + } + + #[rstest] + fn dict_targets_keep_their_api_key_out_of_debug() { + let target = json!({"model": "b", "api_key": "credential-value"}); + let Value::Object(target) = target else { + unreachable!("a JSON object"); + }; + let shown = format!("{:?}", DictTargets(vec![target])); + assert!(!shown.contains("credential-value")); + assert!(shown.contains("[REDACTED]")); + } + + #[rstest] + fn a_chain_with_a_dict_target_asks_python_before_any_hop() { + let router = router( + json!({"model_list": [deployment("primary", "a1"), deployment("backup", "b1")]}), + json!({"fallbacks": [{"primary": [{"model": "backup", "api_key": "k"}]}]}), + ); + let mut store = Store::default(); + let mut driver = start(&router, "primary"); + store.resume(&mut driver, None); + let bad_request = + HostCallResponse::Attempt(Err(AttemptError::new(FailureKind::BadRequest, Some(400)))); + let step = store.resume(&mut driver, Some(bad_request)); + assert!( + matches!(&step, Step::HostCall(HostCall::RefuseFallbacks { level: 0, targets }) if targets.0.len() == 1), + "expected the chain's check, got {step:?}" + ); + let Step::Failed(error) = + store.resume(&mut driver, Some(HostCallResponse::RefuseFallbacks(true))) + else { + panic!("a refused chain should fail the call"); + }; + assert_eq!( + error.attempt().map(|a| a.kind), + Some(FailureKind::BadRequest) + ); + } + + trait StepKindOf { + fn unwrap_kind(self) -> StepKind; + } + + impl StepKindOf for Step { + fn unwrap_kind(self) -> StepKind { + match self { + Step::HostCall(call) => call.kind(), + Step::Done(_) => StepKind::Done, + Step::Failed(_) => StepKind::Failed, + } + } + } +} diff --git a/litellm-rust/crates/python-bridge/src/router/mod.rs b/litellm-rust/crates/python-bridge/src/router/mod.rs new file mode 100644 index 00000000000..6b58d63a714 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/mod.rs @@ -0,0 +1,2087 @@ +//! `RouterCoreHandle`: router-core behind a step-driven handle. Python drives each call through +//! [`RouterCall::next`], running every attempt, sleep, fallback check, selection and cache op the +//! call loop asks for and handing the outcome back; Rust never awaits Python. `select` is driven +//! the same way, since reading cooldowns is a cache step. + +mod config; +mod host; +mod retries; +mod stream; + +use std::{ + sync::{Arc, Mutex, PoisonError}, + time::{Duration, SystemTime}, +}; + +use litellm_error::Report; +use litellm_host_python::{from_py_argument, json_loads, to_py}; +use litellm_router_core::lookup::{ListScope, Lookup, Row}; +use litellm_router_core::{ + AttemptError, Blocked, BuildPath, Built, Deployment, DeploymentSpec, Error, FailureKind, + FailureReport, Op, OutcomeCallbacks, PreRouted, Provided, RetryCause, Router, RouterEvent, + SecretAt, SuccessReport, + build::Install, + health::{UsageMetric, current_minute, usage_key}, + registry::{DeploymentFilter, ResolveKind, ResolveScope}, + strategy::routing_context, +}; +use pyo3::{ + exceptions::{PyRuntimeError, PyValueError}, + prelude::*, + types::{PyBool, PyDict, PyList, PyString, PyTuple}, +}; +use serde_json::Value; + +use self::host::{ + CacheOp, CacheOutcome, DriveError, Driver, Finish, HostCall, HostCallResponse, + PythonRandomness, Stage, Step, StepHost, StepKind, +}; +use crate::errors::RustBridgeDeclined; + +/// The `error_kind` of a `failed` step. +#[derive(Clone, Copy, Debug, strum::IntoStaticStr)] +enum ErrorKind { + #[strum(serialize = "config")] + Config, + #[strum(serialize = "unknown_model")] + UnknownModel, + #[strum(serialize = "no_healthy_deployments")] + NoHealthyDeployments, + #[strum(serialize = "attempt")] + Attempt, + #[strum(serialize = "cache")] + Cache, + #[strum(serialize = "retired")] + Retired, + #[strum(serialize = "paused")] + Paused, + #[strum(serialize = "not_pass_through")] + NotPassThrough, + #[strum(serialize = "no_pass_through_deployments")] + NoPassThroughDeployments, + #[strum(serialize = "only_markers")] + OnlyMarkers, + #[strum(serialize = "endpoint_not_supported")] + EndpointNotSupported, + #[strum(serialize = "ptu_reserved")] + PtuReserved, + #[strum(serialize = "context_window_exceeded")] + ContextWindowExceeded, + #[strum(serialize = "unsupported_params")] + UnsupportedParams, + #[strum(serialize = "routing_plugins_left_nothing")] + RoutingPluginsLeftNothing, + #[strum(serialize = "routing_plugins_need_async")] + RoutingPluginsNeedAsync, + #[strum(serialize = "multiple_teams")] + MultipleTeams, + #[strum(serialize = "reserved")] + Reserved, + #[strum(serialize = "invalid_router_weights")] + InvalidRouterWeights, + #[strum(serialize = "non_numeric_weight")] + NonNumericWeight, + #[strum(serialize = "missing_routing_group")] + MissingRoutingGroup, + #[strum(serialize = "default_deployment_raises")] + DefaultDeploymentRaises, + #[strum(serialize = "index_out_of_range")] + IndexOutOfRange, + #[strum(serialize = "model_info")] + ModelInfo, +} + +impl From<&Error> for ErrorKind { + fn from(error: &Error) -> Self { + match error { + Error::Config(_) => Self::Config, + Error::UnknownModel { .. } => Self::UnknownModel, + Error::NoHealthyDeployments { .. } => Self::NoHealthyDeployments, + Error::Attempt(_) => Self::Attempt, + Error::Cache(_) => Self::Cache, + Error::Retired => Self::Retired, + Error::Paused { .. } => Self::Paused, + Error::NotPassThrough { .. } => Self::NotPassThrough, + Error::NoPassThroughDeployments { .. } => Self::NoPassThroughDeployments, + Error::OnlyMarkers { .. } => Self::OnlyMarkers, + Error::EndpointNotSupported { .. } => Self::EndpointNotSupported, + Error::PtuReserved { .. } => Self::PtuReserved, + Error::ContextWindowExceeded { .. } => Self::ContextWindowExceeded, + Error::UnsupportedParams { .. } => Self::UnsupportedParams, + Error::RoutingPluginsLeftNothing { .. } => Self::RoutingPluginsLeftNothing, + Error::RoutingPluginsNeedAsync => Self::RoutingPluginsNeedAsync, + Error::MultipleTeams { .. } => Self::MultipleTeams, + Error::Reserved { .. } => Self::Reserved, + Error::InvalidRouterWeights => Self::InvalidRouterWeights, + Error::NonNumericWeight { .. } => Self::NonNumericWeight, + Error::MissingRoutingGroup { .. } => Self::MissingRoutingGroup, + Error::DefaultDeploymentRaises { .. } => Self::DefaultDeploymentRaises, + Error::IndexOutOfRange { .. } => Self::IndexOutOfRange, + Error::ModelInfo { .. } => Self::ModelInfo, + } + } +} + +fn config_error_to_pyerr(error: config::Error) -> PyErr { + let message = Report::new(&error).to_string(); + if error.declined() { + RustBridgeDeclined::new_err(message) + } else { + PyValueError::new_err(message) + } +} + +fn drive_error_to_pyerr(error: DriveError) -> PyErr { + let message = Report::new(&error).to_string(); + if error.is_request() { + PyValueError::new_err(message) + } else { + PyRuntimeError::new_err(message) + } +} + +/// A document Python passes as a mapping or as its JSON text. +fn document(value: &Bound<'_, PyAny>) -> PyResult { + match value.cast::() { + Ok(text) => { + let parsed = json_loads(value.py(), text.to_str()?.as_bytes())?; + from_py_argument(parsed.bind(value.py())) + } + Err(_) => from_py_argument(value), + } +} + +/// The `OutcomeCallbacks` a reported outcome came through; none is a report no callback makes. +fn outcome_callbacks(value: Option<&Bound<'_, PyAny>>) -> PyResult { + let Some(value) = value else { + return Ok(OutcomeCallbacks::default()); + }; + serde_json::from_value(document(value)?) + .map_err(|error| PyValueError::new_err(error.to_string())) +} + +/// A router-core router whose host operations Python runs. +#[pyclass(frozen)] +pub(crate) struct RouterCoreHandle { + router: Router, +} + +#[pymethods] +impl RouterCoreHandle { + #[new] + fn new(config: &Bound<'_, PyAny>, dynamic: &Bound<'_, PyAny>) -> PyResult { + let (config, dynamic) = (document(config)?, document(dynamic)?); + let build = || { + let config = config::parse_router_config(config)?; + let dynamic = config::parse_dynamic_config(dynamic)?; + Ok(Router::builder(config, dynamic, Arc::new(StepHost)) + .randomness(Arc::new(PythonRandomness)) + .build()?) + }; + build() + .map(|router| Self { router }) + .map_err(config_error_to_pyerr) + } + + /// Start a call; nothing runs until the first `next(None)`. + fn start(&self, op: &str, request: &Bound<'_, PyAny>) -> PyResult { + let op = Op::new(op); + let request = + config::parse_request(document(request)?, None).map_err(config_error_to_pyerr)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + router.call(op, request).await.map(Finish::Call) + })) + } + + /// Pick the deployment `model` routes to now, without calling it. Driven like `start`: its + /// steps are the cache reads and selections the pick needs, and it ends with `done` + /// (`deployment_id`, `model_group`) or `failed`. + fn select(&self, model: &str, request: &Bound<'_, PyAny>) -> PyResult { + let request = config::parse_request(document(request)?, Some(model)) + .map_err(config_error_to_pyerr)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let deployment = router.select(&Op::from("acompletion"), &request).await?; + Ok(Finish::Selected { + deployment_id: deployment.id.clone(), + model_group: deployment.model_name.clone(), + deployment: host::deployment_record(&deployment), + }) + })) + } + + /// List the deployments a pick for `model` would choose among, driven like `select`. + fn healthy(&self, model: &str, request: &Bound<'_, PyAny>) -> PyResult { + let request = config::parse_request(document(request)?, Some(model)) + .map_err(config_error_to_pyerr)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let candidates = router.healthy(&Op::from("acompletion"), &request).await?; + // A deployment named by its id is listed alone, as a dict (`_core_async_get_healthy_deployments`). + let single = candidates.kind.is_single() && candidates.healthy.len() == 1; + let deployments = candidates.healthy; + Ok(Finish::Healthy { + single, + deployment_ids: deployments.iter().map(|d| d.id.clone()).collect(), + deployments: deployments + .iter() + .map(|d| host::deployment_record(d)) + .collect(), + }) + })) + } + + /// Apply `{"upsert": [deployments], "delete": [ids], "settings": {dynamic document}}` and + /// return the new registry version. + fn apply(&self, changes: &Bound<'_, PyAny>) -> PyResult { + let changes = config::parse_changes(document(changes)?).map_err(config_error_to_pyerr)?; + self.router + .apply(changes) + .map_err(|error| config_error_to_pyerr(error.into())) + } + + /// Build `entries` (model_list entries) through the host, driven like `start`: its steps + /// are `secrets`, `validate`, `provider` and `added`, and it ends with `done` (`built`: one + /// `{"index", "id", "provider"}` per deployment installed). `path` is `create` + /// (`set_model_list`) or `add` (`add_deployment`); `install` is `replace`, `append` or + /// `upsert`. + #[pyo3(signature = (entries, path, resolve_secrets, install))] + fn build( + &self, + entries: &Bound<'_, PyAny>, + path: &str, + resolve_secrets: bool, + install: &str, + ) -> PyResult { + let entries: Vec> = + serde_json::from_value(document(entries)?) + .map_err(|error| PyValueError::new_err(format!("bad model_list: {error}")))?; + let path: BuildPath = path + .parse() + .map_err(|_| PyValueError::new_err(format!("`{path}` is not a build path")))?; + let install: Install = install + .parse() + .map_err(|_| PyValueError::new_err(format!("`{install}` is not a build install")))?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + router + .build(path, entries, resolve_secrets, install) + .await + .map(Finish::Built) + })) + } + + /// `get_router_model_info` for the deployment `query` describes (a + /// `RouterModelInfoQuery` document), driven like `start`: its steps are `model_info` + /// lookups, and it ends with `done` (`info`: the merged map or None, `user_keys`: the keys + /// whose values are the deployment's own). + fn router_model_info(&self, query: &Bound<'_, PyAny>) -> PyResult { + let query: litellm_router_core::model_info::RouterModelInfoQuery = + serde_json::from_value(document(query)?) + .map_err(|error| PyValueError::new_err(format!("bad model-info query: {error}")))?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + router + .router_model_info(&query) + .await + .map(Finish::RouterModelInfo) + })) + } + + /// `get_deployment_model_info` for deployment `model_id` serving `model_name`, driven like + /// `router_model_info`; it ends with `done` (`info`: the merged map or None). + fn deployment_model_info( + &self, + model_id: &str, + model_name: &Bound<'_, PyAny>, + ) -> PyResult { + let model_id = model_id.to_owned(); + let model_name = from_py_argument::(model_name)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + router + .deployment_model_info(&model_id, &model_name) + .await + .map(Finish::DeploymentModelInfo) + })) + } + + /// `_set_model_group_info` for the deployments `query` names (a `GroupInfoQuery` document), + /// driven like `router_model_info`; it ends with `done` (`value`: None when no deployment + /// matched, else `{"assignments": {field: value}}`, what the merge set on the group the host + /// built from its first deployment). + fn model_group_info(&self, query: &Bound<'_, PyAny>) -> PyResult { + let query: litellm_router_core::group_info::GroupInfoQuery = + serde_json::from_value(document(query)?) + .map_err(|error| PyValueError::new_err(format!("bad group-info query: {error}")))?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let info = router.model_group_info(&query).await?; + Ok(Finish::Value(info.map_or( + Value::Null, + |info| serde_json::json!({"assignments": info.assignments}), + ))) + })) + } + + /// `get_configured_token_limits` for the deployment with `id` and its `configured` model_info, + /// driven like `router_model_info`; it ends with `done` (`value`: `[max_input, max_output]`). + fn configured_token_limits( + &self, + id: &Bound<'_, PyAny>, + configured: &Bound<'_, PyAny>, + ) -> PyResult { + let id = from_py_argument::(id)?; + let Value::Object(configured) = document(configured)? else { + return Err(PyValueError::new_err( + "the configured model_info must be a dict", + )); + }; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let (input, output) = router.configured_token_limits(id, &configured).await?; + Ok(Finish::Value(serde_json::json!([input, output]))) + })) + } + + /// `get_model_listing_info` over the `deployments` a listed name names, driven like + /// `router_model_info`; it ends with `done` (`value`: `{"cost_map_keys", "max_input_tokens", + /// "max_output_tokens"}`). + fn model_listing_info(&self, deployments: &Bound<'_, PyAny>) -> PyResult { + let deployments: Vec = from_py_argument(deployments)?; + let router = self.router.clone(); + Ok(RouterCall::new(async move { + let listing = router.model_listing_info(&deployments).await?; + Ok(Finish::Value(serde_json::json!({ + "cost_map_keys": listing.cost_map_keys, + "max_input_tokens": listing.max_input_tokens, + "max_output_tokens": listing.max_output_tokens, + }))) + })) + } + + /// `get_configured_mode` / `get_configured_display_name`: `value` when it is a string that is + /// not blank. + #[staticmethod] + fn configured_text(value: &Bound<'_, PyAny>) -> PyResult> { + let value = from_py_argument::(value).ok(); + Ok(litellm_router_core::group_info::configured_text( + value.as_ref(), + )) + } + + /// `get_configured_service_tiers`: each of `model_infos`' `service_tiers`, unvalidated. + #[staticmethod] + fn configured_service_tiers( + py: Python<'_>, + model_infos: &Bound<'_, PyAny>, + ) -> PyResult>> { + let model_infos: Vec> = from_py_argument(model_infos)?; + litellm_router_core::group_info::configured_service_tiers(&model_infos) + .iter() + .map(|tiers| to_py(py, tiers)) + .collect() + } + + /// The routing-cache key deployment `id` serving litellm `model` counts `metric` (`tpm`, + /// `rpm`, `itpm` or `otpm`) under in `minute` (`HH-MM`). + fn usage_key(&self, id: &str, model: &str, metric: &str, minute: &str) -> PyResult { + let metric: UsageMetric = metric + .parse() + .map_err(|_| PyValueError::new_err(format!("`{metric}` is not a usage metric")))?; + Ok(usage_key(id, model, metric, minute)) + } + + /// Keep the providers `other`'s build resolved for the deployments this router serves too. + fn adopt_providers(&self, other: &Self) { + self.router.adopt_providers(&other.router); + } + + fn registry_version(&self) -> u64 { + self.router.registry().version() + } + + /// The version the deployments before the newest appends last changed at: a list read at + /// or after it is current once it reads `deployments_from(len(list))`. + fn registry_rewritten(&self) -> u64 { + self.router.registry().rewritten() + } + + /// Record a failure the router did not see itself (a health check, an interceptor, a + /// logging callback), driven like `start`: its steps are cache ops, and it ends with `done` + /// (`cooled`, `cooldown_time`). `counts_request` also counts the failed request against the deployment's + /// per-minute requests, as an async call's failure callback does. `callbacks` is the logged + /// request's `OutcomeCallbacks`. + #[pyo3(signature = (id, failure, deployment_cooldown=false, count_only=false, requested_group=None, cooldown_time=None, uncounted=false, counts_request=false, usage_model=None, callbacks=None))] + #[allow(clippy::too_many_arguments)] + fn report_failure( + &self, + id: &str, + failure: &Bound<'_, PyDict>, + deployment_cooldown: bool, + count_only: bool, + requested_group: Option, + cooldown_time: Option, + uncounted: bool, + counts_request: bool, + usage_model: Option, + callbacks: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let failure = match attempt_outcome(failure)? { + Err(failure) => failure, + Ok(_) => { + return Err(PyValueError::new_err( + "a reported failure needs `ok: False`", + )); + } + }; + let router = self.router.clone(); + let id = id.to_owned(); + let report = FailureReport { + deployment_cooldown, + cooldown_time, + count_only, + requested_group, + uncounted, + counts_request, + usage_model, + callbacks: outcome_callbacks(callbacks)?, + }; + Ok(RouterCall::new(async move { + router + .report_failure(&id, &failure, report) + .await + .map(|cooldown| Finish::Reported { cooldown }) + })) + } + + /// Record a success a logging callback saw, driven like `report_failure`; it ends with + /// `done` (`key`: the tokens counter's key when usage was added, else None). + /// `usage_model` is the model its usage is keyed by, and `tracks_usage` the host's answer to + /// whether it counts any (see `Output::tracks_usage`). `callbacks`: see `report_failure`. + #[pyo3(signature = (id, total_tokens, requests, usage_model=None, tracks_usage=None, callbacks=None))] + fn report_success( + &self, + id: &str, + total_tokens: f64, + requests: f64, + usage_model: Option, + tracks_usage: Option, + callbacks: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let router = self.router.clone(); + let id = id.to_owned(); + let report = SuccessReport { + tracks_usage, + usage_model, + callbacks: outcome_callbacks(callbacks)?, + }; + Ok(RouterCall::new(async move { + router + .report_success(&id, total_tokens, requests, report) + .await + .map(|key| Finish::Counted { key }) + })) + } + + /// Count a success against `id`, configured or not; ends with `done` (`key`). `callbacks`: + /// see `report_failure`. + #[pyo3(signature = (id, callbacks=None))] + fn count_success( + &self, + id: &str, + callbacks: Option<&Bound<'_, PyAny>>, + ) -> PyResult { + let router = self.router.clone(); + let id = id.to_owned(); + let callbacks = outcome_callbacks(callbacks)?; + Ok(RouterCall::new(async move { + router + .count_success(&id, callbacks) + .await + .map(|key| Finish::Counted { key }) + })) + } + + /// The usage keys of this minute for every deployment of `model_group`: + /// `[(id, tpm_key, rpm_key)]`, or None when the group has no deployment. + fn usage_keys(&self, model_group: &str) -> Option> { + let minute = current_minute(SystemTime::now()); + let group = self.router.registry().group(model_group)?; + Some( + group + .iter() + .map(|deployment| { + let model = &deployment.params.model; + ( + deployment.id.clone(), + usage_key(&deployment.id, model, UsageMetric::Tpm, &minute), + usage_key(&deployment.id, model, UsageMetric::Rpm, &minute), + ) + }) + .collect(), + ) + } + + /// Every configured deployment, as `model_list` entries with their ids, in config order. + fn deployments<'py>(&self, py: Python<'py>) -> PyResult> { + let registry = self.router.registry(); + let records = registry + .deployments(&DeploymentFilter::default()) + .iter() + .map(|deployment| to_py(py, &host::deployment_record(deployment))) + .collect::>>()?; + PyList::new(py, records) + } + + /// The deployments from position `start` on, as `deployments()` lists them. + fn deployments_from<'py>(&self, py: Python<'py>, start: usize) -> PyResult> { + let registry = self.router.registry(); + let records = registry + .deployments_from(start) + .iter() + .map(|deployment| to_py(py, &host::deployment_record(deployment))) + .collect::>>()?; + PyList::new(py, records) + } + + /// The deployment with this id, or None. + fn deployment(&self, py: Python<'_>, id: &str) -> PyResult>> { + self.router + .registry() + .deployment(id) + .map(|deployment| to_py(py, &host::deployment_record(&deployment))) + .transpose() + } + + // -- registry lookups (router-core `lookup.rs`): rows the host reads its stored entries at -- + + /// `_core_get_all_deployments`, as rows. + #[pyo3(signature = (model_name, team_id=None, rename=None))] + fn lookup_all<'py>( + &self, + py: Python<'py>, + model_name: &str, + team_id: Option<&str>, + rename: Option<&str>, + ) -> PyResult> { + let registry = self.router.registry(); + rows_to_py(py, &Lookup::new(®istry).all(model_name, team_id, rename)) + } + + /// `_team_deployments_across_teams`: the rows, and how many teams own them. + fn lookup_across_teams<'py>( + &self, + py: Python<'py>, + model: &str, + ) -> PyResult<(Bound<'py, PyAny>, usize)> { + let registry = self.router.registry(); + let (rows, teams) = Lookup::new(®istry).across_teams(model); + Ok((rows_to_py(py, &rows)?, teams)) + } + + /// `_core_config_deployments`, as rows. + fn lookup_config<'py>(&self, py: Python<'py>) -> PyResult> { + let registry = self.router.registry(); + rows_to_py(py, &Lookup::new(®istry).config()) + } + + /// `_core_get_model_ids`: the ids, or the row the read raises at. + #[pyo3(signature = (model_name=None, exclude_team_models=false))] + fn lookup_model_ids<'py>( + &self, + py: Python<'py>, + model_name: Option<&str>, + exclude_team_models: bool, + ) -> PyResult<(Vec, Option>)> { + let registry = self.router.registry(); + match Lookup::new(®istry).model_ids(model_name, exclude_team_models) { + Ok(ids) => Ok((ids, None)), + Err(raw) => Ok((Vec::new(), Some(row_to_py(py, &raw)?))), + } + } + + /// `_core_get_model_list`, as rows. + #[pyo3(signature = (model_name, team_id, scope))] + fn lookup_model_list<'py>( + &self, + py: Python<'py>, + model_name: Option<&str>, + team_id: Option<&str>, + scope: &Bound<'_, PyAny>, + ) -> PyResult> { + let scope = list_scope(scope)?; + let registry = self.router.registry(); + rows_to_py( + py, + &Lookup::new(®istry).model_list(model_name, team_id, &scope), + ) + } + + /// `_core_get_model_names`, or the row the read raises at. + fn lookup_model_names<'py>( + &self, + py: Python<'py>, + team_id: Option<&str>, + scope: &Bound<'_, PyAny>, + ) -> PyResult<(Vec, Option>)> { + let scope = list_scope(scope)?; + let registry = self.router.registry(); + match Lookup::new(®istry).model_names(team_id, &scope) { + Ok(names) => Ok((names, None)), + Err(raw) => Ok((Vec::new(), Some(row_to_py(py, &raw)?))), + } + } + + /// `_core_get_model_access_groups`: `(group, names)` pairs in the order the groups appear, + /// or the row the read raises at. + fn lookup_access_groups<'py>( + &self, + py: Python<'py>, + model_name: Option<&str>, + access_group: Option<&str>, + team_id: Option<&str>, + scope: &Bound<'_, PyAny>, + ) -> PyResult<(AccessGroups, Option>)> { + let scope = list_scope(scope)?; + let registry = self.router.registry(); + match Lookup::new(®istry).access_groups(model_name, access_group, team_id, &scope) { + Ok(groups) => Ok((groups, None)), + Err(raw) => Ok((Vec::new(), Some(row_to_py(py, &raw)?))), + } + } + + /// Whether the whole list lists any row, and the name of its first row whose litellm model + /// is `model_id` (or ends in `/model_id` or `:model_id`); or the row the read raises at. + fn lookup_name_serving<'py>( + &self, + py: Python<'py>, + model_id: &str, + scope: &Bound<'_, PyAny>, + ) -> PyResult<(bool, Option, Option>)> { + let scope = list_scope(scope)?; + let registry = self.router.registry(); + match Lookup::new(®istry).name_serving(model_id, &scope) { + Ok((listed, name)) => Ok((listed, name, None)), + Err(raw) => Ok((false, None, Some(row_to_py(py, &raw)?))), + } + } + + /// Whether `model_names` (the group names) holds `name`. + fn has_model_name(&self, name: &str) -> bool { + self.router + .registry() + .host_indexes() + .model_names + .contains(&Value::String(name.to_owned())) + } + + /// `_core_team_model_has_alternatives`, or the row the read raises at. + fn lookup_team_model_has_alternatives<'py>( + &self, + py: Python<'py>, + id: &str, + ) -> PyResult<(bool, Option>)> { + let registry = self.router.registry(); + match Lookup::new(®istry).team_model_has_alternatives(id) { + Ok(found) => Ok((found, None)), + Err(raw) => Ok((false, Some(row_to_py(py, &raw)?))), + } + } + + /// The position of the group's first deployment the team may use, or the row the read + /// raises at. + fn lookup_group_usable_by_team<'py>( + &self, + py: Python<'py>, + model_group: &str, + team_id: Option<&str>, + ) -> PyResult<(Option, Option>)> { + let registry = self.router.registry(); + match Lookup::new(®istry).group_deployment_usable_by_team(model_group, team_id) { + Ok(position) => Ok((position, None)), + Err(raw) => Ok((None, Some(row_to_py(py, &raw)?))), + } + } + + /// The position of the team public name's first deployment, or the row the read raises at. + fn lookup_team_public_name<'py>( + &self, + py: Python<'py>, + model_id: &str, + team_id: &str, + ) -> PyResult<(Option, Option>)> { + let registry = self.router.registry(); + match Lookup::new(®istry).team_public_name_deployment(model_id, team_id) { + Ok(position) => Ok((position, None)), + Err(raw) => Ok((None, Some(row_to_py(py, &raw)?))), + } + } + + /// `_routable_deployments`, as rows. + fn lookup_routable<'py>( + &self, + py: Python<'py>, + model_group: &str, + team_id: Option<&str>, + ) -> PyResult> { + let registry = self.router.registry(); + rows_to_py(py, &Lookup::new(®istry).routable(model_group, team_id)) + } + + /// `_core_wildcard_route_deployments`: the team router's routes, else the global router's. + fn lookup_wildcard_route<'py>( + &self, + py: Python<'py>, + model: &str, + team_id: Option<&str>, + requests: Vec, + ) -> PyResult> { + let registry = self.router.registry(); + rows_to_py( + py, + &Lookup::new(®istry).wildcard_route(model, team_id, &requests), + ) + } + + /// `_core_get_candidate_model_ids_for_route` for the resolved name: the members' ids when it + /// names a callable routing group (`members`), else for a name that is no model group the + /// team's or the wildcard route's, else the group's; or the row the read raises at. + fn lookup_candidate_ids<'py>( + &self, + py: Python<'py>, + resolved: &str, + team_id: Option<&str>, + members: Option>, + requests: Vec, + ) -> PyResult<(Vec, Option>)> { + let registry = self.router.registry(); + let lookup = Lookup::new(®istry); + match lookup.candidate_ids(resolved, team_id, members.as_deref(), &requests) { + Ok(ids) => Ok((ids, None)), + Err(raw) => Ok((Vec::new(), Some(row_to_py(py, &raw)?))), + } + } + + /// Whether the id map holds `id` (`_has_model_id`). + fn has_model_id(&self, id: &str) -> bool { + self.router.registry().host_indexes().has_id(id) + } + + /// What the id map holds for `id`, as written (a position, or what a caller wrote), or None. + fn id_position<'py>(&self, py: Python<'py>, id: &str) -> PyResult>> { + let registry = self.router.registry(); + registry + .host_indexes() + .id_entry(id) + .map(|held| to_py(py, held).map(|value| value.into_bound(py))) + .transpose() + } + + /// The id map's entries, in order: `(id, what it holds)`. + fn id_positions<'py>(&self, py: Python<'py>) -> PyResult> { + let registry = self.router.registry(); + let pairs: Vec = registry + .host_indexes() + .id_map + .iter() + .map(|(id, held)| Value::Array(vec![Value::String(id.clone()), held.clone()])) + .collect(); + to_py(py, &Value::Array(pairs)).map(|value| value.into_bound(py)) + } + + /// What the name index holds for `name`, as written, or None when it does not index it. + fn named_positions<'py>( + &self, + py: Python<'py>, + name: &str, + ) -> PyResult>> { + let registry = self.router.registry(); + registry + .host_indexes() + .named(name) + .map(|held| to_py(py, held).map(|value| value.into_bound(py))) + .transpose() + } + + /// What the team index holds for `(team, public)`, as written, or None. + fn team_positions<'py>( + &self, + py: Python<'py>, + team: &str, + public: &str, + ) -> PyResult>> { + let registry = self.router.registry(); + registry + .host_indexes() + .team_named(team, public) + .map(|held| to_py(py, held).map(|value| value.into_bound(py))) + .transpose() + } + + /// `team_public_model_names_of`: the public names the team index holds for `team`. + fn team_public_names_of(&self, team: &str) -> Vec { + self.router + .registry() + .host_indexes() + .team_public_names_of(team) + } + + /// The positions of the deployments whose `litellm_params.model` is `model`. + fn litellm_model_positions(&self, model: &str) -> Vec { + self.router + .registry() + .deployments(&DeploymentFilter::default()) + .iter() + .enumerate() + .filter(|(_, deployment)| { + deployment.raw_params.get("model").and_then(Value::as_str) == Some(model) + }) + .map(|(position, _)| position) + .collect() + } + + /// `PatternMatchRouter.route` over the wildcard router (or `team`'s): the first pattern's + /// deployments, each with the model the wildcards matched, or None. + #[pyo3(signature = (team, request, filtered=None))] + fn pattern_route<'py>( + &self, + py: Python<'py>, + team: Option<&str>, + request: &str, + filtered: Option>, + ) -> PyResult>> { + let registry = self.router.registry(); + let indexes = registry.host_indexes(); + let router = match team { + None => Some(&indexes.patterns), + Some(team) => indexes + .team_patterns + .iter() + .find(|(held, _)| held == team) + .map(|(_, router)| router), + }; + router + .and_then(|router| router.route(request, filtered.as_deref())) + .map(|routed| to_py(py, &Value::Array(routed)).map(|value| value.into_bound(py))) + .transpose() + } + + /// `get_models_by_pattern`: the models the first matching (request, pattern) pair routes to. + fn models_by_pattern(&self, team: Option<&str>, requests: Vec) -> Option> { + let registry = self.router.registry(); + let indexes = registry.host_indexes(); + let router = match team { + None => Some(&indexes.patterns), + Some(team) => indexes + .team_patterns + .iter() + .find(|(held, _)| held == team) + .map(|(_, router)| router), + }; + router.and_then(|router| router.models_by_pattern(&requests)) + } + + /// The indexes the host hands out (`HostIndexes`): id map, name and team indexes, group and + /// deployment names, team public names, provider default ids and the pattern routers. + fn host_indexes<'py>(&self, py: Python<'py>) -> PyResult> { + let indexes = serde_json::to_value(self.router.registry().host_indexes()) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + to_py(py, &indexes).map(|value| value.into_bound(py)) + } + + /// `provider_default_deployment_ids` as the indexes hold it now: the ids of the wildcard + /// deployments the cooldown check skips. + fn provider_default_ids(&self) -> Vec { + self.router + .registry() + .host_indexes() + .provider_default_ids + .clone() + } + + /// The provider the host resolved when it built the deployment with this id (its + /// `provider` build step), or None when there is no such deployment or it never went + /// through a host build. + fn provider(&self, id: &str) -> Option { + self.router + .registry() + .deployment(id) + .and_then(|deployment| deployment.resolved_provider.clone()) + } + + /// What `name` resolves to: `{"model_group", "kind", "ids"}`, or None for an unknown name. + #[pyo3(signature = (name, specific_deployment=false))] + fn resolve<'py>( + &self, + py: Python<'py>, + name: &str, + specific_deployment: bool, + ) -> PyResult>> { + let registry = self.router.registry(); + let Some(resolution) = registry.resolve( + name, + ResolveScope { + specific_deployment, + }, + ) else { + return Ok(None); + }; + let dict = PyDict::new(py); + dict.set_item("model_group", resolution.group)?; + dict.set_item("kind", format!("{:?}", resolution.kind))?; + dict.set_item( + "ids", + PyList::new(py, resolution.deployments.iter().map(|d| d.id.clone()))?, + )?; + Ok(Some(dict)) + } + + /// The deployments a pick for `model` would resolve to for `request`, before any team, + /// cooldown or strategy filter (`_core_get_routable_deployments`); empty when it refuses. + fn routable<'py>( + &self, + py: Python<'py>, + model: &str, + request: &Bound<'py, PyAny>, + ) -> PyResult> { + let request = config::parse_request(document(request)?, Some(model)) + .map_err(config_error_to_pyerr)?; + let records = litellm_router_core::resolve::routable( + &self.router.registry(), + &self.router.settings().dynamic, + &request, + ) + .iter() + .map(|d| to_py(py, &host::deployment_record(d))) + .collect::>>()?; + PyList::new(py, records) + } + + /// `_get_routing_context` for a pick of `model`: `(strategy, scope, group)`, spelled as a + /// strategy select step spells them. A name that resolves to nothing reads its routing group + /// membership as is. + #[pyo3(signature = (model, routing_strategy=None))] + fn routing_context( + &self, + model: &str, + routing_strategy: Option<&str>, + ) -> (String, &'static str, Option) { + let registry = self.router.registry(); + let settings = self.router.settings(); + let (kind, group) = registry + .resolve(model, ResolveScope::default()) + .map_or((ResolveKind::Group, model.to_owned()), |resolution| { + (resolution.kind, resolution.group) + }); + let (strategy, scope) = routing_context( + ®istry, + &settings.routing_strategy, + routing_strategy, + &kind, + &group, + ); + let (scope, group) = scope.wire(); + (strategy.to_string(), scope, group.map(str::to_owned)) + } + + fn retire(&self) { + self.router.retire(); + } + + /// What may still take over a stream from `origin` that fails before content, and the + /// retry budget for a failure of `kind` (see `stream::policy`). + #[pyo3(signature = (request, origin, kind=None))] + fn stream_policy<'py>( + &self, + py: Python<'py>, + request: &Bound<'py, PyAny>, + origin: &Bound<'py, PyAny>, + kind: Option<&str>, + ) -> PyResult> { + stream::policy(py, &self.router, request, origin, kind) + } + + /// Retry a stream from `origin` that failed before content within its group, driven like + /// `start`: `done` when a retry returned a response, `failed` with the newest failure. + fn retry_stream( + &self, + op: &str, + request: &Bound<'_, PyAny>, + origin: &Bound<'_, PyAny>, + failure: &Bound<'_, PyDict>, + ) -> PyResult { + stream::recover( + &self.router, + stream::Recovery::Retry, + op, + request, + origin, + failure, + ) + } + + /// The fallback walk for a stream from `origin` that failed before content, driven like + /// `start`. + fn fallback_stream( + &self, + op: &str, + request: &Bound<'_, PyAny>, + origin: &Bound<'_, PyAny>, + failure: &Bound<'_, PyDict>, + ) -> PyResult { + stream::recover( + &self.router, + stream::Recovery::Fallback, + op, + request, + origin, + failure, + ) + } + + /// `get_num_retries_from_retry_policy` for a failure shaped like an attempt outcome. + #[pyo3(signature = (failure, model_group=None))] + fn retry_policy_retries( + &self, + failure: &Bound<'_, PyDict>, + model_group: Option<&str>, + ) -> PyResult> { + retries::policy_retries(&self.router, failure, model_group) + } + + /// `_core_get_allowed_fails_from_policy` for a failure shaped like an attempt outcome. + fn allowed_fails_for(&self, failure: &Bound<'_, PyDict>) -> PyResult> { + retries::allowed_fails(&self.router, failure) + } + + /// `should_retry_this_error` as a predicate, for a failure shaped like an attempt outcome. + fn should_retry( + &self, + failure: &Bound<'_, PyDict>, + counts: (usize, usize), + fallbacks: (bool, bool, bool), + ) -> PyResult { + retries::should_retry(failure, counts, fallbacks) + } +} + +/// One router call, driven a step at a time by Python. +#[pyclass(frozen)] +pub(crate) struct RouterCall { + driver: Mutex, +} + +impl RouterCall { + fn new( + future: impl std::future::Future> + Send + 'static, + ) -> Self { + Self { + driver: Mutex::new(Driver::new(future)), + } + } +} + +#[pymethods] +impl RouterCall { + /// Hand back the outcome of the last step (`None` on the first call) and return the next + /// step, or the call's end. + #[pyo3(signature = (outcome))] + fn next<'py>( + &self, + py: Python<'py>, + outcome: Option>, + ) -> PyResult> { + let mut driver = self.driver.lock().unwrap_or_else(PoisonError::into_inner); + let response = match driver.pending_call() { + None => match outcome { + None => None, + Some(_) => return Err(drive_error_to_pyerr(DriveError::UnexpectedResponse)), + }, + Some(call) => Some(host_call_response(call, outcome)?), + }; + let step = driver.resume(response).map_err(drive_error_to_pyerr)?; + let dict = step_dict(py, step)?; + dict.set_item("events", events_list(py, driver.take_events())?)?; + Ok(dict) + } +} + +/// `(group, names)` pairs, in the order the groups appear. +type AccessGroups = Vec<(String, Vec)>; + +/// Lookup rows as the host reads them. +fn rows_to_py<'py>(py: Python<'py>, rows: &[Row]) -> PyResult> { + let rows = Value::Array(rows.iter().map(Row::wire).collect()); + to_py(py, &rows).map(|value| value.into_bound(py)) +} + +fn row_to_py<'py>(py: Python<'py>, row: &Row) -> PyResult> { + to_py(py, &row.wire()).map(|value| value.into_bound(py)) +} + +/// `{"aliases": [{alias, target}], "groups": [{name, members}], "wildcard": [requests]}`. +fn list_scope(scope: &Bound<'_, PyAny>) -> PyResult { + serde_json::from_value(document(scope)?) + .map_err(|error| PyValueError::new_err(format!("bad lookup scope: {error}"))) +} + +/// Read Python's outcome for `call`. +fn host_call_response( + call: &HostCall, + outcome: Option>, +) -> PyResult { + let missing = || { + drive_error_to_pyerr(DriveError::MissingResponse { + expected: call.kind(), + }) + }; + // `{"raised": True, ...}`: the step's work raised, reported as an attempt's failure. + if let HostCall::Select { .. } | HostCall::PreRoute { .. } | HostCall::AllowFallback { .. } = + call + && let Some(failure) = outcome.as_ref().and_then(|o| o.cast::().ok()) + && failure.contains("raised")? + { + return match attempt_outcome(failure)? { + Err(error) => Ok(HostCallResponse::Raised(error)), + Ok(_) => Err(PyValueError::new_err("a `raised` outcome is a failure")), + }; + } + match call { + HostCall::Sleep { .. } => Ok(HostCallResponse::Slept), + HostCall::AllowFallback { .. } => Ok(HostCallResponse::AllowFallback( + outcome.ok_or_else(missing)?.extract()?, + )), + HostCall::RefuseFallbacks { .. } => Ok(HostCallResponse::RefuseFallbacks( + outcome.ok_or_else(missing)?.extract()?, + )), + HostCall::Select { + stage: Stage::PreCall, + .. + } => { + let facts = from_py_argument(&outcome.ok_or_else(missing)?)?; + serde_json::from_value(facts) + .map(HostCallResponse::PreCall) + .map_err(|error| PyValueError::new_err(format!("bad pre-call facts: {error}"))) + } + HostCall::Select { + stage: Stage::RoutingPlugins { .. } | Stage::ProviderRequest, + .. + } => { + // `{"models": [...] | None}`: a bare None would read as no outcome. + let outcome = outcome.ok_or_else(missing)?; + let models = outcome.cast::()?.get_item("models")?; + Ok(HostCallResponse::Models(match models { + Some(models) if !models.is_none() => Some(models.extract()?), + _ => None, + })) + } + HostCall::Select { + stage: Stage::CallbackFilters { .. }, + .. + } => { + // The remaining ids, alone or paired with the id a filter prefers, or a failure + // mapping as an attempt reports one. + let outcome = outcome.ok_or_else(missing)?; + if let Ok(failure) = outcome.cast::() { + return match attempt_outcome(failure)? { + Err(error) => Ok(HostCallResponse::Filtered(Err(error))), + Ok(_) => Err(PyValueError::new_err( + "a `callback_filters` outcome is the remaining ids or a failure", + )), + }; + } + if let Ok(answer) = outcome.cast::() { + return Ok(HostCallResponse::Filtered(Ok(answer.extract()?))); + } + Ok(HostCallResponse::Filtered(Ok((outcome.extract()?, None)))) + } + HostCall::Select { + stage: Stage::Strategy { records, .. }, + .. + } if !records.is_empty() => Ok(HostCallResponse::Positions( + outcome.ok_or_else(missing)?.extract()?, + )), + HostCall::Select { .. } => Ok(HostCallResponse::Select( + outcome.ok_or_else(missing)?.extract()?, + )), + HostCall::PreRoute { .. } => pre_route_outcome(outcome).map(HostCallResponse::PreRoute), + HostCall::Attempt { .. } => { + let outcome = outcome.ok_or_else(missing)?; + let outcome = outcome.cast::()?; + let blocked = match outcome.get_item("blocked")? { + Some(value) if !value.is_none() => { + let name: String = value.extract()?; + Some(name.parse::().map_err(|_| { + PyValueError::new_err(format!( + "`{name}` is not a way a response is blocked" + )) + })?) + } + Some(_) | None => None, + }; + let deferred = outcome + .get_item("deferred")? + .is_some_and(|flag| flag.is_truthy().unwrap_or(false)); + let tracks_usage = outcome + .get_item("tracks_usage")? + .map(|flag| flag.extract::>()) + .transpose()? + .flatten(); + attempt_outcome(outcome).map(|result| match result { + Ok(total_tokens) if blocked.is_some() || deferred || tracks_usage.is_some() => { + HostCallResponse::Served { + total_tokens, + blocked, + deferred, + tracks_usage, + } + } + result => HostCallResponse::Attempt(result), + }) + } + HostCall::Cache { op, .. } => cache_outcome(*op, outcome).map(HostCallResponse::Cache), + HostCall::Secrets { .. } => Ok(HostCallResponse::Secrets(from_py_argument( + &outcome.ok_or_else(missing)?, + )?)), + // `None` is a response here (no row, no entry), so a missing outcome reads as one. + HostCall::ModelInfo { .. } => Ok(HostCallResponse::ModelInfo(match outcome { + Some(outcome) if !outcome.is_none() => from_py_argument(&outcome)?, + _ => Value::Null, + })), + HostCall::Build { kind, .. } => { + let outcome = outcome.ok_or_else(missing)?; + let outcome = outcome.cast::()?; + let invalid = outcome + .get_item("invalid")? + .is_some_and(|flag| flag.is_truthy().unwrap_or(false)); + match kind { + StepKind::Provider => Ok(HostCallResponse::Provided(if invalid { + Provided::Invalid + } else { + Provided::Provider(match outcome.get_item("provider")? { + Some(provider) if !provider.is_none() => Some(provider.extract()?), + _ => None, + }) + })), + _ => { + let built = if invalid { + Built::Invalid + } else { + match outcome.get_item("entry")? { + Some(entry) if !entry.is_none() => { + Built::Entry(from_py_argument(&entry)?) + } + _ => Built::Skip, + } + }; + Ok(if *kind == StepKind::Validate { + HostCallResponse::Validated(built) + } else { + HostCallResponse::Added(built) + }) + } + } + } + } +} + +/// A pre-route response: `None`, a model group name, or a custom strategy's deployment document. +fn pre_route_outcome(outcome: Option>) -> PyResult> { + let Some(outcome) = outcome.filter(|outcome| !outcome.is_none()) else { + return Ok(None); + }; + if let Ok(model) = outcome.cast::() { + return Ok(Some(PreRouted::Model(model.to_str()?.to_owned()))); + } + // `(model, group)`: the model to resolve, served as the group a session binding stamped. + if let Ok(pair) = outcome.cast::() + && pair.len() == 2 + { + return Ok(Some(PreRouted::Bound { + model: pair.get_item(0)?.extract()?, + group: pair.get_item(1)?.extract()?, + })); + } + let spec: DeploymentSpec = serde_json::from_value(document(&outcome)?) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let deployment = Deployment::from_spec(spec) + .map_err(|error| PyValueError::new_err(Report::new(&error).to_string()))?; + Ok(Some(PreRouted::Deployment(Arc::new(deployment)))) +} + +/// `get`: the JSON value or None; `batch_get`: a list of JSON values or None, one per key; +/// `increment`: the new int or float; `set` and `delete`: None. +fn cache_outcome(op: CacheOp, outcome: Option>) -> PyResult { + let outcome = outcome.filter(|outcome| !outcome.is_none()); + let json = |value: Option| value.filter(|value| !value.is_null()); + match op { + CacheOp::Get => outcome + .map(|outcome| from_py_argument::(&outcome)) + .transpose() + .map(|value| CacheOutcome::Value(json(value))), + CacheOp::BatchGet => { + let outcome = outcome.ok_or_else(|| { + PyValueError::new_err("a `batch_get` cache outcome must be a list, not None") + })?; + if !outcome.is_instance_of::() { + return Err(PyValueError::new_err( + "a `batch_get` cache outcome must be a list with one entry per key", + )); + } + let values: Vec> = from_py_argument(&outcome)?; + Ok(CacheOutcome::Values(values.into_iter().map(json).collect())) + } + CacheOp::Increment => outcome + .filter(|outcome| !outcome.is_instance_of::()) + .and_then(|outcome| outcome.extract::().ok()) + .map(CacheOutcome::Number) + .ok_or_else(|| { + PyValueError::new_err("an `increment` cache outcome must be the new int or float") + }), + CacheOp::Set | CacheOp::Delete | CacheOp::IncrementUsage => match outcome { + None => Ok(CacheOutcome::Stored), + Some(_) => Err(PyValueError::new_err(format!( + "a `{op}` cache step has no outcome, so pass None" + ))), + }, + } +} + +/// `{"ok": True, "total_tokens": int}` (an attempt's may add `"blocked": str`, `"deferred": bool` +/// and `"tracks_usage": bool`) or `{"ok": False, "kind": str, "status": int | None, +/// "retry_after": float | None, "num_retries": int | None, "detail": str | None, +/// "outside_deployment": bool, "caller_scoped": bool, "status_absent": bool, +/// "uncounted_request": bool}`. +pub(crate) fn attempt_outcome(outcome: &Bound<'_, PyDict>) -> PyResult> { + fn item<'py, T: FromPyObjectOwned<'py>>( + outcome: &Bound<'py, PyDict>, + key: &str, + ) -> PyResult> { + match outcome.get_item(key)? { + Some(value) if !value.is_none() => Ok(Some(value.extract().map_err(Into::into)?)), + Some(_) | None => Ok(None), + } + } + let ok: bool = item(outcome, "ok")? + .ok_or_else(|| PyValueError::new_err("an attempt outcome needs `ok`"))?; + if ok { + return Ok(Ok(item(outcome, "total_tokens")?.unwrap_or(0))); + } + let kind: String = item(outcome, "kind")? + .ok_or_else(|| PyValueError::new_err("a failed attempt outcome needs `kind`"))?; + let kind: FailureKind = kind + .parse() + .map_err(|_| PyValueError::new_err(format!("`{kind}` is not a router failure kind")))?; + let retry_after = item::(outcome, "retry_after")? + .map(|seconds| { + Duration::try_from_secs_f64(seconds) + .map_err(|_| PyValueError::new_err("`retry_after` must be a non-negative number")) + }) + .transpose()?; + let mut failure = AttemptError::new(kind, item(outcome, "status")?); + failure.retry_after = retry_after; + failure.num_retries = item(outcome, "num_retries")?; + failure.failed_deployment_id = item(outcome, "failed_deployment_id")?; + failure.detail = item::(outcome, "detail")?.map(Into::into); + failure.recorded = item::(outcome, "recorded")?.unwrap_or(false); + failure.names_provider = item::(outcome, "llm_provider")?.unwrap_or(false); + failure.agentic_followup = item::(outcome, "agentic_followup")?.unwrap_or(false); + failure.outside_deployment = item::(outcome, "outside_deployment")?.unwrap_or(false); + failure.status_absent = item::(outcome, "status_absent")?.unwrap_or(false); + failure.uncounted_request = item::(outcome, "uncounted_request")?.unwrap_or(false); + failure.caller_scoped = item::(outcome, "caller_scoped")?.unwrap_or(false); + failure.classes = item::>(outcome, "classes")? + .unwrap_or_default() + .into_iter() + .map(Into::into) + .collect(); + Ok(Err(failure)) +} + +/// `[{"kind": "retries_exhausted", "model_group": str, "attempted": int, "max": int} +/// | {"kind": "fallback", "to": str, "order": int | None, "level": int}]`. +/// The `cause` of a `retry_logged` or `retry_policy_raised` event, with what names it. +fn cause_items(dict: &Bound<'_, PyDict>, cause: RetryCause) -> PyResult<()> { + match cause { + RetryCause::Attempt => dict.set_item("cause", "attempt"), + RetryCause::Level => dict.set_item("cause", "level"), + RetryCause::Other => dict.set_item("cause", "other"), + RetryCause::Pick(picked) => { + dict.set_item("cause", "pick")?; + pick_items(dict, &picked.0) + } + } +} + +/// A pick's error, as the `failed` step names it (`error_kind` and what names it), with the +/// model its message names (`failed_model`) when the error carries one. +fn pick_items(dict: &Bound<'_, PyDict>, error: &Error) -> PyResult<()> { + error_items(dict, error)?; + match error { + Error::NoHealthyDeployments { model, .. } | Error::UnknownModel { model } => { + dict.set_item("failed_model", model) + } + _ => Ok(()), + } +} + +/// `error_kind` and the fields that name the error. +fn error_items(dict: &Bound<'_, PyDict>, error: &Error) -> PyResult<()> { + dict.set_item("error", Report::new(error).to_string())?; + dict.set_item("error_kind", <&'static str>::from(ErrorKind::from(error)))?; + match error { + Error::EndpointNotSupported { model, message } => { + dict.set_item("error_model", model)?; + dict.set_item("error_detail", message) + } + Error::OnlyMarkers { model } + | Error::UnknownModel { model } + | Error::NoPassThroughDeployments { model } + | Error::PtuReserved { model } + | Error::MultipleTeams { model } + | Error::DefaultDeploymentRaises { model } + | Error::IndexOutOfRange { model } => dict.set_item("error_model", model), + Error::ContextWindowExceeded { model, detail } => { + dict.set_item("error_model", model)?; + dict.set_item("error_detail", detail) + } + Error::UnsupportedParams { model, params } => { + dict.set_item("error_model", model)?; + dict.set_item("error_params", PyList::new(dict.py(), params)?) + } + Error::RoutingPluginsLeftNothing { models } => { + dict.set_item("error_params", PyList::new(dict.py(), models)?) + } + Error::Reserved { + model, + until, + timezone, + } => { + dict.set_item("error_model", model)?; + dict.set_item("error_detail", format!("{until} {timezone}")) + } + Error::NoHealthyDeployments { + request_ids: Some(ids), + .. + } => dict.set_item("error_request_ids", PyList::new(dict.py(), ids)?), + Error::NonNumericWeight { metric } => dict.set_item("error_detail", metric), + Error::MissingRoutingGroup { group } => dict.set_item("error_model", group), + Error::ModelInfo { exception, detail } => { + dict.set_item("error_exception", *exception)?; + dict.set_item("error_detail", detail) + } + _ => Ok(()), + } +} + +fn events_list(py: Python<'_>, events: Vec) -> PyResult> { + let list = PyList::empty(py); + for event in events { + let dict = PyDict::new(py); + match event { + RouterEvent::RetriesExhausted { + model_group, + attempted, + max, + level, + picked, + } => { + dict.set_item("kind", "retries_exhausted")?; + dict.set_item("model_group", model_group)?; + dict.set_item("attempted", attempted)?; + dict.set_item("max", max)?; + dict.set_item("level", level)?; + dict.set_item("picked", picked)?; + } + RouterEvent::Retry { retry, .. } => { + dict.set_item("kind", "retry")?; + dict.set_item("retry", retry)?; + } + RouterEvent::FallingBack { level, model } => { + dict.set_item("kind", "falling_back")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + } + RouterEvent::OrderChainFailed { level, last_hop } => { + dict.set_item("kind", "order_chain_failed")?; + dict.set_item("level", level)?; + dict.set_item("last_hop", last_hop)?; + } + RouterEvent::FallbacksFailed { + level, + chain, + last_hop, + } => { + dict.set_item("kind", "fallbacks_failed")?; + dict.set_item("level", level)?; + dict.set_item("chain", chain)?; + dict.set_item("last_hop", last_hop)?; + } + RouterEvent::Fallback { + from, + to, + order, + depth, + level, + scope, + parent, + max_fallbacks, + unroutable, + } => { + dict.set_item("kind", "fallback")?; + dict.set_item("from", from)?; + dict.set_item("to", to)?; + dict.set_item("order", order)?; + dict.set_item("depth", depth)?; + dict.set_item("level", level)?; + dict.set_item("scope", scope)?; + dict.set_item("parent", parent)?; + dict.set_item("max_fallbacks", max_fallbacks)?; + if let Some(model) = unroutable { + dict.set_item("unroutable", to_py(py, &model)?)?; + } + } + RouterEvent::Blocked { + level, + model, + blocked, + } => { + dict.set_item("kind", "blocked")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + dict.set_item("blocked", <&'static str>::from(blocked))?; + } + RouterEvent::FallbackDone { + from, + to, + level, + scope, + parent, + succeeded, + } => { + dict.set_item("kind", "fallback_done")?; + dict.set_item("from", from)?; + dict.set_item("to", to)?; + dict.set_item("level", level)?; + dict.set_item("scope", scope)?; + dict.set_item("parent", parent)?; + dict.set_item("succeeded", succeeded)?; + } + RouterEvent::RetryLogged { + scope, + level, + model, + attempted_retries, + cause, + } => { + dict.set_item("kind", "retry_logged")?; + dict.set_item("scope", scope)?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + dict.set_item("attempted_retries", attempted_retries)?; + cause_items(&dict, cause)?; + } + RouterEvent::RetryPolicyRaised { + level, + model, + cause, + } => { + dict.set_item("kind", "retry_policy_raised")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + cause_items(&dict, cause)?; + } + RouterEvent::RetryCountRaised { + level, + model, + cause, + } => { + dict.set_item("kind", "retry_count_raised")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + cause_items(&dict, cause)?; + } + RouterEvent::RetriesOpened { + scope, + max_retries, + model_group_size, + } => { + dict.set_item("kind", "retries_opened")?; + dict.set_item("scope", scope)?; + dict.set_item("max_retries", max_retries)?; + dict.set_item("model_group_size", model_group_size)?; + } + RouterEvent::RetryBudget { scope, max_retries } => { + dict.set_item("kind", "retry_budget")?; + dict.set_item("scope", scope)?; + dict.set_item("max_retries", max_retries)?; + } + RouterEvent::PickFailed { + level, + model, + error, + } => { + dict.set_item("kind", "pick_failed")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + pick_items(&dict, &error.0)?; + } + RouterEvent::LevelFailed { + level, + model, + error, + handling, + } => { + dict.set_item("kind", "level_failed")?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + dict.set_item("handling", handling)?; + pick_items(&dict, &error.0)?; + } + RouterEvent::MockFailed { + mock, + level, + model, + num_retries, + } => { + dict.set_item("kind", "mock_failed")?; + dict.set_item("mock", <&'static str>::from(mock))?; + dict.set_item("level", level)?; + dict.set_item("model", model)?; + dict.set_item("num_retries", num_retries)?; + } + RouterEvent::CooldownSet { + deployment_id, + cooldown, + } => { + dict.set_item("kind", "cooldown_set")?; + dict.set_item("deployment_id", deployment_id)?; + dict.set_item("cooldown_time", cooldown.as_secs_f64())?; + } + RouterEvent::PickStarted { model } => { + dict.set_item("kind", "pick_started")?; + dict.set_item("model", model)?; + } + RouterEvent::Selected { + deployment_id, + pick, + .. + } => { + dict.set_item("kind", "selected")?; + dict.set_item("deployment_id", deployment_id)?; + dict.set_item("pick_seconds", pick.as_secs_f64())?; + } + _ => continue, + } + list.append(dict)?; + } + Ok(list) +} + +fn step_dict(py: Python<'_>, step: Step) -> PyResult> { + let dict = PyDict::new(py); + let kind = match &step { + Step::HostCall(call) => call.kind(), + Step::Done(_) => StepKind::Done, + Step::Failed(_) => StepKind::Failed, + }; + dict.set_item("kind", <&'static str>::from(kind))?; + match step { + Step::HostCall(call) => host_call_items(py, &dict, call)?, + Step::Done(finish) => done_items(&dict, finish)?, + Step::Failed(error) => { + let attempt = error.attempt(); + error_items(&dict, &error)?; + dict.set_item("attempt_failed", attempt.is_some())?; + dict.set_item("status", attempt.and_then(|failure| failure.status))?; + dict.set_item( + "failure_kind", + attempt.map(|failure| <&'static str>::from(failure.kind)), + )?; + } + } + Ok(dict) +} + +fn host_call_items(py: Python<'_>, dict: &Bound<'_, PyDict>, call: HostCall) -> PyResult<()> { + match call { + HostCall::Attempt { + deployment_id, + deployment, + model, + model_group, + routed, + target_order, + candidate_ids, + retry, + fallback_depth, + fallback_level, + timeout, + params, + scope, + max_retries, + model_group_size, + } => { + dict.set_item("routed", routed)?; + dict.set_item("scope", scope)?; + dict.set_item("max_retries", max_retries)?; + dict.set_item("model_group_size", model_group_size)?; + dict.set_item("deployment_id", deployment_id)?; + dict.set_item("deployment", to_py(py, &deployment)?)?; + dict.set_item("model", model)?; + dict.set_item("model_group", model_group)?; + dict.set_item("target_order", target_order)?; + dict.set_item("candidate_ids", PyList::new(py, candidate_ids)?)?; + dict.set_item("retry", retry)?; + dict.set_item("fallback_depth", fallback_depth)?; + dict.set_item("fallback_level", fallback_level)?; + dict.set_item("timeout", timeout)?; + dict.set_item("params", params.map(|p| to_py(py, &p)).transpose()?) + } + HostCall::Sleep { seconds } => dict.set_item("seconds", seconds), + HostCall::AllowFallback { target } => dict.set_item("target", target), + HostCall::RefuseFallbacks { level, targets } => { + dict.set_item("level", level)?; + let targets = targets.0.iter().cloned().map(Value::Object).collect(); + dict.set_item("targets", to_py(py, &Value::Array(targets))?) + } + HostCall::Select { + stage, + model_group, + name, + scope, + group, + pool, + candidates, + } => { + let (stage, strategy) = match stage { + Stage::Plugins => ("plugins", None), + Stage::Strategy { strategy, records } => { + if !records.is_empty() { + dict.set_item("records", to_py(py, &Value::Array(records))?)?; + } + ("strategy", Some(strategy)) + } + Stage::Tags => ("tags", None), + Stage::PreCall => ("pre_call", None), + Stage::TextOnly => ("text_only", None), + Stage::RouteIncapable { handler } => { + dict.set_item("handler", handler)?; + ("route_incapable", None) + } + Stage::MediaRefusing => ("media_refusing", None), + Stage::EmbeddingBatch => ("embedding_batch", None), + Stage::OverrideCheck(strategy) => ("override_check", Some(strategy)), + Stage::ProviderRequest => ("provider_request", None), + Stage::CallbackFilters { + excluded_ids, + skipped_ids, + target_order, + } => { + dict.set_item("excluded_ids", excluded_ids)?; + dict.set_item("skipped_ids", skipped_ids)?; + dict.set_item("target_order", target_order)?; + ("callback_filters", None) + } + Stage::RoutingPlugins { run } => { + dict.set_item("run", run)?; + ("routing_plugins", None) + } + }; + dict.set_item("pool", PyList::new(py, pool)?)?; + dict.set_item("stage", stage)?; + dict.set_item("strategy", strategy)?; + dict.set_item("model_group", model_group)?; + dict.set_item("model", name)?; + dict.set_item("scope", scope)?; + dict.set_item("group", group)?; + dict.set_item("candidates", PyList::new(py, candidates)?) + } + HostCall::PreRoute { + model, + fallback_depth, + retry, + } => { + dict.set_item("model", model)?; + dict.set_item("fallback_depth", fallback_depth)?; + dict.set_item("retry", retry) + } + HostCall::Cache { + op, + key, + keys, + value, + ttl, + local, + } => { + dict.set_item("op", <&'static str>::from(op))?; + dict.set_item("key", key)?; + dict.set_item("keys", keys.map(|keys| PyList::new(py, keys)).transpose()?)?; + dict.set_item("value", value.map(|value| to_py(py, &value)).transpose()?)?; + dict.set_item("ttl", ttl)?; + dict.set_item("local", local) + } + HostCall::Secrets { index, refs } => { + dict.set_item("index", index)?; + let refs = refs + .into_iter() + .map(|secret| { + let item = PyDict::new(py); + item.set_item("name", secret.name)?; + match secret.at { + SecretAt::Param(key) => { + item.set_item("key", key)?; + item.set_item("env_only", false)?; + } + SecretAt::DataSource { source, key } => { + item.set_item("source", source)?; + item.set_item("key", key)?; + item.set_item("env_only", true)?; + } + } + Ok(item) + }) + .collect::>>()?; + dict.set_item("refs", PyList::new(py, refs)?) + } + HostCall::Build { + kind: _, + path, + index, + entry, + declared_id, + duplicate_id, + provider, + } => { + dict.set_item("path", <&'static str>::from(path))?; + dict.set_item("index", index)?; + dict.set_item("entry", to_py(py, &entry)?)?; + dict.set_item("declared_id", declared_id)?; + dict.set_item("duplicate_id", duplicate_id)?; + dict.set_item("provider", provider) + } + HostCall::ModelInfo { lookup } => { + if let Value::Object(lookup) = lookup { + for (key, value) in lookup { + dict.set_item(key, to_py(py, &value)?)?; + } + } + Ok(()) + } + } +} + +fn done_items(dict: &Bound<'_, PyDict>, finish: Finish) -> PyResult<()> { + match finish { + Finish::Call(response) => { + dict.set_item("deployment_id", response.deployment_id)?; + dict.set_item("model_group", response.model_group)?; + dict.set_item("attempted_retries", response.attempted_retries)?; + dict.set_item("max_retries", response.max_retries)?; + dict.set_item("attempted_fallbacks", response.attempted_fallbacks)?; + dict.set_item("total_tokens", response.output.total_tokens) + } + Finish::Selected { + deployment_id, + model_group, + deployment, + } => { + dict.set_item("deployment_id", deployment_id)?; + dict.set_item("model_group", model_group)?; + dict.set_item("deployment", to_py(dict.py(), &deployment)?) + } + Finish::Reported { cooldown } => { + dict.set_item("cooled", cooldown.is_some())?; + dict.set_item("cooldown_time", cooldown.map(|c| c.as_secs_f64())) + } + Finish::Built(built) => { + let py = dict.py(); + let records = built + .into_iter() + .map(|deployment| { + let item = PyDict::new(py); + item.set_item("index", deployment.index)?; + item.set_item( + "id", + deployment.spec.model_info.get("id").and_then(Value::as_str), + )?; + item.set_item("provider", deployment.spec.provider)?; + Ok(item) + }) + .collect::>>()?; + dict.set_item("built", PyList::new(py, records)?) + } + Finish::Counted { key } => dict.set_item("key", key), + Finish::RouterModelInfo(merged) => { + let py = dict.py(); + match merged { + Some(merged) => { + dict.set_item("info", to_py(py, &Value::Object(merged.info))?)?; + dict.set_item("user_keys", merged.user_keys) + } + None => dict.set_item("info", py.None()), + } + } + Finish::DeploymentModelInfo(info) => { + let py = dict.py(); + match info { + Some(info) => dict.set_item("info", to_py(py, &Value::Object(info))?), + None => dict.set_item("info", py.None()), + } + } + Finish::Value(value) => dict.set_item("value", to_py(dict.py(), &value)?), + Finish::Healthy { + single, + deployment_ids, + deployments, + } => { + dict.set_item("single", single)?; + dict.set_item("deployment_ids", deployment_ids)?; + let records = deployments + .iter() + .map(|d| to_py(dict.py(), d)) + .collect::>>()?; + dict.set_item("deployments", PyList::new(dict.py(), records)?) + } + } +} + +#[cfg(test)] +mod tests { + use pyo3::{ffi::c_str, prelude::*, types::PyDict}; + use rstest::rstest; + + const CONFIG: &str = r#"{"model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}} + ]}"#; + + const PAIR: &str = r#"{"model_list": [ + {"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}}, + {"model_name": "primary", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "a2"}} + ]}"#; + + #[rstest] + fn python_drives_calls_and_selections_through_the_handle() { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + locals.set_item("native", crate::native_module(py)).unwrap(); + locals.set_item("config", CONFIG).unwrap(); + locals.set_item("pair", PAIR).unwrap(); + py.run( + c_str!( + r#" +import time +store = {} +asked = [] + +def answer_cache(step): + asked.append(step) + op, key = step["op"], step["key"] + if op == "get": + return store.get(key) + if op == "batch_get": + return [store.get(k) for k in step["keys"]] + if op == "set": + store[key] = step["value"] + return None + if op == "increment": + store[key] = store.get(key, 0) + step["value"] + return store[key] + if op == "delete": + store.pop(key, None) + return None + raise AssertionError(op) + +def drive(call, outcome): + step = call.next(outcome) + while step["kind"] == "cache": + step = call.next(answer_cache(step)) + return step + +handle = native.RouterCoreHandle(config, {"num_retries": 1}) +picked = drive(handle.select("primary", {}), None) +assert (picked["kind"], picked["deployment_id"], picked["model_group"]) == ("done", "a1", "primary"), picked +assert [step["op"] for step in asked] == ["batch_get"] +assert list(asked[0]) == ["kind", "op", "key", "keys", "value", "ttl", "local", "events"], list(asked[0]) +call = handle.start("acompletion", '{"model": "primary"}') +version = handle.registry_version() +assert handle.apply({"delete": ["gone"]}) == version + 1 +steps = [drive(call, None)] +steps.append(drive(call, {"ok": False, "kind": "RateLimitError", "status": 429})) +steps.append(drive(call, None)) +steps.append(drive(call, {"ok": True, "total_tokens": 5})) +kinds = [step["kind"] for step in steps] + +cooled = native.RouterCoreHandle(pair, {"num_retries": 0}) +call = cooled.start("acompletion", {"model": "primary"}) +attempt = drive(call, None) +tried = attempt["deployment_id"] +assert attempt["deployment"]["model_info"]["id"] == tried, attempt +assert attempt["deployment"]["litellm_params"]["model"].startswith("openai/"), attempt +assert [d["model_info"]["id"] for d in cooled.deployments()] == ["a1", "a2"] +assert cooled.deployment("a2")["model_name"] == "primary" +assert cooled.resolve("primary")["ids"] == ["a1", "a2"] +assert cooled.resolve("nope") is None +assert drive(call, {"ok": False, "kind": "RateLimitError", "status": 429})["kind"] == "failed" +# The core records the failure itself: the 429 cools the deployment down in the host's store. +assert f"deployment:{tried}:cooldown" in store, store +other = drive(cooled.select("primary", {}), None) +assert other["kind"] == "done" and other["deployment_id"] != tried, other + +call = handle.select("primary", {}) +step = call.next(None) +assert step["op"] == "batch_get" +for bad in ([], [None, None], "nope", None): + try: + call.next(bad) + except ValueError: + pass + else: + raise AssertionError(f"{bad!r} should not answer a batch_get of one key") +assert drive(call, [None])["kind"] == "done" +"# + ), + Some(&locals), + Some(&locals), + ) + .unwrap(); + let kinds: Vec = locals + .get_item("kinds") + .unwrap() + .unwrap() + .extract() + .unwrap(); + assert_eq!(kinds, ["attempt", "sleep", "attempt", "done"]); + }); + } + + #[rstest] + #[case::unknown_key(r#"{"routing_strategy_args": {}}"#, true)] + #[case::bad_json("{", false)] + #[case::not_an_object("[]", false)] + fn construction_failures_raise_the_class_python_reads( + #[case] config: &str, + #[case] declined: bool, + ) { + Python::initialize(); + Python::attach(|py| { + let error = crate::native_module(py) + .getattr("RouterCoreHandle") + .and_then(|handle| handle.call1((config, "{}"))) + .unwrap_err(); + assert_eq!( + error.is_instance_of::(py), + declined, + "{error}" + ); + assert_eq!( + error.is_instance_of::(py), + !declined, + "{error}" + ); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/router/retries.rs b/litellm-rust/crates/python-bridge/src/router/retries.rs new file mode 100644 index 00000000000..949aae96b34 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/retries.rs @@ -0,0 +1,55 @@ +//! The retry decisions `RouterCoreHandle` exposes as members: `get_num_retries_from_retry_policy` +//! and `should_retry_this_error`, decided by router-core for a failure Python describes the way it +//! describes a failed attempt. + +use litellm_router_core::{ + AttemptError, Router, + retry::{RetryContext, resolve_retry_policy, should_retry as core_should_retry}, +}; +use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; + +fn failure(outcome: &Bound<'_, PyDict>) -> PyResult { + match super::attempt_outcome(outcome)? { + Err(failure) => Ok(failure), + Ok(_) => Err(PyValueError::new_err("a failure needs `ok: False`")), + } +} + +pub(crate) fn policy_retries( + router: &Router, + outcome: &Bound<'_, PyDict>, + model_group: Option<&str>, +) -> PyResult> { + let failure = failure(outcome)?; + let settings = router.settings(); + let policy = match model_group { + Some(group) => resolve_retry_policy( + settings.retry_policy.as_ref(), + group, + Some(&settings.model_group_retry_policy), + ), + None => settings.retry_policy.as_ref(), + }; + Ok(policy.and_then(|policy| policy.retries_for(&failure))) +} + +pub(crate) fn allowed_fails(router: &Router, outcome: &Bound<'_, PyDict>) -> PyResult> { + Ok(router.settings().allowed_fails_for(&failure(outcome)?)) +} + +pub(crate) fn should_retry( + outcome: &Bound<'_, PyDict>, + (healthy, all): (usize, usize), + (context_window, content_policy, regular): (bool, bool, bool), +) -> PyResult { + Ok(core_should_retry( + &failure(outcome)?, + RetryContext { + healthy, + all, + has_context_window_fallbacks: context_window, + has_content_policy_fallbacks: content_policy, + has_regular_fallbacks: regular, + }, + )) +} diff --git a/litellm-rust/crates/python-bridge/src/router/stream.rs b/litellm-rust/crates/python-bridge/src/router/stream.rs new file mode 100644 index 00000000000..f85d73e0778 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/router/stream.rs @@ -0,0 +1,94 @@ +//! The stream-recovery entry points of `RouterCoreHandle`: what may still take over a stream that +//! failed before content, the same-group stream retry, and the fallback walk re-entered from a +//! stream. The host's stream wrappers call them; the calls are driven like `start`. + +use litellm_router_core::{ + FailureKind, Op, Router, + call::{StreamOrigin, StreamPolicy}, +}; +use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; + +use super::{RouterCall, attempt_outcome, config, config_error_to_pyerr, document, host::Finish}; + +/// A stream recovery a host wrapper asks for. +#[derive(Clone, Copy)] +pub(super) enum Recovery { + /// `_astream_retry_same_group`. + Retry, + /// The fallback walk re-entered from the stream wrapper. + Fallback, +} + +fn origin(value: &Bound<'_, PyAny>) -> PyResult { + serde_json::from_value(document(value)?) + .map_err(|error| PyValueError::new_err(format!("a stream origin: {error}"))) +} + +fn kind(kind: Option<&str>) -> PyResult> { + kind.map(|kind| { + kind.parse() + .map_err(|_| PyValueError::new_err(format!("`{kind}` is not a router failure kind"))) + }) + .transpose() +} + +/// `{"can_retry", "can_fall_back", "refusal_fallback", "retry_budget": int | None, +/// "policy_applies": bool}`. +pub(super) fn policy<'py>( + py: Python<'py>, + router: &Router, + request: &Bound<'py, PyAny>, + stream_origin: &Bound<'py, PyAny>, + failure_kind: Option<&str>, +) -> PyResult> { + let request = config::parse_request(document(request)?, None).map_err(config_error_to_pyerr)?; + let StreamPolicy { + can_retry, + can_fall_back, + refusal_fallback, + retry_budget, + } = router.stream_policy(&request, &origin(stream_origin)?, kind(failure_kind)?); + let dict = PyDict::new(py); + dict.set_item("can_retry", can_retry)?; + dict.set_item("can_fall_back", can_fall_back)?; + dict.set_item("refusal_fallback", refusal_fallback)?; + dict.set_item("retry_budget", retry_budget.map(|(budget, _)| budget))?; + dict.set_item( + "policy_applies", + retry_budget.is_some_and(|(_, applies)| applies), + )?; + Ok(dict) +} + +/// A recovery call for a stream from `stream_origin` that failed with `failure`. +pub(super) fn recover( + router: &Router, + recovery: Recovery, + op: &str, + request: &Bound<'_, PyAny>, + stream_origin: &Bound<'_, PyAny>, + failure: &Bound<'_, PyDict>, +) -> PyResult { + let op = Op::new(op); + let request = config::parse_request(document(request)?, None).map_err(config_error_to_pyerr)?; + let stream_origin = origin(stream_origin)?; + let Err(failure) = attempt_outcome(failure)? else { + return Err(PyValueError::new_err("a stream failure needs `ok: False`")); + }; + let router = router.clone(); + Ok(RouterCall::new(async move { + let response = match recovery { + Recovery::Retry => { + router + .retry_stream(op, request, stream_origin, failure) + .await + } + Recovery::Fallback => { + router + .fallback_stream(op, request, stream_origin, failure) + .await + } + }; + response.map(Finish::Call) + })) +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/agentic.rs b/litellm-rust/crates/python-bridge/src/routes/messages/agentic.rs new file mode 100644 index 00000000000..f6652d82050 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages/agentic.rs @@ -0,0 +1,172 @@ +use std::sync::{Arc, Mutex, PoisonError}; + +use litellm_host::{ + hooks::CallHooks, + interceptors::{RequestContext, WireRequest}, + lifecycle::Timing, +}; +use litellm_host_python::{HookStep, PythonOwned, PythonRuntime, json_to_py, to_py}; +use pyo3::{ + gc::{PyTraverseError, PyVisit}, + prelude::*, + types::PyDict, +}; + +use super::host::ROUTE_HOST_MODULE; + +/// What the provider was sent, which Python's agentic hooks read back as the handler's own +/// model, provider and request params. +struct Sent { + request: Py, + hooked: bool, + body: Py, + model: String, + provider: String, + requested: Py, +} + +/// Python's Messages handler hands the provider's answer to the callbacks overriding +/// `async_should_run_agentic_loop` before `@client` sees it, and wraps a stream so they run +/// once it ends. This hook asks the route host to do both at the same points. +pub(super) type HookedRequest = Arc>>>; + +pub(super) struct AgenticLoop { + asynchronous: bool, + request: Option>, + hooked: HookedRequest, + decided: bool, + sent: Option, +} + +impl AgenticLoop { + pub(super) fn new(asynchronous: bool, hooked: HookedRequest) -> Self { + Self { + asynchronous, + request: None, + hooked, + decided: false, + sent: None, + } + } + + fn hooked_request(&self, py: Python<'_>) -> Option> { + self.hooked + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .map(|request| request.clone_ref(py)) + } + + fn host_call<'py>( + &self, + py: Python<'py>, + name: &str, + subject: &Bound<'py, PyAny>, + sent: &Sent, + ) -> PyResult> { + py.import(ROUTE_HOST_MODULE)?.getattr(name)?.call1(( + subject, + sent.request.bind(py), + sent.body.bind(py), + &sent.model, + &sent.provider, + sent.requested.bind(py), + sent.hooked, + )) + } + + fn resume_response( + &mut self, + _: Python<'_>, + result: PyResult>, + ) -> PyResult>> { + result.map(HookStep::Ready) + } +} + +impl CallHooks for AgenticLoop { + fn arguments_prepared(&mut self, py: Python<'_>, arguments: &Py) -> PyResult<()> { + if self.asynchronous { + self.request = Some(arguments.clone_ref(py)); + } + Ok(()) + } + + fn before_provider_request( + &mut self, + py: Python<'_>, + wire: Box, + context: &RequestContext, + ) -> PyResult>> { + if self.decided { + return Ok(HookStep::Ready(wire)); + } + let Some(caller) = &self.request else { + return Ok(HookStep::Ready(wire)); + }; + self.decided = true; + let hooked = self.hooked_request(py); + let hooked_flag = hooked.is_some(); + let request = hooked.unwrap_or_else(|| caller.clone_ref(py)); + let module = py.import(ROUTE_HOST_MODULE)?; + if module + .getattr("runs_agentic_hooks")? + .call1((request.bind(py),))? + .extract::()? + { + module + .getattr("record_agentic_loop")? + .call1((request.bind(py), &context.custom_llm_provider))?; + self.sent = Some(Sent { + request, + hooked: hooked_flag, + body: to_py(py, &wire.body)?, + model: context.model.clone(), + provider: context.custom_llm_provider.clone(), + requested: json_to_py(py, &context.optional_params)?.unbind(), + }); + } + Ok(HookStep::Ready(wire)) + } + + fn transform_response( + &mut self, + py: Python<'_>, + response: Py, + _: Timing, + ) -> PyResult>> { + let Some(sent) = &self.sent else { + return Ok(HookStep::Ready(response)); + }; + let awaitable = self.host_call(py, "agentic_response", response.bind(py), sent)?; + Ok(HookStep::Await(awaitable.unbind(), Self::resume_response)) + } + + fn on_stream_open(&mut self, py: Python<'_>, head: &Py) -> PyResult<()> { + if let Some(sent) = &self.sent { + self.host_call(py, "agentic_stream", head.bind(py), sent)?; + } + Ok(()) + } +} + +impl PythonOwned for AgenticLoop { + fn close(&mut self, _: Python<'_>) { + self.request = None; + self.sent = None; + self.hooked + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.request)?; + if let Some(sent) = &self.sent { + visit.call(&sent.request)?; + visit.call(&sent.body)?; + visit.call(&sent.requested)?; + } + Ok(()) + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/answer.rs b/litellm-rust/crates/python-bridge/src/routes/messages/answer.rs new file mode 100644 index 00000000000..80cf2002287 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages/answer.rs @@ -0,0 +1,29 @@ +use futures_util::future::BoxFuture; +use litellm_host::machine::HostServices; +use litellm_inference_messages::{Error, HostAnswer}; + +use super::host::{MessagesHostCall, MessagesHosted}; + +pub(super) struct PythonAnswer(HostServices); + +impl PythonAnswer { + pub(super) fn new(services: HostServices) -> Self { + Self(services) + } +} + +impl HostAnswer for PythonAnswer { + fn answer(&self, stream: bool) -> BoxFuture<'_, Result<(), Error>> { + Box::pin( + self.0 + .call(move |reply| MessagesHostCall::Answer { stream, reply }), + ) + } + + fn intercept(&self, stream: bool) -> BoxFuture<'_, Result> { + Box::pin( + self.0 + .call(move |reply| MessagesHostCall::Intercept { stream, reply }), + ) + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index af614f17360..67a96cac32d 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -1,34 +1,122 @@ -use crate::cache::{CacheCall, Cached, PythonCache, Selection}; +use crate::cache::{CacheCall, PythonCache, Selection}; use litellm_host_python::{PythonHostCalls, PythonOwned}; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; + use bytes::Bytes; -use litellm_host_python::{InvokeError, PythonBinding, from_py, to_py}; +use litellm_host::protocol::Reply; +use litellm_host_python::{ + InvokeError, PythonBinding, effective_py_args, from_py, json_from_py, to_py, +}; use litellm_inference_messages::{ - Error, MessagesCall, MessagesSettings, MessagesShaping, litellm_params, messages_body, + DecodingError, Error, MessagesCall, MessagesSettings, MessagesShaping, UnencodableHeader, + litellm_params, messages_body, route::{Messages, MessagesStreamHead}, }; use litellm_llms::base_llm::messages::context::MessagesModelCapabilities; use litellm_llms_types::headers::ProviderSpecificHeaders; use litellm_router_types::LitellmParams; use pyo3::{ - exceptions::PyException, + exceptions::{PyException, PyUnicodeEncodeError}, gc::{PyTraverseError, PyVisit}, prelude::*, - types::{PyBytes, PyDict}, + sync::PyOnceLock, + types::{PyBytes, PyDict, PyList, PyMapping, PyString, PyTuple}, }; use serde_json::{Map, Value}; +use super::agentic::HookedRequest; +use super::safeguards::{SafeguardsCall, SafeguardsHost}; +use super::transport::{HostedTransport, TransportCall, python_failure}; use crate::{ errors::{RequestMarker, route_failure}, marshal::{ - ArgumentError, optional_timeout, project_optional_fields, public_response, + ArgumentError, optional_timeout, project_optional_fields, public_json_response, python_timeout_seconds, }, - python_settings::missing_module, + python_settings::PythonSettings, routes::RequestView, }; -const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages"; +type MergedHeaders = (Option>, Option>); + +fn ascii_headers<'py>( + headers: &Bound<'py, PyAny>, + unencodable: &mut Option>, +) -> PyResult> { + let Ok(headers) = headers.cast::() else { + return Ok(headers.clone()); + }; + let py = headers.py(); + let kept = PyDict::new(py); + for (name, value) in headers.iter() { + if value.is_instance_of::() + && let Err(error) = value.call_method1("encode", ("ascii",)) + { + if !error.is_instance_of::(py) { + return Err(error); + } + if unencodable.is_none() { + let sent = value.call_method1("encode", ("utf-8", "surrogatepass"))?; + *unencodable = Some(sent.cast::()?.as_bytes().to_vec()); + } + continue; + } + kept.set_item(name, value)?; + } + Ok(kept.into_any()) +} + +const PROVIDER_SPECIFIC_HEADERS_MODULE: &str = + "litellm.litellm_core_utils.get_provider_specific_headers"; + +fn ascii_scoped_headers<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let ascii_entry = |entry: Bound<'py, PyAny>| -> PyResult> { + let Ok(entry) = entry.cast::() else { + return Ok(entry); + }; + let Some(headers) = entry.get_item("extra_headers")? else { + return Ok(entry.clone().into_any()); + }; + let kept = entry.copy()?; + kept.set_item("extra_headers", ascii_headers(&headers, &mut None)?)?; + Ok(kept.into_any()) + }; + if let Ok(entries) = value.cast::() { + return Ok(PyList::new( + value.py(), + entries + .iter() + .map(ascii_entry) + .collect::>>()?, + )? + .into_any()); + } + if let Ok(entries) = value.cast::() { + return Ok(PyTuple::new( + value.py(), + entries + .iter() + .map(ascii_entry) + .collect::>>()?, + )? + .into_any()); + } + ascii_entry(value.clone()) +} + +fn json_messages(messages: &Bound<'_, PyAny>) -> PyResult { + if messages.is_instance_of::() || messages.is_instance_of::() { + return json_from_py(messages); + } + json_from_py(&messages.py().get_type::().call1((messages,))?) +} + +pub(super) const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.messages"; +const MODEL_CAPABILITIES_MODULE: &str = "litellm.rust_bridge.host.model_capabilities"; fn merge_headers( forwarded: Option>, @@ -40,26 +128,39 @@ fn merge_headers( (!merged.is_empty()).then_some(merged) } -/// Reads `litellm.`, the module global a param spec names, the way -/// `VertexBase.get_vertex_ai_project` reads `litellm.vertex_project`. +const FEDERATING_PROVIDER: &str = "anthropic"; + +fn workload_identity_fields(py: Python<'_>) -> PyResult<&'static [String]> { + static FIELDS: PyOnceLock> = PyOnceLock::new(); + FIELDS + .get_or_try_init(py, || { + py.import(ROUTE_HOST_MODULE)? + .getattr("WORKLOAD_IDENTITY_FIELDS")? + .extract() + }) + .map(Vec::as_slice) +} + fn module_global<'py>(py: Python<'py>, name: &str) -> PyResult>> { - let module = match py.import("litellm") { - Ok(module) => module, - Err(error) if missing_module(py, &error, "litellm")? => return Ok(None), - Err(error) => return Err(error), - }; - let value = module.getattr(name)?; - Ok((!value.is_none()).then_some(value)) + PythonSettings::ProviderDefaults + .read_or_unset(py)? + .map_or(Ok(None), |defaults| defaults.attribute(name)) } /// The caller's litellm params, read from the kwargs by the names the typed params declare, /// so a key the configs do not read is never converted; a spec whose spellings the call all /// leaves out falls back to the module global it names, which stays the host's concern. -fn project_litellm_params<'py>( +fn project_litellm_params<'a, 'py>( + names: impl IntoIterator, argument: impl Fn(&str) -> PyResult>>, global: impl Fn(&str) -> PyResult>>, ) -> PyResult> { - let fields = project_optional_fields(LitellmParams::fields(), &argument)?; + let fields = project_optional_fields( + LitellmParams::fields() + .map(|name| -> &'a str { name }) + .chain(names), + &argument, + )?; let globals = LitellmParams::specs() .filter(|spec| !spec.wire.iter().any(|name| fields.contains_key(*name))) .filter_map(|spec| spec.module_global); @@ -67,36 +168,279 @@ fn project_litellm_params<'py>( Ok(litellm_params(fields.into_iter().chain(folded).collect())) } +pub(super) struct MessagesHosted; + +impl litellm_host::protocol::Protocol for MessagesHosted { + type Request = (MessagesCall, Selection, bool); + type Response = ::Response; + type Error = Error; + type HostCall = MessagesHostCall; + type Chunk = Bytes; + type StreamHead = MessagesStreamHead; +} + +pub(super) enum MessagesHostCall { + Answer { stream: bool, reply: Reply<()> }, + Intercept { stream: bool, reply: Reply }, + Cache(CacheCall), + PreRequest(Reply>), + Safeguards(SafeguardsCall), + Transport(TransportCall), +} + +impl From for MessagesHostCall { + fn from(call: CacheCall) -> Self { + Self::Cache(call) + } +} + +impl From for MessagesHostCall { + fn from(call: TransportCall) -> Self { + Self::Transport(call) + } +} + +enum Answered { + Stream, + Whole { verbatim: bool }, +} + +#[derive(serde::Deserialize)] +struct HostSettings { + #[serde(flatten)] + settings: MessagesSettings, + model: String, + stream_cost_model: Option, + extra_headers: Option>, +} + /// The Python side of the Messages route: projects the prepared arguments and builds the /// public response, chunks and exceptions. pub(super) struct MessagesPythonHost { view: RequestView, cache: PythonCache, + transport: HostedTransport, + safeguards: SafeguardsHost, + classifies: Arc, + awaiting_safeguards: bool, + stream_cost_model: Option, + asynchronous: bool, + pre_request: Option>>, + hooked: bool, + hooked_request: HookedRequest, + requested_stream: bool, + arguments: Option>, + interceptor: Option>, + pending_answer: Option>, + pending_intercept: Option>, + pending_short_circuit: Option<(bool, Reply)>, + answer: Option>, + answered: Option, } impl MessagesPythonHost { - pub(super) fn new(view: RequestView, asynchronous: bool) -> Self { + pub(super) fn new( + view: RequestView, + asynchronous: bool, + hooked_request: HookedRequest, + classifies: Arc, + ) -> Self { Self { view, cache: PythonCache::new(asynchronous), + transport: HostedTransport::default(), + safeguards: SafeguardsHost::default(), + classifies, + awaiting_safeguards: false, + stream_cost_model: None, + asynchronous, + pre_request: None, + hooked: false, + hooked_request, + requested_stream: false, + arguments: None, + interceptor: None, + pending_answer: None, + pending_intercept: None, + pending_short_circuit: None, + answer: None, + answered: None, } } + fn interceptor(&self, py: Python<'_>) -> PyResult>> { + if !self.asynchronous { + return Ok(None); + } + let interceptor = py + .import(ROUTE_HOST_MODULE)? + .getattr("interceptor")? + .call1((self.view.request(py)?,))?; + Ok((!interceptor.is_none()).then(|| interceptor.unbind())) + } + + fn may_intercept(&self, py: Python<'_>) -> PyResult { + if !self.asynchronous { + return Ok(false); + } + let module = py.import(ROUTE_HOST_MODULE)?; + Ok(module.getattr("has_interceptors")?.call0()?.extract()? + || module.getattr("short_circuits")?.call0()?.extract()?) + } + + fn begin_intercept( + &mut self, + py: Python<'_>, + stream: bool, + reply: Reply, + ) -> PyResult>> { + let module = py.import(ROUTE_HOST_MODULE)?; + if !module + .getattr("short_circuits")? + .call0()? + .extract::()? + { + return self.begin_interceptor(py, stream, reply); + } + let pending = module.getattr("short_circuit")?.call1(( + self.view.request(py)?, + stream, + self.hooked, + ))?; + self.pending_short_circuit = Some((stream, reply)); + Ok(Some(pending.unbind())) + } + + fn resume_short_circuit( + &mut self, + py: Python<'_>, + stream: bool, + reply: Reply, + answer: Py, + ) -> PyResult>> { + if answer.is_none(py) { + return self.begin_interceptor(py, stream, reply); + } + self.answer = Some(answer); + self.answered = Some(match stream { + true => Answered::Stream, + false => Answered::Whole { verbatim: false }, + }); + reply.send(true); + Ok(None) + } + + fn begin_interceptor( + &mut self, + py: Python<'_>, + stream: bool, + reply: Reply, + ) -> PyResult>> { + self.interceptor = self.interceptor(py)?; + if self.interceptor.is_none() { + reply.send(false); + return Ok(None); + } + let pending = self.start_answer(py, stream)?; + self.pending_intercept = Some(reply); + Ok(Some(pending)) + } + + fn begin_answer( + &mut self, + py: Python<'_>, + stream: bool, + reply: Reply<()>, + ) -> PyResult>> { + let pending = self.start_answer(py, stream)?; + self.pending_answer = Some(reply); + Ok(Some(pending)) + } + + fn start_answer(&mut self, py: Python<'_>, stream: bool) -> PyResult> { + let module = py.import(ROUTE_HOST_MODULE)?; + let request = self.view.request(py)?; + let pending = match &self.interceptor { + Some(interceptor) => { + module + .getattr("answer")? + .call1((interceptor, request, stream, self.hooked))? + } + None => module + .getattr("gateway")? + .call1((request, self.asynchronous, self.hooked))?, + }; + if pending.is_none() { + return Err(litellm_host_python::Error::MissingState.into()); + } + self.answered = Some(match (stream, &self.interceptor) { + (true, _) => Answered::Stream, + (false, Some(_)) => Answered::Whole { verbatim: false }, + (false, None) => Answered::Whole { verbatim: true }, + }); + Ok(pending.unbind()) + } + + fn host_settings( + &self, + py: Python<'_>, + arguments: &Bound<'_, PyDict>, + ) -> PyResult { + from_py(&py.import(ROUTE_HOST_MODULE)?.getattr("settings")?.call1(( + arguments, + self.view.request(py)?, + self.asynchronous, + ))?) + } + + fn body_params( + &self, + py: Python<'_>, + arguments: &Bound<'_, PyDict>, + ) -> PyResult> { + let request = effective_py_args(self.view.request(py)?, arguments)?; + let params = py + .import(ROUTE_HOST_MODULE)? + .getattr("body_params")? + .call1((request, self.asynchronous, self.hooked))? + .cast_into::()?; + params + .items()? + .iter() + .map(|item| { + let (name, value): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item.extract()?; + let name: String = name.extract()?; + let value = match name.as_str() { + "messages" => json_messages(&value)?, + _ => json_from_py(&value)?, + }; + Ok((name, value)) + }) + .collect() + } + fn projection( &self, py: Python<'_>, arguments: &Bound<'_, PyDict>, + settings: HostSettings, ) -> PyResult> { let argument = |name: &str| self.view.argument(py, arguments, name); let string = |name: &str| -> PyResult> { argument(name)?.map(|value| value.extract()).transpose() }; - let model = string("model")?.ok_or(ArgumentError::Required("model"))?; + let model = settings.model; let messages = argument("messages")?.ok_or(ArgumentError::Required("messages"))?; - let fields = self.view.body_params(py, arguments, ROUTE_HOST_MODULE)?; + let mut fields = self.body_params(py, arguments)?; + let prepared = fields.remove("messages"); + let presanitized = prepared.is_some(); + let messages = match prepared { + Some(messages) => messages, + None => json_messages(&messages)?, + }; let body = [ ("model".to_string(), Value::String(model.clone())), - ("messages".to_string(), from_py(&messages)?), + ("messages".to_string(), messages), ] .into_iter() .chain(fields) @@ -117,10 +461,35 @@ impl MessagesPythonHost { .call1((&model, api_base.as_deref()))? .extract()?, }; - let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?; - let extra_headers = self.merged_headers(py, arguments)?; - let provider_specific_header = self.provider_specific_header(py, arguments)?; - let litellm_params = project_litellm_params(argument, |name| module_global(py, name))?; + let (extra_headers, mut unencodable_request_header) = match settings.extra_headers { + Some(headers) => (Some(headers), None), + None => self.merged_headers(py, arguments)?, + }; + let provider_specific_header = self.provider_specific_header( + py, + arguments, + custom_llm_provider.as_deref(), + &mut unencodable_request_header, + )?; + let shaping = MessagesShaping { + capabilities: self.capabilities(py, &model, custom_llm_provider.as_deref())?, + settings: MessagesSettings { + whole_sse_frames: settings.stream_cost_model.is_some(), + presanitized, + intercepted: self.may_intercept(py)?, + unencodable_request_header, + ..settings.settings + }, + }; + let federation_fields: &[String] = match custom_llm_provider.as_deref() { + Some(FEDERATING_PROVIDER) => workload_identity_fields(py)?, + _ => &[], + }; + let litellm_params = project_litellm_params( + federation_fields.iter().map(String::as_str), + argument, + |name| module_global(py, name), + )?; Ok(messages_body(body).and_then(|body| { Ok(MessagesCall { body, @@ -140,49 +509,55 @@ impl MessagesPythonHost { &self, py: Python<'_>, arguments: &Bound<'_, PyDict>, - ) -> PyResult>> { - let mapping = |name: &str| -> PyResult>> { + ) -> PyResult { + let mut unencodable = None; + let mut mapping = |name: &str| -> PyResult>> { self.view .argument(py, arguments, name)? - .map(|value| from_py(&value)) + .map(|value| from_py(&ascii_headers(&value, &mut unencodable)?)) .transpose() }; - Ok(merge_headers( - mapping("headers")?, - mapping("extra_headers")?, - )) + let headers = mapping("headers")?; + let extra_headers = mapping("extra_headers")?; + Ok((merge_headers(headers, extra_headers), unencodable)) } fn provider_specific_header( &self, py: Python<'_>, arguments: &Bound<'_, PyDict>, + provider: Option<&str>, + unencodable: &mut Option>, ) -> PyResult> { - self.view + let Some(value) = self + .view .argument(py, arguments, "provider_specific_header")? - .map(|value| from_py(&value)) - .transpose() + else { + return Ok(None); + }; + if unencodable.is_none() + && let Some(provider) = provider + && let Ok(scoped) = py + .import(PROVIDER_SPECIFIC_HEADERS_MODULE)? + .getattr("ProviderSpecificHeaderUtils")? + .call_method1("get_provider_specific_headers", (&value, provider, true)) + { + ascii_headers(&scoped, unencodable)?; + } + from_py(&ascii_scoped_headers(&value)?).map(Some) } - fn shaping( + fn capabilities( &self, py: Python<'_>, model: &str, custom_llm_provider: Option<&str>, - arguments: &Bound<'_, PyDict>, - ) -> PyResult { - let module = py.import(ROUTE_HOST_MODULE)?; - let capabilities: MessagesModelCapabilities = from_py( - &py.import("litellm.rust_bridge.host.model_capabilities")? + ) -> PyResult { + from_py( + &py.import(MODEL_CAPABILITIES_MODULE)? .getattr("anthropic_model_capabilities")? .call1((model, custom_llm_provider))?, - )?; - let settings: MessagesSettings = - from_py(&module.getattr("settings")?.call1((arguments,))?)?; - Ok(MessagesShaping { - capabilities, - settings, - }) + ) } fn provider(&self, py: Python<'_>) -> String { @@ -195,6 +570,14 @@ impl MessagesPythonHost { .unwrap_or_else(|| "anthropic".into()) } + fn map_transport_failure(&self, py: Python<'_>, error: PyErr) -> PyResult { + let mapped = py + .import(ROUTE_HOST_MODULE)? + .getattr("map_transport_failure")? + .call1((error.value(py), self.view.request(py)?, self.provider(py)))?; + Ok(PyErr::from_value(mapped)) + } + fn map_failure(&self, py: Python<'_>, error: PyErr) -> PyErr { if !error.is_instance_of::(py) { return error; @@ -202,7 +585,14 @@ impl MessagesPythonHost { let mapped = py .import(ROUTE_HOST_MODULE) .and_then(|module| module.getattr("map_failure")) - .and_then(|map| map.call1((error.value(py), self.view.request(py)?, self.provider(py)))) + .and_then(|map| { + map.call1(( + error.value(py), + self.view.request(py)?, + self.provider(py), + self.asynchronous, + )) + }) .and_then(|mapped| { mapped .extract::>() @@ -215,33 +605,133 @@ impl MessagesPythonHost { } } +impl MessagesPythonHost { + fn decode_call( + &mut self, + py: Python<'_>, + arguments: &Bound<'_, PyDict>, + ) -> Result> { + let settings = self + .host_settings(py, arguments) + .map_err(InvokeError::Python)?; + self.stream_cost_model = settings.stream_cost_model.clone(); + self.projection(py, arguments, settings) + .map_err(|error| InvokeError::Python(self.map_failure(py, error)))? + .map_err(InvokeError::Native) + } + + fn begin_pre_request( + &mut self, + py: Python<'_>, + reply: Reply>, + ) -> PyResult> { + let hooks = py + .import(ROUTE_HOST_MODULE)? + .getattr("pre_request_hooks")? + .call1((self.view.request(py)?,))?; + self.pre_request = Some(reply); + Ok(hooks.unbind()) + } + + fn resume_pre_request( + &mut self, + py: Python<'_>, + reply: Reply>, + result: PyResult>, + ) -> Result<(), InvokeError> { + let hooked = result?; + let arguments = hooked + .bind(py) + .cast::() + .map_err(|error| InvokeError::Python(error.into()))?; + self.view + .resolve(py, arguments) + .map_err(InvokeError::Python)?; + self.hooked = true; + let effective = self + .view + .request(py) + .and_then(|request| effective_py_args(request, arguments)) + .map_err(InvokeError::Python)?; + self.safeguards + .rebind(py, &effective) + .map_err(InvokeError::Python)?; + *self + .hooked_request + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(arguments.clone().unbind()); + match self.decode_call(py, arguments) { + Ok(call) => reply.send(Ok(call)), + Err(InvokeError::Native(error)) => reply.send(Err(error)), + Err(error) => return Err(error), + } + Ok(()) + } +} + impl PythonBinding for MessagesPythonHost { - type Protocol = Cached; + type Protocol = MessagesHosted; type Failure = PyErr; fn decode_request( &mut self, py: Python<'_>, arguments: &Bound<'_, PyDict>, - ) -> Result<(MessagesCall, Selection), InvokeError> { + ) -> Result<(MessagesCall, Selection, bool), InvokeError> { self.view .resolve(py, arguments) .map_err(InvokeError::Python)?; - let selection = - crate::cache::configure(&mut self.cache, py, arguments, "anthropic_messages") - .map_err(InvokeError::Python)?; - self.projection(py, arguments) - .map_err(|error| InvokeError::Python(self.map_failure(py, error)))? - .map_err(InvokeError::Native) - .map(|request| (request, selection)) + let function = py + .import(ROUTE_HOST_MODULE) + .and_then(|module| module.call_method0("cached_function")) + .map_err(InvokeError::Python)?; + let selection = crate::cache::configure(&mut self.cache, py, arguments, &function) + .map_err(InvokeError::Python)?; + let effective = self + .view + .request(py) + .and_then(|request| effective_py_args(request, arguments)) + .map_err(InvokeError::Python)?; + let classifies = self + .safeguards + .configure(py, &effective) + .map_err(InvokeError::Python)?; + self.classifies.store(classifies, Ordering::Release); + self.requested_stream = arguments + .get_item("stream") + .map_err(InvokeError::Python)? + .is_some_and(|stream| stream.is(pyo3::types::PyBool::new(py, true))); + self.arguments = Some(arguments.clone().unbind()); + let request = self.decode_call(py, arguments)?; + let hosted = self + .transport + .select(py, self.view.request(py).map_err(InvokeError::Python)?) + .map_err(InvokeError::Python)?; + Ok((request, selection, hosted)) } fn encode_response( &mut self, py: Python<'_>, - response: Box, + response: litellm_inference_messages::MessagesBody, ) -> PyResult> { - public_response(py, ROUTE_HOST_MODULE, response.as_ref()) + if let Some(answer) = self.answer.take() { + return Ok(answer); + } + let public = public_json_response(py, ROUTE_HOST_MODULE, &response)?; + self.safeguards.published(py, public) + } + + fn deliver_response(&mut self, py: Python<'_>, response: Py) -> PyResult> { + match self.answered.take() { + Some(Answered::Stream) => return self.cache.wrap_stream(py, response), + Some(Answered::Whole { verbatim }) => { + self.cache.store_response(py, response.bind(py), verbatim)?; + } + None if self.requested_stream => {} + None => self.cache.store_response(py, response.bind(py), false)?, + } + Ok(response) } fn encode_stream_head( @@ -249,17 +739,63 @@ impl PythonBinding for MessagesPythonHost { py: Python<'_>, head: MessagesStreamHead, ) -> PyResult> { - py.import(ROUTE_HOST_MODULE)? + let module = py.import(ROUTE_HOST_MODULE)?; + let head = module .getattr("stream_hidden_params")? - .call1((to_py(py, &head.headers)?,)) - .map(Bound::unbind) + .call1((to_py(py, &head.headers)?,))?; + if let Some(arguments) = self + .arguments + .as_ref() + .filter(|_| self.asynchronous && !self.requested_stream) + { + module + .getattr("unrequested_stream")? + .call1((&head, arguments))?; + } + Ok(head.unbind()) } fn encode_chunk(&mut self, py: Python<'_>, chunk: Bytes) -> PyResult> { - Ok(PyBytes::new(py, &chunk).into_any().unbind()) + let frames = PyBytes::new(py, &chunk).into_any(); + let Some(model) = &self.stream_cost_model else { + return Ok(frames.unbind()); + }; + py.import(ROUTE_HOST_MODULE)? + .getattr("inject_stream_cost")? + .call1((frames, model, self.view.request(py)?)) + .map(Bound::unbind) } fn map_error(&self, py: Python<'_>, error: Error) -> PyResult { + if let Some((raised, raw)) = python_failure(py, &error) { + return if raw { + Ok(raised) + } else { + self.map_transport_failure(py, raised) + }; + } + if let Some(unencodable) = litellm_error::find::(&error) { + let (value, received) = match unencodable { + UnencodableHeader::Sent(value) => (value, false), + UnencodableHeader::Received(value) => (value, true), + }; + let raised = py + .import(ROUTE_HOST_MODULE)? + .getattr("unencodable_header")? + .call1((PyBytes::new(py, value), received))?; + return self.map_transport_failure(py, PyErr::from_value(raised)); + } + if let Some(undecodable) = litellm_error::find::(&error) { + let raised = py + .import(ROUTE_HOST_MODULE)? + .getattr("decoding_failure")? + .call1(( + undecodable.to_string(), + self.view.request(py)?, + self.provider(py), + ))?; + return Ok(PyErr::from_value(raised)); + } route_failure(py, error, RequestMarker::Messages, |native| { Ok(self.map_failure(py, native)) }) @@ -270,24 +806,35 @@ impl PythonBinding for MessagesPythonHost { } } -impl PythonHostCalls> for MessagesPythonHost { +impl PythonHostCalls for MessagesPythonHost { fn handle_host_call( &mut self, py: Python<'_>, - op: CacheCall, + op: MessagesHostCall, ) -> Result<(), InvokeError> { - self.cache - .begin(py, op) - .map(|_| ()) - .map_err(InvokeError::Python) + self.begin_host_call(py, op).map(|_| ()) } fn begin_host_call( &mut self, py: Python<'_>, - op: CacheCall, + op: MessagesHostCall, ) -> Result>, InvokeError> { - self.cache.begin(py, op).map_err(InvokeError::Python) + self.awaiting_safeguards = matches!(op, MessagesHostCall::Safeguards(_)); + match op { + MessagesHostCall::Cache(op) => self.cache.begin(py, op), + MessagesHostCall::Transport(op) => { + let request = self.view.request(py).map_err(InvokeError::Python)?; + self.transport.begin(py, op, request) + } + MessagesHostCall::PreRequest(reply) => self.begin_pre_request(py, reply).map(Some), + MessagesHostCall::Safeguards(op) => self.safeguards.begin(py, op).map(Some), + MessagesHostCall::Answer { stream, reply } => self.begin_answer(py, stream, reply), + MessagesHostCall::Intercept { stream, reply } => { + self.begin_intercept(py, stream, reply) + } + } + .map_err(InvokeError::Python) } fn resume_host_call( @@ -295,6 +842,37 @@ impl PythonHostCalls> for MessagesPythonHost { py: Python<'_>, result: PyResult>, ) -> Result>, InvokeError> { + if let Some(reply) = self.pre_request.take() { + return self.resume_pre_request(py, reply, result).map(|()| None); + } + if let Some((stream, reply)) = self.pending_short_circuit.take() { + return self + .resume_short_circuit(py, stream, reply, result?) + .map_err(InvokeError::Python); + } + if let Some(reply) = self.pending_answer.take() { + self.answer = Some(result?); + reply.send(()); + return Ok(None); + } + if let Some(reply) = self.pending_intercept.take() { + self.answer = Some(result?); + reply.send(true); + return Ok(None); + } + if std::mem::take(&mut self.awaiting_safeguards) { + return self + .safeguards + .resume(py, result) + .map(|()| None) + .map_err(InvokeError::Python); + } + if self.transport.pending() { + return self + .transport + .resume(py, result) + .map_err(InvokeError::Python); + } self.cache.resume(py, result).map_err(InvokeError::Python) } } @@ -303,10 +881,25 @@ impl PythonOwned for MessagesPythonHost { fn close(&mut self, _: Python<'_>) { self.view.close(); self.cache.close(); + self.transport.close(); + self.pre_request = None; + self.safeguards.close(); + self.interceptor = None; + self.pending_answer = None; + self.pending_intercept = None; + self.pending_short_circuit = None; + self.answer = None; + self.answered = None; + self.arguments = None; } fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { self.view.traverse(visit)?; - self.cache.traverse(visit) + self.cache.traverse(visit)?; + self.transport.traverse(visit)?; + self.safeguards.traverse(visit)?; + visit.call(&self.interceptor)?; + visit.call(&self.arguments)?; + visit.call(&self.answer) } } @@ -417,7 +1010,40 @@ mod tests { assert_projection(kwargs, globals, expected); } + #[rstest] + #[case::named_fields_are_projected( + json!({"model": "m", "anthropic_federation_rule_id": "fdrl_1", "anthropic_issuer_ttl_seconds": 60}), + Ok(json!({"model": "m", "anthropic_federation_rule_id": "fdrl_1", "anthropic_issuer_ttl_seconds": 60})), + )] + #[case::unnamed_fields_are_not( + json!({"model": "m", "anthropic_unrelated": "x"}), + Ok(json!({"model": "m"})), + )] + fn workload_identity_fields_are_projected_by_the_names_the_host_lists( + #[case] kwargs: Value, + #[case] expected: Result, + ) { + assert_projection_of( + &[ + "anthropic_federation_rule_id", + "anthropic_issuer_ttl_seconds", + ], + kwargs, + json!({}), + expected, + ); + } + fn assert_projection(kwargs: Value, globals: Value, expected: Result) { + assert_projection_of(&[], kwargs, globals, expected); + } + + fn assert_projection_of( + names: &[&str], + kwargs: Value, + globals: Value, + expected: Result, + ) { Python::initialize(); Python::attach(|py| { let dict = |value: &Value| { @@ -431,6 +1057,7 @@ mod tests { let globals = dict(&globals); let bound = PyDict::new(py); let projected = project_litellm_params( + names.iter().copied(), |name| present(&kwargs, &bound, name), |name| present(&globals, &bound, name), ) diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index a9ae01bd7ee..9a1de064051 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -1,7 +1,15 @@ +mod agentic; +mod answer; mod host; +mod modify_params; +mod pre_request; +mod safeguards; +mod transport; +pub(crate) mod workload_identity; use host::MessagesPythonHost; use litellm_callbacks_legacy_python::LoggingOperation; +use modify_params::ModifyParams; use pyo3::prelude::*; use super::NativeCall; @@ -9,23 +17,61 @@ use super::NativeCall; fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyResult> { let (arguments, hooks) = crate::routes::call_hooks(py, LoggingOperation::Messages, &call, asynchronous)?; - crate::routes::run_public_call( + let hooked_request = agentic::HookedRequest::default(); + let hooks = litellm_host_python::HookChain::new() + .with(agentic::AgenticLoop::new( + asynchronous, + std::sync::Arc::clone(&hooked_request), + )) + .with(hooks); + let modify_params = ModifyParams::new(&call.args, &call.kwargs)?; + let classifies = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let classified = std::sync::Arc::clone(&classifies); + crate::routes::run_public_call_after_policy( py, arguments, move |py, arguments, request| { + let hooked = pre_request::runs_pre_request_hooks(py)?; let route = litellm_inference_messages::MessagesRoute::new( - crate::http::provider_client(py, arguments, asynchronous)? + crate::http::messages_client(py, arguments, asynchronous)? .map_err(crate::http::client_error)?, crate::http::resources().auth.clone(), crate::secrets::source(py)?, - ); + ) + .with_workload_identity(std::sync::Arc::new( + workload_identity::PythonWorkloadIdentity::new( + litellm_host_python::PythonContext::capture(py)?, + ), + )); Ok(litellm_host::call::hosted_call( request, None, - move |(call, selection): (_, crate::cache::Selection), + move |(call, selection, hosted): (_, crate::cache::Selection, bool), services, interceptors, observers| async move { + let route = if hosted { + route.with_transport(std::sync::Arc::new(transport::PythonTransport::new( + services.clone(), + ))) + } else { + route + }; + let route = match hooked { + true => route.with_pre_request_hooks(std::sync::Arc::new( + pre_request::HostPreRequestHooks::new(services.clone()), + )), + false => route, + }; + let route = match classified.load(std::sync::atomic::Ordering::Acquire) { + true => route.with_safeguards(std::sync::Arc::new( + safeguards::PythonSafeguards::new(services.clone()), + )), + false => route, + }; + let route = route.with_host_answer(std::sync::Arc::new( + answer::PythonAnswer::new(services.clone()), + )); let (cache, options) = selection.attach(services); let route = match cache { Some(cache) => route.with_cache(cache), @@ -44,24 +90,32 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR }, )) }, - MessagesPythonHost::new(call.view()?, asynchronous), + MessagesPythonHost::new(call.view()?, asynchronous, hooked_request, classifies), hooks, + modify_params, asynchronous, ) } -/// Python's sync Messages handler raises for every native provider before it sends anything -/// or opens a logging object (`BaseLLMHTTPHandler.anthropic_messages_handler`), and Rust copies -/// that until both sides serve sync calls. +/// Python's sync Messages handler answers a mock without logging, hands proxy MCP tools to the +/// gateway, and raises for every other native provider call before it sends anything or opens a +/// logging object (`BaseLLMHTTPHandler.anthropic_messages_handler`), and Rust copies that until +/// both sides serve sync calls. const SYNC_NOT_IMPLEMENTED: &str = "anthropic_messages_handler is not implemented for sync calls"; #[pyfunction] #[pyo3(signature = (call))] -pub(crate) fn messages(call: NativeCall<'_>) -> PyResult> { - drop(call); - Err(pyo3::exceptions::PyValueError::new_err( - SYNC_NOT_IMPLEMENTED, - )) +pub(crate) fn messages(py: Python<'_>, call: NativeCall<'_>) -> PyResult> { + let answer = py + .import(host::ROUTE_HOST_MODULE)? + .getattr("sync_answer")? + .call1((call.resolved()?,))?; + if answer.is_none() { + return Err(pyo3::exceptions::PyValueError::new_err( + SYNC_NOT_IMPLEMENTED, + )); + } + Ok(answer.unbind()) } #[pyfunction] diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/modify_params.rs b/litellm-rust/crates/python-bridge/src/routes/messages/modify_params.rs new file mode 100644 index 00000000000..c6f846478ad --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages/modify_params.rs @@ -0,0 +1,152 @@ +use litellm_host::hooks::CallHooks; +use litellm_host_python::{PythonOwned, PythonRuntime}; +use pyo3::{ + exceptions::PyException, + gc::{PyTraverseError, PyVisit}, + prelude::*, + types::{PyDict, PyTuple}, +}; + +use super::host::ROUTE_HOST_MODULE; + +pub(super) struct ModifyParams { + call: Option<(Py, Py)>, +} + +impl ModifyParams { + pub(super) fn new(args: &Bound<'_, PyTuple>, kwargs: &Bound<'_, PyDict>) -> PyResult { + let model = match args.get_item(0) { + Ok(model) => model, + Err(_) => kwargs + .get_item("model")? + .unwrap_or_else(|| args.py().None().into_bound(args.py())), + }; + Ok(Self { + call: Some((model.unbind(), args.clone().unbind())), + }) + } +} + +impl CallHooks for ModifyParams { + fn arguments_prepared(&mut self, py: Python<'_>, arguments: &Py) -> PyResult<()> { + let Some((model, args)) = &self.call else { + return Ok(()); + }; + let clamped = py + .import(ROUTE_HOST_MODULE) + .and_then(|module| module.getattr("modify_max_tokens")) + .and_then(|clamp| clamp.call1((model, args, arguments))); + match clamped { + Err(error) if !error.is_instance_of::(py) => Err(error), + _ => Ok(()), + } + } +} + +impl PythonOwned for ModifyParams { + fn close(&mut self, _: Python<'_>) { + self.call = None; + } + + fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + if let Some((model, args)) = &self.call { + visit.call(model)?; + visit.call(args)?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use pyo3::{ + prelude::*, + types::{PyDict, PyTuple}, + }; + use rstest::rstest; + + use super::{ModifyParams, ROUTE_HOST_MODULE}; + use litellm_host::hooks::CallHooks; + + #[rstest] + #[case::keyword_model( + "()", + "{'model': 'anthropic/claude-sonnet-4-5'}", + "'anthropic/claude-sonnet-4-5'" + )] + #[case::first_positional_wins("(64, [])", "{'model': 'anthropic/claude-sonnet-4-5'}", "64")] + #[case::no_model("()", "{}", "None")] + fn reads_the_model_the_client_wrapper_reads( + #[case] args: &str, + #[case] kwargs: &str, + #[case] expected: &str, + ) { + Python::initialize(); + Python::attach(|py| { + let eval = |source: &str| { + py.eval(&std::ffi::CString::new(source).unwrap(), None, None) + .unwrap() + }; + let args = eval(args).cast_into::().unwrap(); + let kwargs = eval(kwargs).cast_into::().unwrap(); + let hook = ModifyParams::new(&args, &kwargs).unwrap(); + let (model, captured) = hook.call.as_ref().unwrap(); + assert!(model.bind(py).eq(eval(expected)).unwrap()); + assert!(captured.bind(py).is(&args)); + }); + } + + #[rstest] + #[case::exception_leaves_max_tokens("ValueError", true)] + #[case::base_exception_propagates("KeyboardInterrupt", false)] + fn a_failing_clamp_behaves_like_the_client_wrappers_swallowed_try( + #[case] raised: &str, + #[case] swallowed: bool, + ) { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + locals.set_item("name", ROUTE_HOST_MODULE).unwrap(); + locals.set_item("raised", raised).unwrap(); + py.run( + c" +import builtins, sys, types +for parent in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host', 'litellm.rust_bridge.host.routes'): + sys.modules.setdefault(parent, types.ModuleType(parent)) +saved = sys.modules.get(name) +stub = types.ModuleType(name) +def modify_max_tokens(model, args, kwargs): + kwargs['max_tokens'] = 1 + raise getattr(builtins, raised)() +stub.modify_max_tokens = modify_max_tokens +sys.modules[name] = stub +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + let args = PyTuple::empty(py); + let kwargs = PyDict::new(py); + kwargs + .set_item("model", "anthropic/claude-sonnet-4-5") + .unwrap(); + let mut hook = ModifyParams::new(&args, &kwargs).unwrap(); + let result = hook.arguments_prepared(py, &kwargs.clone().unbind()); + py.run( + c" +if saved is None: + sys.modules.pop(name, None) +else: + sys.modules[name] = saved +", + Some(&locals), + Some(&locals), + ) + .unwrap(); + assert_eq!(result.is_ok(), swallowed); + if let Err(error) = result { + assert!(error.is_instance_of::(py)); + } + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/pre_request.rs b/litellm-rust/crates/python-bridge/src/routes/messages/pre_request.rs new file mode 100644 index 00000000000..0553fc0fa7f --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages/pre_request.rs @@ -0,0 +1,35 @@ +use futures_util::future::BoxFuture; +use litellm_host::machine::HostServices; +use litellm_inference_messages::{Error, MessagesCall, PreRequestHooks}; +use pyo3::prelude::*; + +use super::host::{MessagesHostCall, MessagesHosted, ROUTE_HOST_MODULE}; + +pub(super) fn runs_pre_request_hooks(py: Python<'_>) -> PyResult { + py.import(ROUTE_HOST_MODULE)? + .getattr("runs_pre_request_hooks")? + .call0()? + .extract() +} + +pub(super) struct HostPreRequestHooks { + services: HostServices, +} + +impl HostPreRequestHooks { + pub(super) fn new(services: HostServices) -> Self { + Self { services } + } +} + +impl PreRequestHooks for HostPreRequestHooks { + fn run(&self, call: MessagesCall) -> BoxFuture<'_, Result> { + drop(call); + Box::pin(async move { + self.services + .call(MessagesHostCall::PreRequest) + .await + .and_then(|hooked| hooked) + }) + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/safeguards.rs b/litellm-rust/crates/python-bridge/src/routes/messages/safeguards.rs new file mode 100644 index 00000000000..af478521cb3 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages/safeguards.rs @@ -0,0 +1,183 @@ +use bytes::Bytes; +use futures_util::future::BoxFuture; +use litellm_host::{machine::HostServices, protocol::Reply}; +use litellm_host_python::{from_py, to_py}; +use litellm_inference_messages::{Error, MessagesBody, Safeguards}; +use pyo3::{ + exceptions::PyRuntimeError, + gc::{PyTraverseError, PyVisit}, + prelude::*, + types::{PyBytes, PyDict}, +}; +use serde_json::Value; + +use super::host::{MessagesHostCall, MessagesHosted, ROUTE_HOST_MODULE}; + +pub(super) enum SafeguardsCall { + Results { + response: MessagesBody, + reply: Reply>, + }, + OpenStream { + reply: Reply, + }, + Frames { + chunk: Bytes, + reply: Reply>, + }, + Flush { + reply: Reply>, + }, +} + +pub(super) struct PythonSafeguards { + services: HostServices, +} + +impl PythonSafeguards { + pub(super) fn new(services: HostServices) -> Self { + Self { services } + } + + async fn ask( + &self, + call: impl FnOnce(Reply) -> SafeguardsCall + Send, + ) -> Result { + self.services + .call(|reply| MessagesHostCall::Safeguards(call(reply))) + .await + } +} + +impl Safeguards for PythonSafeguards { + fn results<'a>( + &'a self, + response: &'a MessagesBody, + ) -> BoxFuture<'a, Result, Error>> { + let response = response.clone(); + Box::pin(self.ask(|reply| SafeguardsCall::Results { response, reply })) + } + + fn open_stream(&self) -> BoxFuture<'_, Result> { + Box::pin(self.ask(|reply| SafeguardsCall::OpenStream { reply })) + } + + fn frames(&self, chunk: Bytes) -> BoxFuture<'_, Result, Error>> { + Box::pin(self.ask(|reply| SafeguardsCall::Frames { chunk, reply })) + } + + fn flush(&self) -> BoxFuture<'_, Result, Error>> { + Box::pin(self.ask(|reply| SafeguardsCall::Flush { reply })) + } +} + +enum Pending { + Results(Reply>), + OpenStream(Reply), + Frames(Reply>), +} + +#[derive(Default)] +pub(super) struct SafeguardsHost { + classifier: Option>, + pending: Option, +} + +impl SafeguardsHost { + pub(super) fn configure( + &mut self, + py: Python<'_>, + request: &Bound<'_, PyDict>, + ) -> PyResult { + let classifier = py + .import(ROUTE_HOST_MODULE)? + .getattr("safeguards")? + .call1((request,))?; + self.classifier = (!classifier.is_none()).then(|| classifier.unbind()); + Ok(self.classifier.is_some()) + } + + pub(super) fn rebind(&mut self, py: Python<'_>, request: &Bound<'_, PyDict>) -> PyResult<()> { + if self.classifier.is_none() { + return Ok(()); + } + let classifier = py + .import(ROUTE_HOST_MODULE)? + .getattr("safeguards")? + .call1((request,))?; + if !classifier.is_none() { + self.classifier = Some(classifier.unbind()); + } + Ok(()) + } + + pub(super) fn begin(&mut self, py: Python<'_>, call: SafeguardsCall) -> PyResult> { + let classifier = self + .classifier + .as_ref() + .ok_or_else(|| PyRuntimeError::new_err("safeguards call without a classifier"))? + .bind(py); + let (awaitable, pending) = match call { + SafeguardsCall::Results { response, reply } => ( + classifier.call_method1("results", (to_py(py, &response)?,))?, + Pending::Results(reply), + ), + SafeguardsCall::OpenStream { reply } => ( + classifier.call_method0("open_stream")?, + Pending::OpenStream(reply), + ), + SafeguardsCall::Frames { chunk, reply } => ( + classifier.call_method1("frames", (PyBytes::new(py, &chunk),))?, + Pending::Frames(reply), + ), + SafeguardsCall::Flush { reply } => { + (classifier.call_method0("flush")?, Pending::Frames(reply)) + } + }; + self.pending = Some(pending); + Ok(awaitable.unbind()) + } + + pub(super) fn resume(&mut self, py: Python<'_>, result: PyResult>) -> PyResult<()> { + let pending = self + .pending + .take() + .ok_or_else(|| PyRuntimeError::new_err("safeguards reply without a pending call"))?; + let value = result?.into_bound(py); + match pending { + Pending::Results(reply) => reply.send(if value.is_none() { + None + } else { + Some(from_py(&value)?) + }), + Pending::OpenStream(reply) => reply.send(value.extract()?), + Pending::Frames(reply) => reply.send( + value + .extract::>>()? + .iter() + .map(|frame| Bytes::copy_from_slice(frame.as_bytes())) + .collect(), + ), + } + Ok(()) + } + + pub(super) fn published(&self, py: Python<'_>, public: Py) -> PyResult> { + match &self.classifier { + Some(classifier) => classifier + .bind(py) + .call_method1("published", (public,)) + .map(Bound::unbind), + None => Ok(public), + } + } + + pub(super) fn close(&mut self) { + self.classifier = None; + self.pending = None; + } + + pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.classifier) + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/transport.rs b/litellm-rust/crates/python-bridge/src/routes/messages/transport.rs new file mode 100644 index 00000000000..12e782f61aa --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages/transport.rs @@ -0,0 +1,275 @@ +use std::fmt; + +use bytes::Bytes; +use futures_util::{StreamExt, future::BoxFuture}; +use litellm_host::{ + machine::HostServices, + protocol::{Protocol, Reply}, +}; +use litellm_host_python::{json_to_py, to_py}; +use litellm_inference_messages::{ + Delivered, Error, Transport, TransportBody, TransportRequest, TransportResponse, +}; +use pyo3::{ + exceptions::{PyBaseException, PyException, PyStopAsyncIteration}, + gc::{PyTraverseError, PyVisit}, + prelude::*, + types::{PyBytes, PyDict}, +}; + +use super::host::ROUTE_HOST_MODULE; + +pub(crate) enum TransportCall { + Send { + request: TransportRequest, + reply: Reply>, + }, + Read { + reply: Reply, Error>>, + }, +} + +pub(crate) struct Sent { + status: u16, + headers: Vec<(String, String)>, + body: SentBody, +} + +enum SentBody { + Text(String), + Stream, + Rejected { text: String, error: Error }, +} + +pub(super) struct PythonTransport { + services: HostServices

, +} + +impl PythonTransport

{ + pub(super) fn new(services: HostServices

) -> Self { + Self { services } + } +} + +impl

Transport for PythonTransport

+where + P: Protocol, + P::HostCall: From, +{ + fn send( + &self, + request: TransportRequest, + ) -> BoxFuture<'static, Result> { + let services = self.services.clone(); + Box::pin(async move { + let sent = services + .call(|reply| TransportCall::Send { request, reply }.into()) + .await??; + let delivered = match sent.body { + SentBody::Text(text) => Delivered::Complete(text), + SentBody::Rejected { text, error } => Delivered::Rejected { text, error }, + SentBody::Stream => Delivered::Stream( + futures_util::stream::try_unfold(services, |services| async move { + let chunk = services + .call(|reply| TransportCall::Read { reply }.into()) + .await??; + Ok(chunk.map(|chunk| (chunk, services))) + }) + .boxed(), + ), + }; + Ok(TransportResponse { + status: sent.status, + headers: sent.headers, + delivered, + }) + }) + } +} + +#[derive(thiserror::Error)] +#[error("the Python HTTP client failed")] +struct HostTransportFailure { + error: Py, + raw: bool, +} + +impl fmt::Debug for HostTransportFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("HostTransportFailure") + } +} + +fn failed(py: Python<'_>, error: PyErr, raw: bool) -> Error { + Error::transport(HostTransportFailure { + error: error.into_value(py), + raw, + }) +} + +pub(super) fn python_failure( + py: Python<'_>, + error: &(dyn std::error::Error + 'static), +) -> Option<(PyErr, bool)> { + litellm_error::find::(error).map(|failure| { + ( + PyErr::from_value(failure.error.clone_ref(py).into_bound(py).into_any()), + failure.raw, + ) + }) +} + +enum Pending { + Send(Reply>), + Read(Reply, Error>>), +} + +#[derive(Default)] +pub(super) struct HostedTransport { + handler: Option>, + chunks: Option>, + pending: Option, +} + +impl HostedTransport { + pub(super) fn select(&mut self, py: Python<'_>, request: &Bound<'_, PyDict>) -> PyResult { + let handler = py + .import(ROUTE_HOST_MODULE)? + .getattr("transport_handler")? + .call1((request,))?; + self.handler = (!handler.is_none()).then(|| handler.unbind()); + Ok(self.handler.is_some()) + } + + pub(super) fn pending(&self) -> bool { + self.pending.is_some() + } + + pub(super) fn begin( + &mut self, + py: Python<'_>, + call: TransportCall, + request: &Bound<'_, PyDict>, + ) -> PyResult>> { + match call { + TransportCall::Send { + request: sent, + reply, + } => { + let handler = self + .handler + .as_ref() + .ok_or(litellm_host_python::Error::MissingState)?; + let body = match &sent.body { + TransportBody::Json(body) => json_to_py(py, body)?.unbind(), + TransportBody::Signed(bytes) => PyBytes::new(py, bytes).into_any().unbind(), + }; + let awaitable = py + .import(ROUTE_HOST_MODULE)? + .getattr("transport_send")? + .call1(( + handler, + request, + &sent.url, + to_py(py, &sent.headers)?, + body, + sent.stream, + ))?; + self.pending = Some(Pending::Send(reply)); + Ok(Some(awaitable.unbind())) + } + TransportCall::Read { reply } => { + let Some(chunks) = self.chunks.as_ref() else { + reply.send(Ok(None)); + return Ok(None); + }; + let awaitable = chunks.bind(py).call_method0("__anext__")?; + self.pending = Some(Pending::Read(reply)); + Ok(Some(awaitable.unbind())) + } + } + } + + pub(super) fn resume( + &mut self, + py: Python<'_>, + result: PyResult>, + ) -> PyResult>> { + let result = match result { + Err(error) if !error.is_instance_of::(py) => { + self.pending = None; + return Err(error); + } + result => result, + }; + match self.pending.take() { + Some(Pending::Send(reply)) => { + reply.send(match result { + Ok(sent) => self.sent(py, sent.bind(py))?, + Err(error) => Err(self.send_failure(py, error)?), + }); + } + Some(Pending::Read(reply)) => reply.send(match result { + Ok(chunk) => Ok(Some(Bytes::copy_from_slice( + chunk.bind(py).cast::()?.as_bytes(), + ))), + Err(error) if error.is_instance_of::(py) => { + self.chunks = None; + Ok(None) + } + Err(error) => { + self.chunks = None; + Err(failed(py, error, true)) + } + }), + None => return Err(litellm_host_python::Error::MissingState.into()), + } + Ok(None) + } + + fn send_failure(&self, py: Python<'_>, error: PyErr) -> PyResult { + let raw = py + .import(ROUTE_HOST_MODULE)? + .getattr("RawTransportFailure")?; + if error.value(py).is_instance(&raw)? { + let raised = error.value(py).getattr("error")?; + return Ok(failed(py, PyErr::from_value(raised), true)); + } + Ok(failed(py, error, false)) + } + + fn sent(&mut self, py: Python<'_>, sent: &Bound<'_, PyAny>) -> PyResult> { + let status: u16 = sent.getattr("status")?.extract()?; + let headers: Vec<(String, String)> = sent.getattr("headers")?.extract()?; + let text: Option = sent.getattr("text")?.extract()?; + let rejection = sent.getattr("rejection")?; + let chunks = sent.getattr("chunks")?; + let body = if !rejection.is_none() { + SentBody::Rejected { + text: text.unwrap_or_default(), + error: failed(py, PyErr::from_value(rejection), false), + } + } else if !chunks.is_none() { + self.chunks = Some(chunks.unbind()); + SentBody::Stream + } else { + SentBody::Text(text.unwrap_or_default()) + }; + Ok(Ok(Sent { + status, + headers, + body, + })) + } + + pub(super) fn close(&mut self) { + self.pending = None; + self.chunks = None; + self.handler = None; + } + + pub(super) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.handler)?; + visit.call(&self.chunks) + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/workload_identity.rs b/litellm-rust/crates/python-bridge/src/routes/messages/workload_identity.rs new file mode 100644 index 00000000000..b8824d96584 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/messages/workload_identity.rs @@ -0,0 +1,74 @@ +use std::fmt; + +use futures_util::future::BoxFuture; +use litellm_host_python::{PythonContext, attach_blocking, to_py}; +use litellm_inference_messages::WorkloadIdentity; +use litellm_router_types::{ExtraParams, LitellmParams}; +use litellm_secret_value::SecretValue; +use pyo3::{exceptions::PyBaseException, prelude::*}; + +const MODULE: &str = "litellm.rust_bridge.host.routes.messages"; + +pub(super) struct PythonWorkloadIdentity { + context: PythonContext, +} + +impl PythonWorkloadIdentity { + pub(super) fn new(context: PythonContext) -> Self { + Self { context } + } +} + +impl WorkloadIdentity for PythonWorkloadIdentity { + fn token<'a>( + &'a self, + litellm_params: &'a LitellmParams, + api_base: Option<&'a str>, + ) -> BoxFuture<'a, Result, litellm_auth::Error>> { + let params = litellm_params.extra.clone(); + let api_base = api_base.map(str::to_owned); + let context = self.context.clone(); + Box::pin(async move { + attach_blocking(context, move |py| { + mint(py, ¶ms, api_base).map_err(|error| raised(py, error)) + }) + .await + .map_err(|error| Python::attach(|py| raised(py, error))) + .and_then(|minted| minted) + .map(|token| token.map(SecretValue::new)) + }) + } +} + +fn mint( + py: Python<'_>, + params: &ExtraParams, + api_base: Option, +) -> PyResult> { + py.import(MODULE)? + .getattr("workload_identity_token")? + .call1((to_py(py, &**params)?, api_base))? + .extract() +} + +#[derive(thiserror::Error)] +#[error("Python workload identity federation failed")] +struct PythonWorkloadIdentityError(Py); + +impl fmt::Debug for PythonWorkloadIdentityError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("PythonWorkloadIdentityError") + } +} + +fn raised(py: Python<'_>, error: PyErr) -> litellm_auth::Error { + litellm_auth::Error::provider(PythonWorkloadIdentityError(error.into_value(py))) +} + +pub(crate) fn python_error( + py: Python<'_>, + error: &(dyn std::error::Error + 'static), +) -> Option { + litellm_error::find::(error) + .map(|error| PyErr::from_value(error.0.clone_ref(py).into_bound(py).into_any())) +} diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index f51c755c33e..9fcd792bdaa 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -166,6 +166,38 @@ fn run_public_call( hooks: impl PythonCallHooks + 'static, asynchronous: bool, ) -> PyResult> +where + H: PythonBinding + PythonHostCalls + 'static, + M: Machine + 'static, + M::Complete: Into::Response>>, +{ + run_public_call_after_policy( + py, + arguments, + start, + host, + hooks, + HookChain::new(), + asynchronous, + ) +} + +fn run_public_call_after_policy( + py: Python<'_>, + arguments: Py, + start: impl FnOnce( + Python<'_>, + &Bound<'_, PyDict>, + ::Request, + ) -> PyResult + + Send + + Sync + + 'static, + host: H, + hooks: impl PythonCallHooks + 'static, + after_policy: impl PythonCallHooks + 'static, + asynchronous: bool, +) -> PyResult> where H: PythonBinding + PythonHostCalls + 'static, M: Machine + 'static, @@ -179,7 +211,8 @@ where host, HookChain::new() .with(hooks) - .with(crate::preflight::SdkPolicy), + .with(crate::preflight::SdkPolicy) + .with(after_policy), arguments, CallOptions::new(asynchronous, execution_steps), ) diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs index b797e334c6a..019f8452737 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs @@ -44,12 +44,9 @@ pub(super) fn project( let value = match (name.as_str(), value) { ("previous_response_id", serde_json::Value::String(id)) => { let decoded: String = py - .import("litellm.responses.utils")? - .getattr("ResponsesAPIRequestUtils")? - .call_method1( - "decode_previous_response_id_to_original_previous_response_id", - (id,), - )? + .import(super::ROUTE_HOST_MODULE)? + .getattr("original_previous_response_id")? + .call1((id,))? .extract()?; serde_json::Value::String(decoded) } diff --git a/litellm-rust/crates/python-bridge/src/secrets/callback.rs b/litellm-rust/crates/python-bridge/src/secrets/callback.rs index af6485a56e1..a2afb42b540 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/callback.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/callback.rs @@ -15,7 +15,6 @@ use pyo3::{ use super::error::{external_error, read_error}; -const HANDLER_MODULE: &str = "litellm.secret_managers.secret_manager_handler"; const DIAGNOSTICS_MODULE: &str = "litellm.rust_bridge.host.diagnostics"; /// A secret manager whose reads execute in Python: a custom manager, a legacy compatible @@ -61,8 +60,8 @@ impl PythonClient { None => kwargs.set_item("key_management_settings", py.None())?, } let result = py - .import(HANDLER_MODULE)? - .getattr("get_secret_from_manager")? + .import(super::SECRET_MANAGER_MODULE)? + .getattr("read_from_manager")? .call((), Some(&kwargs))?; if result.is_instance_of::() { result.extract().map(Some) @@ -133,7 +132,8 @@ mod tests { use litellm_host_python::PythonContext; - use super::{DIAGNOSTICS_MODULE, HANDLER_MODULE, PythonSecretManager, python_name}; + use super::{DIAGNOSTICS_MODULE, PythonSecretManager, python_name}; + use crate::secrets::SECRET_MANAGER_MODULE; use crate::secrets::python_error; /// `sys.modules` is interpreter-global, so tests that install or rely on the handler module @@ -169,12 +169,12 @@ class Manager: raise failure manager = Manager() import sys, types -for name in ('litellm', 'litellm.secret_managers'): +for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host'): sys.modules.setdefault(name, types.ModuleType(name)) -handler = sys.modules.setdefault('litellm.secret_managers.secret_manager_handler', types.ModuleType('litellm.secret_managers.secret_manager_handler')) -def get_secret_from_manager(**kwargs): +handler = sys.modules.setdefault('litellm.rust_bridge.host.secret_manager', types.ModuleType('litellm.rust_bridge.host.secret_manager')) +def read_from_manager(**kwargs): return kwargs['client'].sync_read_secret(kwargs['secret_name']) -handler.get_secret_from_manager = get_secret_from_manager +handler.read_from_manager = read_from_manager ", Some(&locals), Some(&locals), @@ -317,7 +317,7 @@ sys.modules.setdefault('litellm._logging', logging) }); } - /// Installs a fake `get_secret_from_manager` that records its kwargs, runs `body`, and + /// Installs a fake `read_from_manager` that records its kwargs, runs `body`, and /// removes the fake handler again; parent package stubs persist for concurrent tests. /// Callers hold `handler_guard` before attaching so the GIL is never held while waiting on it. fn with_fake_handler<'py>(py: Python<'py>, body: impl FnOnce(&Bound<'py, PyDict>)) { @@ -325,16 +325,16 @@ sys.modules.setdefault('litellm._logging', logging) py.run( c" import sys, types -previous_handler = sys.modules.get('litellm.secret_managers.secret_manager_handler') +previous_handler = sys.modules.get('litellm.rust_bridge.host.secret_manager') calls = [] -def get_secret_from_manager(**kwargs): +def read_from_manager(**kwargs): calls.append(kwargs) return 'handled-' + kwargs['secret_name'] -handler = types.ModuleType('litellm.secret_managers.secret_manager_handler') -handler.get_secret_from_manager = get_secret_from_manager -for name in ('litellm', 'litellm.secret_managers'): +handler = types.ModuleType('litellm.rust_bridge.host.secret_manager') +handler.read_from_manager = read_from_manager +for name in ('litellm', 'litellm.rust_bridge', 'litellm.rust_bridge.host'): sys.modules.setdefault(name, types.ModuleType(name)) -sys.modules['litellm.secret_managers.secret_manager_handler'] = handler +sys.modules['litellm.rust_bridge.host.secret_manager'] = handler ", Some(&locals), Some(&locals), @@ -344,9 +344,9 @@ sys.modules['litellm.secret_managers.secret_manager_handler'] = handler py.run( c" if previous_handler is None: - sys.modules.pop('litellm.secret_managers.secret_manager_handler', None) + sys.modules.pop('litellm.rust_bridge.host.secret_manager', None) else: - sys.modules['litellm.secret_managers.secret_manager_handler'] = previous_handler + sys.modules['litellm.rust_bridge.host.secret_manager'] = previous_handler ", Some(&locals), Some(&locals), @@ -366,7 +366,7 @@ else: with_fake_handler(py, |locals| { locals.set_item("expression", expression).unwrap(); py.run( - c"handler.get_secret_from_manager = lambda **kwargs: eval(expression)", + c"handler.read_from_manager = lambda **kwargs: eval(expression)", Some(locals), Some(locals), ) @@ -417,7 +417,7 @@ else: reader.client.read(py, "API_KEY").unwrap().as_deref(), Some("handled-API_KEY") ); - assert!(py.import(HANDLER_MODULE).is_ok()); + assert!(py.import(SECRET_MANAGER_MODULE).is_ok()); let calls = locals.get_item("calls").unwrap().unwrap(); let call = calls.get_item(0).unwrap().cast_into::().unwrap(); assert!(call.get_item("client").unwrap().unwrap().is(&client)); diff --git a/litellm-rust/crates/python-bridge/src/secrets/mod.rs b/litellm-rust/crates/python-bridge/src/secrets/mod.rs index 57042e6a32e..0840ecdfc6c 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/mod.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/mod.rs @@ -21,6 +21,7 @@ use resolved::ResolvedSecrets; use crate::{coercion::FieldSpec, python_settings::PythonSettings}; +const SECRET_MANAGER_MODULE: &str = "litellm.rust_bridge.host.secret_manager"; const NATIVE: FieldSpec = FieldSpec::new("native", |field| field.schema_bool()); /// Where a Rust route reads provider secrets from. Python's `get_secret_str` when the settings diff --git a/litellm-rust/crates/python-bridge/src/secrets/mutation.rs b/litellm-rust/crates/python-bridge/src/secrets/mutation.rs index b04060651ac..47cd36c4015 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/mutation.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/mutation.rs @@ -2,7 +2,7 @@ use super::operations::{PythonMutationError, PythonMutationResponse}; use litellm_error::Report; use litellm_host_python::{json_loads, to_py}; use litellm_secrets::cyberark; -use pyo3::{prelude::*, types::PyDict}; +use pyo3::prelude::*; pub(super) fn mutation_value( result: Result, @@ -62,14 +62,11 @@ fn cyberark_message( http_message(py, "POST", url, status)? } cyberark::Error::Http(source) if authentication => match os_error_code(&source) { - Some(code) => { - let reason = py.import("os")?.getattr("strerror")?.call1((code,))?; - py.import("builtins")? - .getattr("OSError")? - .call1((code, reason))? - .str()? - .extract()? - } + Some(code) => py + .import(super::SECRET_MANAGER_MODULE)? + .getattr("os_error_message")? + .call1((code,))? + .extract()?, None => Report::new(&cyberark::Error::Http(source)).to_string(), }, cyberark::Error::Http(source) if source.is_connect() => { @@ -103,13 +100,8 @@ pub(super) fn http_message( url: &str, status: u16, ) -> PyResult { - let httpx = py.import("httpx")?; - let request = httpx.getattr("Request")?.call1((method, url))?; - let kwargs = PyDict::new(py); - kwargs.set_item("request", request)?; - let response = httpx.getattr("Response")?.call((status,), Some(&kwargs))?; - match response.call_method0("raise_for_status") { - Err(error) => error.value(py).str()?.extract(), - Ok(_) => Ok(format!("HTTP {status}")), - } + py.import(super::SECRET_MANAGER_MODULE)? + .getattr("http_status_message")? + .call1((method, url, status))? + .extract() } diff --git a/litellm-rust/crates/python-bridge/src/secrets/python.rs b/litellm-rust/crates/python-bridge/src/secrets/python.rs index c5d9e3565b3..a92cf9a497a 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/python.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/python.rs @@ -9,8 +9,6 @@ use pyo3::prelude::*; use super::error::external_error; -const SECRET_MANAGER_MODULE: &str = "litellm.rust_bridge.host.secret_manager"; - /// Reads each secret through the host module's `read_secret_str`, so the configured manager, the key /// management settings and the environment fallback behave exactly as they do in Python. pub(super) struct PythonSecrets { @@ -21,7 +19,7 @@ pub(super) struct PythonSecrets { impl PythonSecrets { pub(super) fn new(py: Python<'_>) -> PyResult { Ok(Self::reading_with( - py.import(SECRET_MANAGER_MODULE)? + py.import(super::SECRET_MANAGER_MODULE)? .getattr("read_secret_str")? .unbind(), PythonContext::capture(py)?, diff --git a/litellm-rust/crates/python-bridge/src/secrets/runtime.rs b/litellm-rust/crates/python-bridge/src/secrets/runtime.rs index 0d52b0aa629..8061d1682ed 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/runtime.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/runtime.rs @@ -12,8 +12,6 @@ use pyo3::{exceptions::PyAttributeError, prelude::*, types::PyDict}; use super::ManagerError; use crate::execution::{run_async_value, run_sync_value}; -const SECRET_MANAGER_MODULE: &str = "litellm.rust_bridge.host.secret_manager"; - #[derive(Clone, PartialEq)] struct Configuration { system: KeyManagementSystem, @@ -98,7 +96,7 @@ impl NativeSecretManager { return Ok(Some(native)); } let config = py - .import(SECRET_MANAGER_MODULE)? + .import(super::SECRET_MANAGER_MODULE)? .getattr("native_secret_manager_config")? .call1((client,))?; if config.is_none() { diff --git a/litellm-rust/crates/python-bridge/src/secrets/vault.rs b/litellm-rust/crates/python-bridge/src/secrets/vault.rs index 30e3bd2c413..5f14727abaa 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/vault.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/vault.rs @@ -134,31 +134,21 @@ fn timeout_message( String::new() }); } - let kwargs = pyo3::types::PyDict::new(py); - kwargs.set_item( - "message", - transport_timeout_message( - TimeoutPhase::Read, - context.timeout, - Some(elapsed.as_secs_f64()), - ), - )?; - kwargs.set_item("model", "default-model-name")?; - kwargs.set_item("llm_provider", "litellm-httpx-handler")?; - py.import("litellm.exceptions")? - .getattr("Timeout")? - .call((), Some(&kwargs))? - .str()? + let message = transport_timeout_message( + TimeoutPhase::Read, + context.timeout, + Some(elapsed.as_secs_f64()), + ); + py.import(super::SECRET_MANAGER_MODULE)? + .getattr("timeout_message")? + .call1((message,))? .extract() } fn response_text(py: Python<'_>, body: &[u8]) -> PyResult { - let kwargs = pyo3::types::PyDict::new(py); - kwargs.set_item("content", pyo3::types::PyBytes::new(py, body))?; - py.import("httpx")? - .getattr("Response")? - .call((200,), Some(&kwargs))? - .getattr("text")? + py.import(super::SECRET_MANAGER_MODULE)? + .getattr("response_text")? + .call1((pyo3::types::PyBytes::new(py, body),))? .extract() } diff --git a/litellm-rust/crates/python-bridge/src/tokenizer.rs b/litellm-rust/crates/python-bridge/src/tokenizer.rs index 2026b2fdc42..11b21bd5673 100644 --- a/litellm-rust/crates/python-bridge/src/tokenizer.rs +++ b/litellm-rust/crates/python-bridge/src/tokenizer.rs @@ -38,16 +38,19 @@ use litellm_token_counter::huggingface::{ #[cfg(feature = "tiktoken")] use litellm_token_counter::tiktoken::{TiktokenTokenizer, Vocabulary}; +#[cfg(any(feature = "tiktoken", feature = "huggingface"))] +const HOST_MODULE: &str = "litellm.rust_bridge.host.tokenizers"; + #[cfg(feature = "tiktoken")] pub(crate) fn load_tiktoken(py: Python<'_>, encoding: &str) -> PyResult { enter_native()?; - let resource: std::path::PathBuf = - PyModule::import(py, "litellm.litellm_core_utils.tokenizers")? - .getattr("__file__")? - .extract()?; + let resources: std::path::PathBuf = PyModule::import(py, HOST_MODULE)? + .getattr("tiktoken_resource_dir")? + .call0()? + .extract()?; release_gil(py, || { TiktokenTokenizer::from_cached_ranks(encoding, |file| { - std::fs::read_to_string(resource.with_file_name(file)) + std::fs::read_to_string(resources.join(file)) }) }) .map_err(|error| token_count_error_to_pyerr(error.into())) @@ -197,14 +200,9 @@ impl Tokenizer { #[cfg(feature = "huggingface")] { enter_native()?; - let kwargs = PyDict::new(py); - kwargs.set_item("repo_id", identifier)?; - kwargs.set_item("filename", "tokenizer.json")?; - kwargs.set_item("revision", revision)?; - kwargs.set_item("token", token)?; - let path: String = PyModule::import(py, "huggingface_hub")? - .getattr("hf_hub_download")? - .call((), Some(&kwargs))? + let path: String = PyModule::import(py, HOST_MODULE)? + .getattr("download_tokenizer")? + .call1((identifier, revision, token))? .extract()?; let json = release_gil(py, || std::fs::read_to_string(path)) .map_err(TokenizerError::Download)?; diff --git a/litellm-rust/crates/python-compat/benches/formats.rs b/litellm-rust/crates/python-compat/benches/formats.rs index f0b83cb1658..2f23b882e19 100644 --- a/litellm-rust/crates/python-compat/benches/formats.rs +++ b/litellm-rust/crates/python-compat/benches/formats.rs @@ -10,7 +10,7 @@ use std::{hint::black_box, time::Duration}; use criterion::{ - BatchSize, BenchmarkGroup, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main, + BenchmarkGroup, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main, measurement::WallTime, }; use litellm_python_compat::{Value, json, literal::literal_eval, pickle, repr::repr}; @@ -72,14 +72,9 @@ fn formats(c: &mut Criterion) { let mut group = c.benchmark_group("cached_completion"); text_formats(&mut group, &text, &value); binary_formats(&mut group, &value, &pickled); - // `from_json` consumes its input, so each iteration gets a freshly parsed one. group.throughput(Throughput::Bytes(dumped.len() as u64)); - group.bench_function("from_json", |bencher| { - bencher.iter_batched( - || serde_json::from_str::(&dumped).expect("dumps output parses"), - json::from_json, - BatchSize::SmallInput, - ) + group.bench_function("json_loads", |bencher| { + bencher.iter(|| json::loads(black_box(&dumped))) }); group.finish(); } diff --git a/litellm-rust/crates/python-compat/src/error.rs b/litellm-rust/crates/python-compat/src/error.rs index 6547907056e..db09925dde9 100644 --- a/litellm-rust/crates/python-compat/src/error.rs +++ b/litellm-rust/crates/python-compat/src/error.rs @@ -6,6 +6,8 @@ use crate::MAX_DEPTH; pub enum Error { #[error("malformed Python literal at byte {0}")] InvalidLiteral(usize), + #[error("malformed JSON at byte {0}")] + InvalidJson(usize), #[error("unhashable type: '{0}'")] Unhashable(&'static str), #[error("value nests deeper than {MAX_DEPTH} levels")] diff --git a/litellm-rust/crates/python-compat/src/json.rs b/litellm-rust/crates/python-compat/src/json.rs index d3436519ac4..37cc88fec78 100644 --- a/litellm-rust/crates/python-compat/src/json.rs +++ b/litellm-rust/crates/python-compat/src/json.rs @@ -1,14 +1,19 @@ -//! `json.dumps` with CPython's default options, and the JSON value `json.loads` returns. +//! `json.dumps` with CPython's default options, and `json.loads` with its default decoder. //! //! Defaults are `ensure_ascii=True`, `allow_nan=True`, separators `(", ", ": ")`, and no key -//! sorting. Python's `json.loads` is mapped by [`from_json`]; a `serde_json` number that does -//! not fit `i64` or `u64` arrives as a float, where Python would keep an `int`. +//! sorting on the way out, and `strict=True` with `NaN`, `Infinity` and `-Infinity` read as floats +//! on the way in. -use std::fmt::Write; +use std::{ + collections::{HashMap, hash_map::Entry}, + fmt::Write, + str::FromStr, +}; +use num_bigint::BigInt; use serde_json::{Map, Number}; -use crate::{Error, Value, repr::float_repr}; +use crate::{Error, MAX_DEPTH, Value, repr::float_repr}; /// `json.dumps(value)`. pub fn dumps(value: &Value) -> Result { @@ -44,6 +49,260 @@ pub fn from_json(value: serde_json::Value) -> Value { } } +/// `json.loads(text)`. Integers keep full precision, and a repeated key keeps its first position +/// with its last value, as `dict(pairs)` does. An escape that leaves a lone surrogate fails, since +/// a Rust `String` cannot hold one. +pub fn loads(text: &str) -> Result { + let mut decoder = Decoder { + text, + bytes: text.as_bytes(), + pos: 0, + depth: 0, + }; + decoder.skip_whitespace(); + let value = decoder.value()?; + decoder.skip_whitespace(); + if decoder.pos != decoder.bytes.len() { + return Err(decoder.error()); + } + Ok(value) +} + +/// CPython's `json.scanner` and `scanstring`, one pass over the bytes. +struct Decoder<'a> { + text: &'a str, + bytes: &'a [u8], + pos: usize, + depth: usize, +} + +impl Decoder<'_> { + fn error(&self) -> Error { + Error::InvalidJson(self.pos) + } + + fn peek(&self) -> Option { + self.bytes.get(self.pos).copied() + } + + fn eat(&mut self, token: &str) -> bool { + let found = self.bytes[self.pos..].starts_with(token.as_bytes()); + if found { + self.pos += token.len(); + } + found + } + + fn expect(&mut self, token: &str) -> Result<(), Error> { + if self.eat(token) { + Ok(()) + } else { + Err(self.error()) + } + } + + fn skip_whitespace(&mut self) { + while matches!(self.peek(), Some(b' ' | b'\t' | b'\n' | b'\r')) { + self.pos += 1; + } + } + + fn value(&mut self) -> Result { + match self.peek() { + Some(b'"') => self.string().map(Value::Str), + Some(b'[') => self.nested(Self::array), + Some(b'{') => self.nested(Self::object), + _ if self.eat("null") => Ok(Value::None), + _ if self.eat("true") => Ok(Value::Bool(true)), + _ if self.eat("false") => Ok(Value::Bool(false)), + _ if self.eat("NaN") => Ok(Value::Float(f64::NAN)), + _ if self.eat("Infinity") => Ok(Value::Float(f64::INFINITY)), + _ if self.eat("-Infinity") => Ok(Value::Float(f64::NEG_INFINITY)), + _ => self.number(), + } + } + + fn nested(&mut self, parse: fn(&mut Self) -> Result) -> Result { + self.depth += 1; + if self.depth > MAX_DEPTH { + return Err(Error::TooDeep); + } + let value = parse(self)?; + self.depth -= 1; + Ok(value) + } + + fn array(&mut self) -> Result { + self.expect("[")?; + self.skip_whitespace(); + let mut values = Vec::new(); + if self.eat("]") { + return Ok(Value::List(values)); + } + loop { + values.push(self.value()?); + self.skip_whitespace(); + if self.eat("]") { + return Ok(Value::List(values)); + } + self.expect(",")?; + self.skip_whitespace(); + } + } + + fn object(&mut self) -> Result { + self.expect("{")?; + self.skip_whitespace(); + let mut entries = Vec::new(); + let mut positions = HashMap::<_, usize>::new(); + if self.eat("}") { + return Ok(Value::Dict(entries)); + } + loop { + if self.peek() != Some(b'"') { + return Err(self.error()); + } + let key = self.string()?; + self.skip_whitespace(); + self.expect(":")?; + self.skip_whitespace(); + let value = self.value()?; + match positions.entry(key) { + Entry::Occupied(position) => entries[*position.get()].1 = value, + Entry::Vacant(position) => { + entries.push((Value::Str(position.key().clone()), value)); + position.insert(entries.len() - 1); + } + } + self.skip_whitespace(); + if self.eat("}") { + return Ok(Value::Dict(entries)); + } + self.expect(",")?; + self.skip_whitespace(); + } + } + + /// `scanstring` with `strict=True`: control characters must be escaped. + fn string(&mut self) -> Result { + self.expect("\"")?; + let mut out = String::new(); + loop { + let start = self.pos; + while self + .peek() + .is_some_and(|byte| byte != b'"' && byte != b'\\' && byte >= 0x20) + { + self.pos += 1; + } + out.push_str(&self.text[start..self.pos]); + match self.peek() { + Some(b'"') => { + self.pos += 1; + return Ok(out); + } + Some(b'\\') => { + self.pos += 1; + out.push(self.escape()?); + } + _ => return Err(self.error()), + } + } + } + + fn escape(&mut self) -> Result { + let escaped = match self.peek() { + Some(b'"') => '"', + Some(b'\\') => '\\', + Some(b'/') => '/', + Some(b'b') => '\u{8}', + Some(b'f') => '\u{c}', + Some(b'n') => '\n', + Some(b'r') => '\r', + Some(b't') => '\t', + Some(b'u') => { + self.pos += 1; + return self.unicode_escape(); + } + _ => return Err(self.error()), + }; + self.pos += 1; + Ok(escaped) + } + + /// `\uXXXX`, joining a high surrogate with the low one escaped right after it. + fn unicode_escape(&mut self) -> Result { + let unit = self.hex4()?; + let code = if (0xd800..0xdc00).contains(&unit) { + self.expect("\\u")?; + let low = self.hex4()?; + if !(0xdc00..0xe000).contains(&low) { + return Err(self.error()); + } + 0x10000 + ((unit - 0xd800) << 10) + (low - 0xdc00) + } else { + unit + }; + char::from_u32(code).ok_or_else(|| self.error()) + } + + fn hex4(&mut self) -> Result { + let digits = self + .text + .get(self.pos..self.pos + 4) + .filter(|digits| digits.bytes().all(|byte| byte.is_ascii_hexdigit())) + .ok_or_else(|| self.error())?; + self.pos += 4; + u32::from_str_radix(digits, 16).map_err(|_| self.error()) + } + + /// `NUMBER_RE`: `(-?(?:0|[1-9][0-9]*))(\.[0-9]+)?([eE][-+]?[0-9]+)?`, an `int` unless a + /// fraction or exponent matched. + fn number(&mut self) -> Result { + let start = self.pos; + self.eat("-"); + match self.peek() { + Some(b'0') => self.pos += 1, + Some(b'1'..=b'9') => self.digits(), + _ => return Err(Error::InvalidJson(start)), + } + let integer_end = self.pos; + if self.peek() == Some(b'.') && self.bytes.get(self.pos + 1).is_some_and(u8::is_ascii_digit) + { + self.pos += 1; + self.digits(); + } + if matches!(self.peek(), Some(b'e' | b'E')) { + let exponent = self.pos; + self.pos += 1; + if matches!(self.peek(), Some(b'+' | b'-')) { + self.pos += 1; + } + if self.peek().is_some_and(|byte| byte.is_ascii_digit()) { + self.digits(); + } else { + self.pos = exponent; + } + } + let text = &self.text[start..self.pos]; + if self.pos == integer_end { + BigInt::from_str(text) + .map(Value::Int) + .map_err(|_| Error::InvalidJson(start)) + } else { + f64::from_str(text) + .map(Value::Float) + .map_err(|_| Error::InvalidJson(start)) + } + } + + fn digits(&mut self) { + while self.peek().is_some_and(|byte| byte.is_ascii_digit()) { + self.pos += 1; + } + } +} + /// `json.loads(json.dumps(value))` as a `serde_json` value: tuples become arrays and dict /// keys are coerced to strings as `json.dumps` does. Non-finite floats, which Python writes /// as `NaN` and `Infinity`, have no `serde_json` form and fail with [`Error::NonFiniteFloat`]. @@ -166,3 +425,86 @@ fn write_string(out: &mut String, text: &str) { } out.push('"'); } + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::{dumps, loads}; + use crate::{Error, Value, repr::repr}; + + // Expected values are the repr of CPython 3.14 json.loads(text) + #[rstest] + #[case::nan("NaN", "nan")] + #[case::infinity("Infinity", "inf")] + #[case::negative_infinity("-Infinity", "-inf")] + #[case::non_finite_in_a_list("[NaN, Infinity, -Infinity]", "[nan, inf, -inf]")] + #[case::beyond_u64("18446744073709551616", "18446744073709551616")] + #[case::exponent_overflow("1e400", "inf")] + #[case::upper_exponent("1E+2", "100.0")] + #[case::negative_zero_int("-0", "0")] + #[case::negative_zero_float("-0.0", "-0.0")] + #[case::repeated_key(r#"{"a": 1, "b": 2, "a": 3}"#, "{'a': 3, 'b': 2}")] + #[case::surrogate_pair(r#""😀""#, "'😀'")] + #[case::upper_hex_surrogate_pair(r#""😀""#, "'😀'")] + #[case::escapes( + r#"{"k": [true, false, null, 1.5, "é\/"]}"#, + "{'k': [True, False, None, 1.5, 'é/']}" + )] + #[case::delete_is_not_a_control_character("\"\x7f\"", "'\\x7f'")] + #[case::surrounding_whitespace(" \n[1]\t", "[1]")] + #[case::empty_containers("[{}, []]", "[{}, []]")] + fn loads_reads_like_python(#[case] text: &str, #[case] expected: &str) { + assert_eq!(repr(&loads(text).unwrap()), expected); + } + + // CPython 3.14 json.loads raises JSONDecodeError on each + #[rstest] + #[case::lowercase_nan("nan")] + #[case::signed_nan("-NaN")] + #[case::plus_sign("+1")] + #[case::bare_minus("-")] + #[case::leading_zero("01")] + #[case::bare_point("1.")] + #[case::bare_exponent("1e")] + #[case::trailing_data("Infinity1")] + #[case::trailing_comma("[1,]")] + #[case::missing_comma("[1 2]")] + #[case::missing_colon(r#"{"a" 1}"#)] + #[case::unquoted_key("{1: 2}")] + #[case::raw_newline_in_string("\"a\nb\"")] + #[case::unknown_escape(r#""\x41""#)] + #[case::empty("")] + #[case::byte_order_mark("\u{feff}1")] + fn loads_rejects_what_python_rejects(#[case] text: &str) { + assert!(matches!(loads(text), Err(Error::InvalidJson(_)))); + } + + #[rstest] + fn a_lone_surrogate_is_a_known_limit() { + assert!(matches!(loads(r#""\ud800""#), Err(Error::InvalidJson(_)))); + } + + #[rstest] + fn loads_reads_back_the_non_finite_floats_dumps_writes() { + let written = dumps(&Value::List(vec![ + Value::Float(f64::NAN), + Value::Float(f64::INFINITY), + Value::Float(f64::NEG_INFINITY), + ])) + .unwrap(); + + assert_eq!(repr(&loads(&written).unwrap()), "[nan, inf, -inf]"); + } + + #[rstest] + fn loads_rejects_nesting_past_max_depth() { + let text = format!( + "{}{}", + "[".repeat(crate::MAX_DEPTH + 1), + "]".repeat(crate::MAX_DEPTH + 1) + ); + + assert!(matches!(loads(&text), Err(Error::TooDeep))); + } +} diff --git a/litellm-rust/crates/python-compat/src/lib.rs b/litellm-rust/crates/python-compat/src/lib.rs index b23aff409f2..e23dbdb285e 100644 --- a/litellm-rust/crates/python-compat/src/lib.rs +++ b/litellm-rust/crates/python-compat/src/lib.rs @@ -7,9 +7,11 @@ //! |---|---| //! | [`literal`] | `ast.literal_eval(text)` | //! | [`repr`] | `repr(value)` and `str(value)` | -//! | [`json`] | `json.dumps(value)`, and `json.loads(json.dumps(value))` as a JSON value | +//! | [`reprlib`] | `reprlib.repr(value)` | +//! | [`json`] | `json.dumps(value)` and `json.loads(text)` | //! | [`pickle`] | `pickle.loads(data)` and `pickle.dumps(value)` for plain data | //! | [`truthy`] | `bool(value)` | +//! | [`number`] | `int(text)`, `float(text)`, and `int(seconds)` as a `Duration` | //! //! [`Value`] is the closed data model these formats share. Live Python objects //! (descriptors, `__bool__`, `__str__`, callbacks) are out of scope: those belong to the @@ -20,12 +22,15 @@ //! (string hashes are randomized per process). Sets keep their literal order. //! - `str` values are Rust `String`s, so lone surrogates cannot be represented. //! - [`pickle::loads`] decodes `tuple`, `set`, and `frozenset` as lists. +//! - [`number`] reads ASCII digits only, while Python's `int` and `float` accept any Unicode decimal digit. mod error; pub mod json; pub mod literal; +pub mod number; pub mod pickle; pub mod repr; +pub mod reprlib; pub mod truthy; mod value; diff --git a/litellm-rust/crates/python-compat/src/number.rs b/litellm-rust/crates/python-compat/src/number.rs new file mode 100644 index 00000000000..6e2f067ae25 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/number.rs @@ -0,0 +1,121 @@ +use std::{str::FromStr, time::Duration}; + +use num_bigint::BigInt; + +/// `int(text)` in base 10, or `None` where Python raises `ValueError`. +pub fn int(text: &str) -> Option { + parse(text) +} + +/// `float(text)`, or `None` where Python raises `ValueError`. +pub fn float(text: &str) -> Option { + parse(text) +} + +/// Why [`seconds`] refused a value, as `int(value)` would. Messages are CPython's. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum SecondsError { + /// `OverflowError`. Finite values past `Duration::MAX` land here too, where Python builds a big int. + #[error("cannot convert float infinity to integer")] + Overflow, + /// `ValueError`. + #[error("cannot convert float NaN to integer")] + NotANumber, +} + +/// A float of seconds as a `Duration`, refusing what `int(value)` refuses. Finite negative values clamp to zero. +pub fn seconds(value: f64) -> Result { + match value { + value if value.is_nan() => Err(SecondsError::NotANumber), + value if value.is_infinite() => Err(SecondsError::Overflow), + value => Duration::try_from_secs_f64(value.max(0.0)).map_err(|_| SecondsError::Overflow), + } +} + +fn parse(text: &str) -> Option { + let text = text.trim(); + let digit_at = |index: Option| { + index + .and_then(|index| text.as_bytes().get(index)) + .is_some_and(u8::is_ascii_digit) + }; + let underscores_between_digits = text + .match_indices('_') + .all(|(index, _)| digit_at(index.checked_sub(1)) && digit_at(Some(index + 1))); + if !underscores_between_digits { + return None; + } + text.replace('_', "").parse().ok() +} + +#[cfg(test)] +mod tests { + use num_bigint::BigInt; + use rstest::rstest; + + use std::time::Duration; + + use super::{SecondsError, float, int, seconds}; + + // Expected values are CPython 3.14 int(text) + #[rstest] + #[case::trailing_newline("2\n", Some(2))] + #[case::surrounding_spaces_and_underscore(" 6_380 ", Some(6380))] + #[case::plus_sign("+5", Some(5))] + #[case::negative_zero("-0", Some(0))] + #[case::leading_zeros("007", Some(7))] + #[case::unicode_spaces("\u{a0}7\u{2003}", Some(7))] + #[case::beyond_i64("99999999999999999999", Some(99_999_999_999_999_999_999_i128))] + #[case::double_underscore("1__0", None)] + #[case::leading_underscore("_1", None)] + #[case::trailing_underscore("1_", None)] + #[case::decimal_point("5.0", None)] + #[case::information_separator("\x1c7", None)] + #[case::empty("", None)] + #[case::sign_only("+", None)] + #[case::non_ascii_digit_is_a_known_limit("\u{667}", None)] + fn int_parses_like_python(#[case] text: &str, #[case] expected: Option) { + assert_eq!(int(text), expected.map(BigInt::from)); + } + + // Expected values are CPython 3.14 float(text) + #[rstest] + #[case::underscore_and_newline(" 1_0.5\n", Some(10.5))] + #[case::exponent("1e3", Some(1000.0))] + #[case::leading_point(".5", Some(0.5))] + #[case::trailing_point("5.", Some(5.0))] + #[case::plus_sign("+1.5", Some(1.5))] + #[case::infinity_word("Infinity", Some(f64::INFINITY))] + #[case::infinity_with_space("infinity ", Some(f64::INFINITY))] + #[case::underscore_before_exponent("1_e3", None)] + #[case::underscore_after_exponent("1e_3", None)] + #[case::underscore_after_point("1._5", None)] + #[case::underscore_after_word("inf_", None)] + #[case::hexadecimal("0x10", None)] + #[case::bare_exponent("1.5e", None)] + fn float_parses_like_python(#[case] text: &str, #[case] expected: Option) { + assert_eq!(float(text), expected); + } + + #[rstest] + fn float_reads_a_signed_nan_like_python() { + assert!(float("-NaN").is_some_and(f64::is_nan)); + } + + // CPython 3.14: int(inf) and int(-inf) raise OverflowError, int(nan) raises ValueError + #[rstest] + #[case::fraction(1.5, Ok(Duration::from_millis(1500)))] + #[case::zero(0.0, Ok(Duration::ZERO))] + #[case::negative_clamps_to_zero(-3.0, Ok(Duration::ZERO))] + #[case::infinity(f64::INFINITY, Err(SecondsError::Overflow))] + #[case::negative_infinity(f64::NEG_INFINITY, Err(SecondsError::Overflow))] + #[case::past_duration_max(1e20, Err(SecondsError::Overflow))] + #[case::nan(f64::NAN, Err(SecondsError::NotANumber))] + #[case::negative_nan(-f64::NAN, Err(SecondsError::NotANumber))] + fn seconds_refuses_what_int_refuses( + #[case] value: f64, + #[case] expected: Result, + ) { + assert_eq!(seconds(value), expected); + } +} diff --git a/litellm-rust/crates/python-compat/src/reprlib.rs b/litellm-rust/crates/python-compat/src/reprlib.rs new file mode 100644 index 00000000000..5d04b4e1ca7 --- /dev/null +++ b/litellm-rust/crates/python-compat/src/reprlib.rs @@ -0,0 +1,179 @@ +use crate::{Value, repr::repr}; + +const MAX_LEVEL: usize = 6; +const MAX_TUPLE: usize = 6; +const MAX_LIST: usize = 6; +const MAX_SET: usize = 6; +const MAX_DICT: usize = 4; +const MAX_STRING: usize = 30; +const MAX_LONG: usize = 40; +const MAX_OTHER: usize = 30; +const FILL: &str = "..."; + +pub fn reprlib(value: &Value) -> String { + repr1(value, MAX_LEVEL) +} + +fn repr1(value: &Value, level: usize) -> String { + match value { + Value::Str(text) => repr_str(text), + Value::Int(_) => shortened(repr(value), MAX_LONG), + Value::Tuple(values) => repr_iterable(values, level, ("(", ")"), MAX_TUPLE, ","), + Value::List(values) => repr_iterable(values, level, ("[", "]"), MAX_LIST, ""), + Value::Set(values) if values.is_empty() => "set()".to_owned(), + Value::Set(values) => repr_iterable(&sorted(values), level, ("{", "}"), MAX_SET, ""), + Value::Dict(entries) => repr_dict(entries, level), + Value::None + | Value::Bool(_) + | Value::Float(_) + | Value::Complex { .. } + | Value::Bytes(_) => shortened(repr(value), MAX_OTHER), + } +} + +fn shortened(text: String, limit: usize) -> String { + let chars: Vec = text.chars().collect(); + if chars.len() <= limit { + return text; + } + let head = (limit - FILL.len()) / 2; + let tail = limit - FILL.len() - head; + let mut out: String = chars[..head].iter().collect(); + out.push_str(FILL); + out.extend(&chars[chars.len() - tail..]); + out +} + +fn repr_str(text: &str) -> String { + let chars: Vec = text.chars().collect(); + let quoted = repr(&Value::Str(chars.iter().take(MAX_STRING).collect())); + if quoted.chars().count() <= MAX_STRING { + return quoted; + } + let head = (MAX_STRING - FILL.len()) / 2; + let tail = MAX_STRING - FILL.len() - head; + let kept: String = chars[..head.min(chars.len())] + .iter() + .chain(&chars[chars.len().saturating_sub(tail)..]) + .collect(); + let quoted: Vec = repr(&Value::Str(kept)).chars().collect(); + let mut out: String = quoted[..head].iter().collect(); + out.push_str(FILL); + out.extend("ed[quoted.len() - tail..]); + out +} + +fn repr_iterable( + values: &[Value], + level: usize, + (left, right): (&str, &str), + max_items: usize, + trail: &str, +) -> String { + if level == 0 && !values.is_empty() { + return format!("{left}{FILL}{right}"); + } + let mut pieces: Vec = values + .iter() + .take(max_items) + .map(|value| repr1(value, level - 1)) + .collect(); + if values.len() > max_items { + pieces.push(FILL.to_owned()); + } + let trail = if values.len() == 1 { trail } else { "" }; + format!("{left}{}{trail}{right}", pieces.join(", ")) +} + +fn repr_dict(entries: &[(Value, Value)], level: usize) -> String { + if entries.is_empty() { + return "{}".to_owned(); + } + if level == 0 { + return format!("{{{FILL}}}"); + } + let mut pieces: Vec = sorted_entries(entries) + .into_iter() + .take(MAX_DICT) + .map(|(key, value)| format!("{}: {}", repr1(key, level - 1), repr1(value, level - 1))) + .collect(); + if entries.len() > MAX_DICT { + pieces.push(FILL.to_owned()); + } + format!("{{{}}}", pieces.join(", ")) +} + +fn sorted_entries(entries: &[(Value, Value)]) -> Vec<&(Value, Value)> { + let mut ordered: Vec<&(Value, Value)> = entries.iter().collect(); + if entries.iter().all(|(key, _)| matches!(key, Value::Str(_))) { + ordered.sort_by(|(left, _), (right, _)| match (left, right) { + (Value::Str(left), Value::Str(right)) => left.cmp(right), + _ => std::cmp::Ordering::Equal, + }); + } + ordered +} + +fn sorted(values: &[Value]) -> Vec { + let mut ordered = values.to_vec(); + if values.iter().all(|value| matches!(value, Value::Str(_))) { + ordered.sort_by(|left, right| match (left, right) { + (Value::Str(left), Value::Str(right)) => left.cmp(right), + _ => std::cmp::Ordering::Equal, + }); + } + ordered +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn list(values: impl IntoIterator) -> Value { + Value::List(values.into_iter().collect()) + } + + fn dict(entries: &[(&str, Value)]) -> Value { + Value::Dict( + entries + .iter() + .map(|(key, value)| (Value::from(*key), value.clone())) + .collect(), + ) + } + + fn nested(depth: usize) -> Value { + (0..depth).fold(list([Value::from(1)]), |inner, _| list([inner])) + } + + #[rstest] + #[case::none(Value::None, "None")] + #[case::bool(Value::Bool(true), "True")] + #[case::int(Value::from(3), "3")] + #[case::wide_int(Value::Int("1".repeat(45).parse().unwrap()), "111111111111111111...1111111111111111111")] + #[case::float(Value::Float(0.1), "0.1")] + #[case::long_float(Value::Float(1.0 / 3.0), "0.3333333333333333")] + #[case::nan(Value::Float(f64::NAN), "nan")] + #[case::short_str(Value::from("high"), "'high'")] + #[case::long_str( + Value::from("abcdefghijklmnopqrstuvwxyz0123456789"), + "'abcdefghijkl...xyz0123456789'" + )] + #[case::quoted_str( + Value::from("it's a fairly long string with a quote"), + "\"it's a fairl... with a quote\"" + )] + #[case::list(list([Value::from(1)]), "[1]")] + #[case::long_list(list((0..8).map(Value::from)), "[0, 1, 2, 3, 4, 5, ...]")] + #[case::one_tuple(Value::Tuple(vec![Value::from(1)]), "(1,)")] + #[case::sorted_dict(dict(&[("b", Value::from(2)), ("a", Value::from(1))]), "{'a': 1, 'b': 2}")] + #[case::long_dict(dict(&[("e", Value::None), ("d", Value::None), ("c", Value::None), ("b", Value::None), ("a", Value::None)]), "{'a': None, 'b': None, 'c': None, 'd': None, ...}")] + #[case::empty_dict(dict(&[]), "{}")] + #[case::deep(nested(6), "[[[[[[[...]]]]]]]")] + #[case::deep_dict(dict(&[("a", dict(&[("b", dict(&[("c", dict(&[("d", dict(&[("e", dict(&[("f", dict(&[("g", Value::None)]))]))]))]))]))]))]), "{'a': {'b': {'c': {'d': {'e': {'f': {...}}}}}}}")] + fn values_shorten_as_reprlib_shortens_them(#[case] value: Value, #[case] expected: &str) { + assert_eq!(reprlib(&value), expected); + } +} diff --git a/litellm-rust/crates/python-compat/tests/fixtures.rs b/litellm-rust/crates/python-compat/tests/fixtures.rs index 9c202fdc5f1..3f9c6fd7010 100644 --- a/litellm-rust/crates/python-compat/tests/fixtures.rs +++ b/litellm-rust/crates/python-compat/tests/fixtures.rs @@ -76,6 +76,11 @@ const KNOWN: &[(&str, &str, &str)] = &[ "literal_eval", "deeper than MAX_DEPTH: rejected for stack safety, where CPython still parses it", ), + ( + "nested_150", + "json.loads", + "deeper than MAX_DEPTH: rejected for stack safety, where CPython still parses it", + ), ( "nested_150", "pickle.loads", @@ -341,3 +346,19 @@ fn to_json_matches_python_json_round_trip(fixtures: &Fixtures) { } mismatches.finish(); } + +/// CPython gives back the same text from `json.dumps(json.loads(text))` for every text its +/// `json.dumps` wrote here, non-finite floats included, so `loads` must too. +#[rstest] +fn loads_reads_back_what_python_json_dumps_wrote(fixtures: &Fixtures) { + let mut mismatches = Mismatches::default(); + for row in &fixtures.rows { + let Some(text) = &row.json else { continue }; + let actual = match json::loads(text).and_then(|value| json::dumps(&value)) { + Ok(text) => text, + Err(error) => format!("error: {error}"), + }; + mismatches.check(row, "json.loads", text, &actual); + } + mismatches.finish(); +} diff --git a/litellm-rust/crates/redis-client/AGENTS.md b/litellm-rust/crates/redis-client/AGENTS.md index c73a4ed8c4d..6135919c539 100644 --- a/litellm-rust/crates/redis-client/AGENTS.md +++ b/litellm-rust/crates/redis-client/AGENTS.md @@ -1,12 +1,12 @@ - The one async Redis client for the gateway and the Rust cache crates. It owns connecting, timeouts, error classes and the key namespace - It knows nothing about domains, so key builders, TTL policy, value formats and fallback decisions stay in the crate that owns the state. Every write takes its TTL from the caller, and there is no default - Async only. Build one `RedisClient` per Redis instance at the process boundary and clone it. Never add a pool, a sync API, `spawn_blocking`, or a client built inside a route -- Set every redis-rs timeout and retry explicitly. Its defaults (500ms response, 1s connect, 6 reconnect retries) are not the Python proxy's (5s socket timeout from `REDIS_SOCKET_TIMEOUT`) +- Every connection sets TCP_NODELAY, like redis-py's `socket.setsockopt(IPPROTO_TCP, TCP_NODELAY, 1)`. Set every redis-rs timeout and retry explicitly, citing the redis-py 5.3.1 or `litellm/` source next to each constant. Its defaults (500ms response, 1s connect, 6 reconnect retries) are not the Python proxy's. `Timeouts` takes `socket_timeout` and `socket_connect_timeout`, and the connect timeout falls back to the socket timeout like redis-py. Standalone reconnects on the next command with no retry of the failed one, matching redis-py's `Retry(NoBackoff(), 0)`. redis-rs keeps a socket whose command timed out, so `Renewing` in `connection.rs` swaps in a fresh `ConnectionManager`, or a fresh cluster connection like redis-py's cluster `aclose`, after a timeout. Without it a socket that went silent times out every command until TCP gives up on it. Cluster keeps redis-py's 3 `cluster_error_retry_attempts` and waits a flat 250ms between them, which is redis-py's CLUSTERDOWN sleep. redis-py's jittered per-node reconnect backoff has no redis-rs equivalent. The retry set also differs, redis-rs retries LOADING and READONLY internally, while redis-py 5.3.1 raises them. Do not claim exact retry parity - Resolving config from `config.yaml` / `REDIS_*` env the way `litellm/_redis.py` does belongs to the consumer that reads that config, not here. `RedisConfig` takes already-resolved values ## Errors - `Transport`, `Server`, `Decode`, each wrapping the redis-rs error as `#[source]`. `ConnectError` is separate because only `connect` returns it -- Classify in `From for Error` by `ErrorKind`. LOADING, CLUSTERDOWN, MASTERDOWN, TRYAGAIN and READONLY are typed server replies, but they mean the node can't serve right now, so they are `Transport`. Codes redis-rs doesn't type (WRONGTYPE) come back as `ErrorKind::Extension` and are `Server`. Each code gets its own rstest case built from a parsed RESP error +- Classify with `Error::classify(error, deployment)`, never a bare `From`, because redis-py (5.3.1) decides by topology. `Transport` is what Python's `_redis_health_error_types` counts: LOADING everywhere, CLUSTERDOWN and MASTERDOWN only on cluster (only `ClusterParser` maps them), READONLY only under Sentinel (where `SentinelManagedConnection` turns it into `ConnectionError` and rediscovers), plus NOAUTH, WRONGPASS and the `ERR` messages redis-py maps to `ConnectionError`, matched exactly like `parse_error`. TRYAGAIN is `Server`, since the cluster client already retried it. Each code gets its own rstest case built from a parsed RESP error - A `Server` error never drops the connection - `Transport` includes timeouts, where the command may still have applied. Callers of counters and locks treat it as an unknown outcome - Never swallow an error or turn it into a miss. The caller picks the fallback @@ -17,27 +17,61 @@ - Every public call (each command, `eval`, one `run` of a pipeline) takes one admission through `guarded` / `admit` in `client.rs`. Never call the connection around it - `Transport` counts as a failure (`Timeout` when `is_timeout()`, else `Hard`). `Server` and `Decode` count as nothing, since Redis answered and the request was bad. A probe that gets one still reopens, through the dropped-probe rule in `circuit-breaker`. A pipeline counts one failure if any op came back `Transport`, else one success - An open breaker returns `Error::BreakerOpen` without touching the socket -- Python doesn't count MASTERDOWN, TRYAGAIN or READONLY as health failures. Here they are `Transport` and do count ## Namespace - Applied once, here, to every key. A key that already starts with `{namespace}:` is left alone, matching `RedisCache.check_and_fix_namespace` - The rule never changes, because Python and Rust pods share keys in the same Redis ## Commands -- One typed method per command, built with `redis::cmd`. Empty batches (MGET, DEL) return without a round trip, since Redis rejects an empty MGET +- A command is a value, one constructor in `commands.rs` returning `Command`: its alone and pipelined forms and its reply decode. `RedisClient::send`, `Pipeline::with` and `RedisBatch::add` are the only entry points, and each applies its namespace to the keys once. A new command is one constructor, never a method on an executor. `ping` and `subscribe` stay methods. Empty batches (MGET, DEL) return without a round trip, since Redis rejects an empty MGET +- A store attaches its value codec with `Command::map`, never a keyspace codec inside this crate - On cluster, redis-rs splits MGET and DEL by slot and returns MGET values in input order. Other multi-key commands are not split, so a new one needs a cross-slot test -- TTLs are `Duration` and are sent in milliseconds (`PX`, `PEXPIRE`, `PTTL`) +- Every command is the one `RedisCache` sends, argument for argument, so `MONITOR` on a mixed fleet shows one shape. A difference from main's Python is a defect even when the result is the same +- TTLs are `Duration` and are sent in whole seconds (`SET ... EX`, `EXPIRE`, `TTL`), like redis-py's `int(ttl)`. `set_nx` sends `EX` before `NX`, the order redis-py builds +- Numbers go out as `Amount`, written the way redis-py encodes them: an `int` with `str()` and a `float` with `repr()` from `python-compat` +- Increments are plain commands, never a script. `incr_by` is `increment_cache`: `INCRBY`, `TTL`, then `EXPIRE` only on a key with no TTL, each its own round trip. `incr_by_float` alone is `async_increment`: `INCRBYFLOAT` with `EXPIRE` (`Expiry::Refresh`) or `TTL` (`Expiry::KeepExisting`) in one round trip, and for `KeepExisting` a third, `EXPIRE`, only on a key with no TTL. Pipelined it is `RedisBatch`'s `_Increment`, `INCRBYFLOAT` then `EXPIRE` when an expiry is given, `KeepExisting` lowered to `Refresh`. `Plan` keeps only the cases whose alone and pipelined forms differ ## Scripts and pipelines - A `Script` declares every key it touches in KEYS. On cluster an undeclared key works or fails depending on which node owns its slot, so it can pass tests and break in production -- `eval` goes through redis-rs `invoke_async`, which loads the script on NOSCRIPT. Inside a pipeline nothing reloads, so a NOSCRIPT op reruns alone with `invoke_async` +- `eval` goes through redis-rs `invoke_async`, which loads the script on NOSCRIPT. Inside a pipeline a script is sent as `EVALSHA`, like `RedisBatch`. A `NOSCRIPT` reply means that op ran nothing, so it alone runs again after the pipeline through `invoke_async` (`EVALSHA`, `SCRIPT LOAD`, `EVALSHA`), like `RedisBatch`'s `_settle_alone`. It then lands after the ops around it, as it does in Python +- `Plan::Script` keeps a script call apart from plain commands, and the one resend takes only a `ScriptCall` whose reply was `NOSCRIPT`. Nothing can resend a pipeline or a command that already ran, so a retry can never apply an increment twice +- `RedisBatch::pipelined` is one `Pipeline` whose ops answer as typed `Declared`s. A store that sends its own pipeline uses it rather than decoding `run`'s raw `Value`s - A pipeline is never atomic. Every op gets its own `Result`, from `.ignore_errors()`, so one rejected op leaves the others alone. A transport failure fails every op of that round trip -- redis-rs rejects an async cluster pipeline whose ops span slots. `Pipeline` groups ops by the slot of their first key, runs the groups concurrently and returns results in input order -- Pipeline ops take one key each. A multi-key op would span slots inside one group -- Command builders live in `commands.rs` and are shared by `RedisClient` and `Pipeline`, so a command is written once +- redis-rs rejects an async cluster pipeline whose ops span slots. `Pipeline` groups ops by the slot of their first key and runs the slots concurrently, returning results in input order. Within a slot, runs of idempotent ops (GET, SET, DEL, EXPIRE) share a request and every script op gets its own, sent in order. redis-rs retries a cluster request whole, so a script op batched with others would be applied twice when only a neighbour failed +- Pipeline ops take one key each. A multi-key op would span slots inside one group. An op can send more than one command for that key (`INCRBYFLOAT` then `EXPIRE`), and settles from the first reply, failing if any of its replies failed +- Command constructors live in `commands.rs` and every executor sends them, so a command is written once -## Tests +## Pub/sub +- Classic `PUBLISH` / `SUBSCRIBE`, never sharded, since redis-py publishes and subscribes through one plain client and cluster broadcasts classic messages to every node. Payloads are raw bytes, and the codec belongs to the store that owns the channel +- Channel names never get the namespace. Python builds `{namespace}:{channel}` itself, so the store does too +- `publish` is a command like any other, guarded by the breaker and reported to events +- `subscribe` returns a `Stream` that opens its own RESP2 `PubSub` connection when first polled, like redis-py's `PubSub`, to the endpoint, the first reachable cluster node, or the master Sentinel names at that moment. It never touches the breaker or events, like Python's subscriber, which reads off its own connection +- After a lost connection or a failed subscribe it waits and resubscribes, 5s doubling to 60s and reset by a successful subscribe, like `AuthCacheInvalidationSubscriber._run`. Messages published while it is down are lost. The connection has no idle ping, so a socket that dies silently is only noticed by TCP +- redis-rs's `PubSub` takes no credentials provider, so a provider is asked for its current credentials once when the subscription connects, which is when redis-py's pubsub connection asks it. The connect is bounded by the connect timeout + +## Batches +- `RedisBatch` is litellm's `RedisBatch` (`litellm/caching/redis_batch.py`). It keeps each declared op as data and, on a standalone or Sentinel Redis, sends them later as one `Pipeline`, so it inherits every pipeline rule above. On a cluster it never pipelines and runs each op on its own, concurrently, the way `_settle_alone` calls the `RedisCache` method behind it. An increment there keeps a TTL the key already has, as `async_increment` does, where the pipelined one refreshes it like `_Increment` +- `mget` is one `MGET` of the keys in order with repeats read once, like `_MGet`, and answers one value per key it was given, in that order. Use it instead of one `get` per key whenever Python reads several keys in one call +- It knows nothing about requests. The host that owns a request (Axum handler, pyo3 adapter) builds one and passes it down as a value, never through a task-local +- `RedisBatch::alone` runs every op on its own like the cluster path, so a fresh alone batch is "send it now" and `Declared` is the one reply handle on every route +- Each op returns a `Declared` backed by a oneshot. `flush` sends everything declared so far, one flush at a time, and an empty flush makes no round trip. `resolve` flushes only when its op has not gone out yet, so an op declared after another caller resolved costs a second round trip. Stages that know their reads up front declare all of them and call `flush` once +- An op whose flush was cancelled resolves as `Transport`, because it may already have reached Redis +- `settled` waits for whichever flush carries the op and never sends one itself, for follow-ups that must not force a flush. `RedisClient::same_client` tells clones of one client apart from a second connection, so a host can keep one batch per client - Against real Redis from `litellm-redis-testing`, never a mocked wire or asserted command bytes +- Tests that only need the public API live in `tests/.rs`, with client builders in `tests/support`. Inline `#[cfg(test)]` modules are only for private items (`error.rs`, `namespace.rs`) +- Observe connections from the server side (`RedisDeployment::clients`, `cli_on_every_node`), never by reaching into `RedisClient`'s connection - Every command test runs on each topology through `#[values(Topology::Standalone, Topology::Cluster)]`. A test is single-topology only when the behavior exists on one (a `db` number has no cluster meaning) -- Assert values and Redis state. Outages come from the container (`restart`, `freeze` / `thaw`), not `DEBUG`, which the official image disables +- Assert values and Redis state. Outages come from the container (`restart`, `freeze` / `thaw`), not `DEBUG`, which the official image disables. A socket that never answers again comes from `TcpProxy::blackhole_open_connections`, since `thaw` revives the same socket - Use a short client timeout in outage tests so they stay fast. Never sleep to wait for a state; poll with a deadline + +## TLS, credentials and Sentinel +- `TlsOptions` follows redis-py 5.3.1. It verifies the certificate by default, skips the hostname check by default (`ssl_check_hostname=False`), and trusts the system roots plus `ca_certs` +- rustls only reads unencrypted keys, so a PKCS#8 `ENCRYPTED PRIVATE KEY` is decrypted with `key_password` before it reaches redis-rs. A legacy `Proc-Type: 4,ENCRYPTED` PEM key is refused with `KeyError::LegacyEncryption` rather than decrypted by hand-rolled OpenSSL key derivation +- redis-rs loads the system trust store for every TLS connection and fails if that fails, even with `ca_certs` set. Where the macOS keychain is blocked (agent sandboxes), run the TLS tests with `SSL_CERT_FILE=/etc/ssl/cert.pem` +- `Credentials::Static` are sent at connect, and URL userinfo wins over them, like redis-py's `from_url` +- A url's `?protocol=` is ignored and every connection speaks RESP2, which every reply decode assumes +- `Credentials::Provider` wraps a redis-rs `StreamingCredentialsProvider`. redis-rs takes the first item at connect and re-`AUTH`s live connections on every later item, on each cluster node. Providers push, redis-rs never polls. With a provider, redis-rs overwrites URL userinfo at connect, which matches Python stripping `user:pass@` from the URL +- Sentinel discovers the master on each new connection and drops it on a transport failure or a READONLY reply, to one command or inside a pipeline, like redis-py's `SentinelConnectionPool` +- redis-rs sends `ROLE` to the master a Sentinel names before using it, which redis-py never does. That probe gets the data credentials (a provider's current pair) and TLS, so the Sentinel client is built on each discovery. redis-rs gives the probe no way to skip the hostname check, and drops the client certificate when the certificate is not verified, so a TLS master whose certificate does not name the announced address, or that requires a client certificate while verification is off, fails discovery where Python connects +- Sentinel monitors get `sentinel_password`, the data username when that password is set, and the TLS mode and certificates. Python's `_sentinel_auth_kwargs` forwards every other connection kwarg too, and it sends the username even without a sentinel password +- A `Debug` impl never shows a secret. Follow `.agents/skills/rust-redaction` diff --git a/litellm-rust/crates/redis-client/Cargo.toml b/litellm-rust/crates/redis-client/Cargo.toml index 3e9447a6bfa..d3d3226abc6 100644 --- a/litellm-rust/crates/redis-client/Cargo.toml +++ b/litellm-rust/crates/redis-client/Cargo.toml @@ -6,13 +6,28 @@ license.workspace = true repository.workspace = true [dependencies] +tracing.workspace = true litellm-circuit-breaker.workspace = true litellm-secret-value.workspace = true +litellm-python-compat.workspace = true +litellm-redis-types.workspace = true futures-util.workspace = true -redis = { workspace = true, features = ["cluster-async", "connection-manager", "tokio-rustls-comp"] } +redis = { workspace = true, features = [ + "cluster-async", + "connection-manager", + "sentinel", + "tls-rustls-insecure", + "token-based-authentication", + "tokio-rustls-comp", +] } +pem = "4.0.0" +pkcs8 = { version = "0.10.2", features = ["3des", "encryption", "std"] } +rustls-native-certs.workspace = true +socket2 = "0.6.5" thiserror.workspace = true +tokio = { workspace = true, features = ["rt", "sync", "time"] } [dev-dependencies] litellm-redis-testing.workspace = true +rustls.workspace = true rstest.workspace = true -tokio.workspace = true diff --git a/litellm-rust/crates/redis-client/src/batch.rs b/litellm-rust/crates/redis-client/src/batch.rs new file mode 100644 index 00000000000..84fc1643424 --- /dev/null +++ b/litellm-rust/crates/redis-client/src/batch.rs @@ -0,0 +1,147 @@ +use std::{ + io, + sync::{Arc, Mutex, MutexGuard}, +}; + +use futures_util::future::join_all; +use redis::{RedisError, Value}; +use tokio::sync::oneshot::{self, error::TryRecvError}; + +use crate::{Command, Error, Pipeline, RedisClient, commands::Op}; + +/// Ops declared by many callers and sent together, like litellm's `RedisBatch`: one pipeline on a +/// standalone or Sentinel Redis, and each op on its own on a cluster or in an alone batch. +#[derive(Clone)] +pub struct RedisBatch(Arc); + +#[derive(Clone, Copy)] +enum Mode { + /// litellm's `RedisBatch`: one pipeline, or each op on its own on a cluster. + Batch, + /// Each op on its own, the way the `RedisCache` method behind it sends it. + Alone, + /// One redis-py pipeline on every topology, split by slot on a cluster. + Pipeline, +} + +struct Shared { + client: RedisClient, + mode: Mode, + pending: Mutex, + flushing: tokio::sync::Mutex<()>, +} + +#[derive(Default)] +struct Pending { + ops: Vec, + replies: Vec>>, +} + +type Decode = Box Result + Send>; + +#[must_use] +pub struct Declared { + batch: RedisBatch, + reply: oneshot::Receiver>, + decode: Decode, +} + +impl RedisBatch { + pub(crate) fn new(client: RedisClient) -> Self { + Self::build(client, Mode::Batch) + } + + /// A batch whose every op goes out on its own, the way the `RedisCache` method behind it sends + /// it, so a fresh one is "send it now": the first `resolve` sends it. + pub fn alone(client: RedisClient) -> Self { + Self::build(client, Mode::Alone) + } + + /// A batch sent as one redis-py pipeline, like `async_increment_pipeline`, on every topology. + pub fn pipelined(client: RedisClient) -> Self { + Self::build(client, Mode::Pipeline) + } + + fn build(client: RedisClient, mode: Mode) -> Self { + Self(Arc::new(Shared { + client, + mode, + pending: Mutex::new(Pending::default()), + flushing: tokio::sync::Mutex::new(()), + })) + } + + pub fn add(&self, command: Command) -> Declared { + let (op, decode) = command.prepare(self.0.client.namespace()); + let (sender, reply) = oneshot::channel(); + let mut pending = self.pending(); + pending.ops.push(op); + pending.replies.push(sender); + Declared { + batch: self.clone(), + reply, + decode, + } + } + + pub async fn flush(&self) { + let _flushing = self.0.flushing.lock().await; + let Pending { ops, replies } = std::mem::take(&mut *self.pending()); + if replies.is_empty() { + return; + } + let client = &self.0.client; + let alone = match self.0.mode { + Mode::Alone => true, + Mode::Batch => client.is_cluster(), + Mode::Pipeline => false, + }; + let results = match alone { + true => join_all(ops.into_iter().map(|op| client.run_alone(op))).await, + false => { + client + .run(&Pipeline::of(client.namespace().clone(), ops)) + .await + } + }; + for (reply, result) in replies.into_iter().zip(results) { + let _ = reply.send(result); + } + } + + fn pending(&self) -> MutexGuard<'_, Pending> { + self.0 + .pending + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } +} + +impl Declared { + pub async fn resolve(mut self) -> Result { + match self.reply.try_recv() { + Ok(reply) => return reply.and_then(self.decode), + Err(TryRecvError::Closed) => return Err(abandoned()), + Err(TryRecvError::Empty) => {} + } + self.batch.flush().await; + self.settled().await + } + + pub async fn settled(self) -> Result { + let reply = self.reply.await.unwrap_or_else(|_| Err(abandoned())); + reply.and_then(self.decode) + } + + /// Every reply in order, flushing their batch once if nobody has yet. + pub async fn all(declared: impl IntoIterator) -> Vec> { + join_all(declared.into_iter().map(Self::resolve)).await + } +} + +fn abandoned() -> Error { + Error::Transport(RedisError::from(io::Error::new( + io::ErrorKind::Interrupted, + "the batch flush carrying this op was cancelled", + ))) +} diff --git a/litellm-rust/crates/redis-client/src/client.rs b/litellm-rust/crates/redis-client/src/client.rs index 48dbc2b2dfb..d9a8b3af7e8 100644 --- a/litellm-rust/crates/redis-client/src/client.rs +++ b/litellm-rust/crates/redis-client/src/client.rs @@ -1,21 +1,43 @@ use std::{ future::Future, + sync::{Arc, Mutex, PoisonError, Weak}, time::{Duration, Instant}, }; +use futures_util::stream::BoxStream; use litellm_circuit_breaker::{Admission, Breaker, Failure}; -use redis::{Cmd, FromRedisValue, IntoConnectionInfo, ToRedisArgs, ToSingleRedisArg, Value}; +use redis::{Cmd, ErrorKind, RedisError, ServerErrorKind, Value}; use crate::{ - ConnectError, Endpoint, Error, Namespace, Pipeline, RedisConfig, Script, commands, + CallEvent, Command, ConnectError, Error, FailureKind, Namespace, Operation, Outcome, Pipeline, + RedisBatch, RedisConfig, RedisEvents, + commands::{self, Op, Plan, Trips}, connection::Connection, + pubsub::Subscriber, }; +/// One Redis client. Clones share the connection, the breaker and the subscriber. #[derive(Clone)] -pub struct RedisClient { - pub(crate) connection: Connection, +pub struct RedisClient(Arc); + +/// A handle that does not keep the client alive, for a registry that must not outlive its users. +#[derive(Clone)] +pub struct WeakRedisClient(Weak); + +impl WeakRedisClient { + pub fn upgrade(&self) -> Option { + self.0.upgrade().map(RedisClient) + } +} + +struct Shared { + connection: Connection, + subscriber: Subscriber, namespace: Namespace, breaker: Option, + events: Option>, + health_check: Option, + last_used: Mutex, } #[derive(Debug, PartialEq, Eq)] @@ -27,122 +49,152 @@ pub enum Ttl { impl RedisClient { pub fn new(config: RedisConfig) -> Result { - let connection = match &config.endpoint { - Endpoint::Url(url) => { - Connection::standalone(url.expose().into_connection_info()?, config.timeout)? - } - Endpoint::Host(host) => { - Connection::standalone(host.connection_info()?, config.timeout)? - } - Endpoint::Cluster(cluster) => Connection::cluster(cluster, config.timeout)?, - }; - Ok(Self { + let connection = Connection::new( + &config.target, + &config.credentials, + config.timeouts, + config.keepalive, + )?; + Ok(Self(Arc::new(Shared { + subscriber: Subscriber::new( + &config.target, + &config.credentials, + config.timeouts, + config.keepalive, + &connection, + )?, connection, namespace: config.namespace, breaker: config.breaker, - }) + events: config.events, + health_check: config.health_check, + last_used: Mutex::new(Instant::now()), + }))) + } + + pub fn downgrade(&self) -> WeakRedisClient { + WeakRedisClient(Arc::downgrade(&self.0)) } pub async fn ping(&self) -> Result<(), Error> { - self.query(&commands::ping()).await - } - - pub async fn get(&self, key: &str) -> Result, Error> { - self.query(&commands::get(self.key(key))).await - } - - pub async fn mget(&self, keys: &[&str]) -> Result>, Error> { - if keys.is_empty() { - return Ok(Vec::new()); - } - self.query(&commands::mget(self.keys(keys))).await - } - - pub async fn set( - &self, - key: &str, - value: impl ToSingleRedisArg, - ttl: Duration, - ) -> Result<(), Error> { - self.query(&commands::set(self.key(key), value, ttl)).await - } - - pub async fn set_nx( - &self, - key: &str, - value: impl ToSingleRedisArg, - ttl: Duration, - ) -> Result { - let reply: Option = self - .query(&commands::set_nx(self.key(key), value, ttl)) - .await?; - Ok(reply.is_some()) - } - - pub async fn del(&self, keys: &[&str]) -> Result { - if keys.is_empty() { - return Ok(0); - } - self.query(&commands::del(self.keys(keys))).await - } - - pub async fn expire(&self, key: &str, ttl: Duration) -> Result { - self.query(&commands::expire(self.key(key), ttl)).await - } - - pub async fn ttl(&self, key: &str) -> Result { - let reply: i64 = self.query(&commands::ttl(self.key(key))).await?; - Ok(match reply { - -2 => Ttl::Missing, - -1 => Ttl::Persistent, - millis => Ttl::Expires(Duration::from_millis(millis.unsigned_abs())), - }) - } - - pub async fn incr_by(&self, key: &str, delta: i64) -> Result { - self.query(&commands::incr_by(self.key(key), delta)).await - } - - pub async fn incr_by_float(&self, key: &str, delta: f64) -> Result { - self.query(&commands::incr_by_float(self.key(key), delta)) - .await - } - - pub async fn rpush(&self, key: &str, values: impl ToRedisArgs) -> Result { - self.query(&commands::rpush(self.key(key), values)).await - } - - pub async fn eval( - &self, - script: &Script, - keys: &[&str], - args: impl ToRedisArgs, - ) -> Result { - let keys = self.keys(keys); - let args = args.to_redis_args(); - self.guarded(async { - Ok(script - .invocation(&keys, &args) - .invoke_async(&mut self.connection.clone()) - .await?) + self.guarded(Operation::Command("PING"), async { + commands::ping() + .query_async(&mut self.0.connection.clone()) + .await }) .await } + /// Sends one command on its own, the way the `RedisCache` method behind it sends it. + pub async fn send(&self, command: Command) -> Result { + let (op, decode) = command.prepare(&self.0.namespace); + self.run_alone(op).await.and_then(decode) + } + + pub(crate) async fn run_alone(&self, op: Op) -> Result { + let Op { + operation, plan, .. + } = op; + self.guarded(operation, async { + let mut connection = self.0.connection.clone(); + match plan { + Plan::Reply(value) => Ok(value), + Plan::Commands(commands) => match <[Cmd; 1]>::try_from(commands) { + Ok([command]) => command.query_async(&mut connection).await, + Err(commands) => { + let mut pipeline = redis::pipe(); + for command in commands { + pipeline.add_command(command); + } + let replies: Vec = pipeline.query_async(&mut connection).await?; + Ok(replies.into_iter().next().unwrap_or(Value::Nil)) + } + }, + Plan::Script(call) => { + call.script + .invocation(&call.keys, &call.args) + .invoke_async(&mut connection) + .await + } + Plan::KeepingTtl { + increment, + key, + ttl, + trips, + } => { + let (value, current): (Value, i64) = match trips { + Trips::Shared => { + redis::pipe() + .add_command(increment) + .add_command(commands::ttl_cmd(&key)) + .query_async(&mut connection) + .await? + } + Trips::Separate => ( + increment.query_async(&mut connection).await?, + commands::ttl_cmd(&key).query_async(&mut connection).await?, + ), + }; + if current == -1 { + redis::cmd("EXPIRE") + .arg(&key) + .arg(ttl.as_secs()) + .exec_async(&mut connection) + .await?; + } + Ok(value) + } + } + }) + .await + } + + /// Messages published on `channel`, on a connection of its own. After the connection is lost + /// it reconnects and resubscribes, and messages published meanwhile are not delivered. + pub fn subscribe(&self, channel: &str) -> BoxStream<'static, Vec> { + self.0.subscriber.clone().messages(channel.to_owned()) + } + pub fn pipeline(&self) -> Pipeline { - Pipeline::new(self.namespace.clone()) + Pipeline::new(self.0.namespace.clone()) + } + + pub(crate) fn namespace(&self) -> &Namespace { + &self.0.namespace + } + + pub fn is_cluster(&self) -> bool { + matches!(self.0.connection, Connection::Cluster(_)) + } + + pub fn same_client(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } + + pub fn batch(&self) -> RedisBatch { + RedisBatch::new(self.clone()) } pub async fn run(&self, pipeline: &Pipeline) -> Vec> { + let operation = Operation::Pipeline { + ops: pipeline.len(), + }; + let started = Instant::now(); let admission = match self.admit() { Ok(admission) => admission, Err(_) => { + self.observe( + operation, + started, + Outcome::Failed(FailureKind::BreakerOpen), + ); return (0..pipeline.len()) .map(|_| Err(Error::BreakerOpen)) .collect(); } }; - let results = pipeline.run(&self.connection).await; + self.check_health().await; + let results = pipeline.run(&self.0.connection).await; let transport = results.iter().find_map(|result| { result .as_ref() @@ -150,31 +202,76 @@ impl RedisClient { .filter(|error| matches!(error, Error::Transport(_))) }); settle(admission, transport); + let failure = transport.or_else(|| results.iter().find_map(|result| result.as_ref().err())); + let outcome = failure.map_or(Outcome::Succeeded, |error| { + Outcome::Failed(FailureKind::of(error)) + }); + self.observe(operation, started, outcome); results } - fn key(&self, key: &str) -> String { - self.namespace.apply(key) - } - - fn keys(&self, keys: &[&str]) -> Vec { - keys.iter().map(|key| self.key(key)).collect() - } - - async fn query(&self, command: &Cmd) -> Result { - self.guarded(async { Ok(command.query_async(&mut self.connection.clone()).await?) }) - .await - } - - async fn guarded(&self, call: impl Future>) -> Result { - let admission = self.admit()?; - let result = call.await; - settle(admission, result.as_ref().err()); + async fn guarded( + &self, + operation: Operation, + call: impl Future>, + ) -> Result { + let started = Instant::now(); + let result = match self.admit() { + Ok(admission) => { + self.check_health().await; + let result = call + .await + .map_err(|error| Error::classify(error, self.0.connection.deployment())); + settle(admission, result.as_ref().err()); + result + } + Err(error) => Err(error), + }; + self.observe(operation, started, Outcome::of(&result)); result } + fn observe(&self, operation: Operation, started: Instant, outcome: Outcome) { + *self + .0 + .last_used + .lock() + .unwrap_or_else(PoisonError::into_inner) = Instant::now(); + if let Some(events) = &self.0.events { + events.call(&CallEvent { + operation, + duration: started.elapsed(), + outcome, + }); + } + } + + /// redis-py pings a connection that sat idle longer than `health_check_interval` before + /// sending the next command on it. A ping that times out or finds the socket dropped leaves + /// that connection behind, as a command would, so the command then goes out on a fresh one, + /// where redis-py 5.3.1 disconnects and fails the command with the ping's error. The ping is + /// not a call of the caller's and never reaches the breaker or the events. + async fn check_health(&self) { + let Some(interval) = self.0.health_check else { + return; + }; + let idle = self + .0 + .last_used + .lock() + .unwrap_or_else(PoisonError::into_inner) + .elapsed(); + if idle < interval { + return; + } + if let Err(error) = self.0.connection.ping_every_node().await { + tracing::debug!(%error, "idle Redis connection failed its health check, replacing it"); + } + } + fn admit(&self) -> Result, Error> { - self.breaker + self.0 + .breaker .as_ref() .map(|breaker| { breaker @@ -195,458 +292,15 @@ fn settle(admission: Option, error: Option<&Error>) { admission.failed(Failure::Timeout, Instant::now()) } Some(Error::Transport(_)) => admission.failed(Failure::Hard, Instant::now()), + Some(Error::Server(error)) if proves_redis_answers(error) => admission.succeeded(), Some(Error::Server(_) | Error::Decode(_) | Error::BreakerOpen) => drop(admission), } } -#[cfg(test)] -mod tests { - use std::time::{Duration, Instant}; - - use litellm_redis_testing::RedisServer; - use litellm_secret_value::SecretValue; - use rstest::rstest; - - use litellm_circuit_breaker::{Breaker, Config, State}; - - use super::{RedisClient, Ttl}; - use crate::{ - DEFAULT_TIMEOUT, Endpoint, Error, HostEndpoint, Namespace, RedisConfig, - test_support::{Server, Topology, client_id}, - }; - - #[rstest] - #[tokio::test] - async fn namespaces_isolate_keys_on_the_server( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let a = server.client("a", DEFAULT_TIMEOUT).await; - let b = server.client("b", DEFAULT_TIMEOUT).await; - let bare = server.client("", DEFAULT_TIMEOUT).await; - - a.set("x", "1", Duration::from_secs(60)).await.unwrap(); - - assert_eq!(a.get::("x").await.unwrap().as_deref(), Some("1")); - assert_eq!(a.get::("a:x").await.unwrap().as_deref(), Some("1")); - assert_eq!( - bare.get::("a:x").await.unwrap().as_deref(), - Some("1") - ); - assert_eq!(b.get::("x").await.unwrap(), None); - } - - #[rstest] - #[tokio::test] - async fn a_rejected_command_keeps_the_connection( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", DEFAULT_TIMEOUT).await; - client.rpush("list", "item").await.unwrap(); - let before = client_id(&client, "list").await; - - let rejected = client.incr_by("list", 1).await; - - assert!(matches!(rejected, Err(Error::Server(_))), "{rejected:?}"); - assert_eq!(client.rpush("list", "next").await.unwrap(), 2); - assert_eq!(client_id(&client, "list").await, before); - } - - #[rstest] - #[tokio::test] - async fn a_reply_of_another_type_is_a_decode_error_and_a_missing_key_is_none( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", DEFAULT_TIMEOUT).await; - client - .set("float", "15.0", Duration::from_secs(60)) - .await - .unwrap(); - - let decoded = client.get::("float").await; - - assert!(matches!(decoded, Err(Error::Decode(_))), "{decoded:?}"); - assert_eq!(client.get::("missing").await.unwrap(), None); - } - - #[rstest] - #[tokio::test] - async fn a_timed_out_command_does_not_leak_its_reply_into_the_next_one( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", Duration::from_millis(500)).await; - client - .set("first", "1", Duration::from_secs(60)) - .await - .unwrap(); - client - .set("second", "2", Duration::from_secs(60)) - .await - .unwrap(); - - server.freeze().await; - let timed_out = client.get::("first").await; - server.thaw().await; - - assert!( - matches!(timed_out, Err(Error::Transport(_))), - "{timed_out:?}" - ); - assert_eq!( - client.get::("second").await.unwrap().as_deref(), - Some("2") - ); - } - - #[rstest] - #[tokio::test] - async fn the_same_client_recovers_after_redis_restarts( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", Duration::from_millis(500)).await; - - server.restart().await; - - let deadline = Instant::now() + Duration::from_secs(30); - let recovered = loop { - match client.incr_by("after_restart", 1).await { - Ok(value) => break Some(value), - Err(_) if Instant::now() < deadline => { - tokio::time::sleep(Duration::from_millis(200)).await - } - Err(_) => break None, - } - }; - assert_eq!(recovered, Some(1)); - } - - #[rstest] - #[tokio::test] - async fn a_client_created_while_redis_is_down_works_once_redis_is_up( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - server.stop().await; - - let client = server.client("", Duration::from_millis(500)).await; - let while_down = client.ping().await; - server.resume().await; - - assert!( - matches!(while_down, Err(Error::Transport(_))), - "{while_down:?}" - ); - let deadline = Instant::now() + Duration::from_secs(30); - let recovered = loop { - match client.ping().await { - Ok(()) => break true, - Err(_) if Instant::now() < deadline => { - tokio::time::sleep(Duration::from_millis(200)).await - } - Err(_) => break false, - } - }; - assert!(recovered); - } - - #[rstest] - #[tokio::test] - async fn empty_batches_succeed_without_asking_redis( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", DEFAULT_TIMEOUT).await; - - assert_eq!( - client.mget::(&[]).await.unwrap(), - Vec::>::new() - ); - assert_eq!(client.del(&[]).await.unwrap(), 0); - } - - #[rstest] - #[tokio::test] - async fn mget_and_del_span_slots_in_input_order( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", DEFAULT_TIMEOUT).await; - client.set("a", "1", Duration::from_secs(60)).await.unwrap(); - client.set("c", "3", Duration::from_secs(60)).await.unwrap(); - - let values = client.mget::(&["c", "b", "a"]).await.unwrap(); - let deleted = client.del(&["a", "b", "c"]).await.unwrap(); - - assert_eq!( - values, - vec![Some("3".to_owned()), None, Some("1".to_owned())] - ); - assert_eq!(deleted, 2); - assert_eq!( - client.mget::(&["a", "c"]).await.unwrap(), - vec![None, None] - ); - } - - #[rstest] - #[tokio::test] - async fn set_nx_only_writes_a_missing_key( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", DEFAULT_TIMEOUT).await; - - let first = client - .set_nx("lock", "owner-1", Duration::from_secs(60)) - .await - .unwrap(); - let second = client - .set_nx("lock", "owner-2", Duration::from_secs(60)) - .await - .unwrap(); - - assert!(first); - assert!(!second); - assert_eq!( - client.get::("lock").await.unwrap().as_deref(), - Some("owner-1") - ); - } - - #[rstest] - #[tokio::test] - async fn ttl_reports_missing_persistent_and_remaining_time( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", DEFAULT_TIMEOUT).await; - client.incr_by("persistent", 1).await.unwrap(); - client - .set("expiring", "1", Duration::from_secs(60)) - .await - .unwrap(); - client.incr_by_float("expired_later", 0.5).await.unwrap(); - assert!( - client - .expire("expired_later", Duration::from_secs(30)) - .await - .unwrap() - ); - - assert_eq!(client.ttl("missing").await.unwrap(), Ttl::Missing); - assert_eq!(client.ttl("persistent").await.unwrap(), Ttl::Persistent); - assert!(matches!( - client.ttl("expiring").await.unwrap(), - Ttl::Expires(left) if left > Duration::from_secs(50) && left <= Duration::from_secs(60) - )); - assert!(matches!( - client.ttl("expired_later").await.unwrap(), - Ttl::Expires(left) if left > Duration::from_secs(20) && left <= Duration::from_secs(30) - )); - assert!( - !client - .expire("missing", Duration::from_secs(30)) - .await - .unwrap() - ); - } - - #[rstest] - #[tokio::test] - async fn incr_by_float_accumulates_fractions( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", DEFAULT_TIMEOUT).await; - - client.incr_by_float("spend", 0.25).await.unwrap(); - - assert_eq!(client.incr_by_float("spend", 0.5).await.unwrap(), 0.75); - } - - #[rstest] - #[tokio::test] - async fn concurrent_commands_on_one_cloned_client_all_apply( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", DEFAULT_TIMEOUT).await; - - let tasks: Vec<_> = (0..100) - .map(|_| { - let client = client.clone(); - tokio::spawn(async move { - for _ in 0..10 { - client.incr_by("shared", 1).await.unwrap(); - } - }) - }) - .collect(); - for task in tasks { - task.await.unwrap(); - } - - assert_eq!(client.get::("shared").await.unwrap(), Some(1000)); - } - - fn breaker(failure_threshold: u32) -> Breaker { - Breaker::new( - Config { - failure_threshold, - recovery_timeout: Duration::from_millis(200), - timeout_min_duration: Duration::ZERO, - }, - Box::new(|_| {}), - ) - } - - #[rstest] - #[tokio::test] - async fn an_open_breaker_rejects_without_waiting_on_redis_and_a_probe_closes_it( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let breaker = breaker(2); - let client = server - .guarded_client("", Duration::from_millis(300), Some(breaker.clone())) - .await; - - server.freeze().await; - let first = client.get::("k").await; - let second = client.get::("k").await; - let started = Instant::now(); - let third = client.get::("k").await; - let rejected_in = started.elapsed(); - server.thaw().await; - - assert!(matches!(first, Err(Error::Transport(_))), "{first:?}"); - assert!(matches!(second, Err(Error::Transport(_))), "{second:?}"); - assert!(matches!(third, Err(Error::BreakerOpen)), "{third:?}"); - assert!(rejected_in < Duration::from_millis(50), "{rejected_in:?}"); - - let deadline = Instant::now() + Duration::from_secs(30); - let recovered = loop { - match client.get::("k").await { - Ok(value) => break Some(value), - Err(_) if Instant::now() < deadline => { - tokio::time::sleep(Duration::from_millis(50)).await - } - Err(_) => break None, - } - }; - assert_eq!(recovered, Some(None)); - assert!( - matches!(breaker.state(), State::Closed { .. }), - "{:?}", - breaker.state() - ); - } - - #[rstest] - #[tokio::test] - async fn rejected_commands_never_open_the_breaker( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let breaker = breaker(2); - let client = server - .guarded_client("", DEFAULT_TIMEOUT, Some(breaker.clone())) - .await; - client.rpush("list", "item").await.unwrap(); - - let rejected = [ - client.incr_by("list", 1).await, - client.incr_by("list", 1).await, - client.incr_by("list", 1).await, - ]; - - assert!( - matches!(rejected[2], Err(Error::Server(_))), - "{:?}", - rejected[2] - ); - assert!( - matches!(breaker.state(), State::Closed { hard: 0, .. }), - "{:?}", - breaker.state() - ); - } - - #[rstest] - #[tokio::test] - async fn an_open_breaker_fails_every_pipeline_op_without_sending( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server - .guarded_client("", Duration::from_millis(300), Some(breaker(1))) - .await; - let pipeline = client.pipeline().incr_by("a", 1).incr_by("c", 1); - - server.freeze().await; - let failed = client.run(&pipeline).await; - let rejected = client.run(&pipeline).await; - server.thaw().await; - - assert!( - matches!(failed[0], Err(Error::Transport(_))), - "{:?}", - failed[0] - ); - assert!( - matches!(rejected[0], Err(Error::BreakerOpen)), - "{:?}", - rejected[0] - ); - assert!( - matches!(rejected[1], Err(Error::BreakerOpen)), - "{:?}", - rejected[1] - ); - } - - #[rstest] - #[tokio::test] - async fn a_host_endpoint_connects_to_its_database() { - let server = RedisServer::start().await.unwrap(); - let port = server.url().rsplit(':').next().unwrap().parse().unwrap(); - let connect = |endpoint| { - RedisClient::new(RedisConfig { - endpoint, - namespace: Namespace::default(), - timeout: DEFAULT_TIMEOUT, - breaker: None, - }) - }; - let in_db_3 = connect(Endpoint::Host(HostEndpoint { - host: "127.0.0.1".to_owned(), - port, - username: None, - password: None, - tls: false, - db: 3, - })) - .unwrap(); - let in_db_0 = connect(Endpoint::Url(SecretValue::new(server.url().to_owned()))).unwrap(); - let via_url = connect(Endpoint::Url(SecretValue::new(format!( - "{}/3", - server.url() - )))) - .unwrap(); - - in_db_3 - .set("k", "3", Duration::from_secs(60)) - .await - .unwrap(); - - assert_eq!(in_db_0.get::("k").await.unwrap(), None); - assert_eq!( - via_url.get::("k").await.unwrap().as_deref(), - Some("3") - ); - } +// A replica refusing a write is the old primary mid-failover, which must not reset a run of failures +fn proves_redis_answers(error: &RedisError) -> bool { + !matches!( + error.kind(), + ErrorKind::InvalidClientConfig | ErrorKind::Server(ServerErrorKind::ReadOnly) + ) } diff --git a/litellm-rust/crates/redis-client/src/commands.rs b/litellm-rust/crates/redis-client/src/commands.rs index fb8a4a28c94..5f8c1eff3f7 100644 --- a/litellm-rust/crates/redis-client/src/commands.rs +++ b/litellm-rust/crates/redis-client/src/commands.rs @@ -1,62 +1,407 @@ use std::time::Duration; -use redis::{Cmd, ToRedisArgs, ToSingleRedisArg}; +use litellm_python_compat::repr::float_repr; +use redis::{ + Cmd, ErrorKind, FromRedisValue, RedisError, RedisWrite, ToRedisArgs, ToSingleRedisArg, Value, +}; + +use crate::{Error, Namespace, Operation, Script, Ttl, script::ScriptCall}; + +/// How an increment treats the key's TTL, the two ways `RedisCache` increments with one. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Expiry { + /// `async_increment(refresh_ttl=True)` and every pipelined increment: `EXPIRE` after each write. + Refresh(Duration), + /// `async_increment(refresh_ttl=False)`: read the `TTL` and `EXPIRE` only a key that has none. + /// Pipelined it is lowered to [`Expiry::Refresh`], like `RedisBatch.increment`. + KeepExisting(Duration), +} + +/// A number written the way redis-py encodes it: an `int` with `str()`, a `float` with `repr()`. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Amount { + Int(i64), + Float(f64), +} + +impl ToRedisArgs for Amount { + fn write_redis_args(&self, out: &mut W) { + match self { + Self::Int(value) => out.write_arg(value.to_string().as_bytes()), + Self::Float(value) => out.write_arg(float_repr(*value).as_bytes()), + } + } +} + +impl ToSingleRedisArg for Amount {} + +type Build = Box) -> Plan + Send>; +type Decode = Box Result + Send>; + +/// One Redis command as a value: what it sends alone and pipelined, and how its reply decodes. Every +/// executor (`RedisClient::send`, `Pipeline::with`, `RedisBatch::add`) applies its namespace to the +/// keys once. +#[must_use] +pub struct Command { + operation: Operation, + keys: Vec, + idempotent: bool, + build: Build, + decode: Decode, +} + +/// What a command sends once its keys carry the namespace. Only the cases whose alone and pipelined +/// forms differ get a variant of their own. +pub(crate) enum Plan { + /// Answered without a round trip, like an empty `MGET` or `DEL`. + Reply(Value), + /// Sent together; the reply is the first command's, failing if any failed. + Commands(Vec), + /// Alone through `invoke_async`, pipelined as `EVALSHA` resent once on `NOSCRIPT`. + Script(ScriptCall), + /// Alone the increment reads the `TTL` and sends `EXPIRE` only to a key without one, pipelined it + /// is the increment then `EXPIRE`. + KeepingTtl { + increment: Cmd, + key: String, + ttl: Duration, + trips: Trips, + }, +} + +#[derive(Clone, Copy)] +pub(crate) enum Trips { + /// `async_increment`: the increment and its `TTL` in one round trip. + Shared, + /// `increment_cache`: the increment and its `TTL` each on its own round trip. + Separate, +} + +/// A command with the namespace applied, ready for an executor to send. +pub(crate) struct Op { + pub(crate) operation: Operation, + pub(crate) key: String, + pub(crate) idempotent: bool, + pub(crate) plan: Plan, +} + +impl Plan { + /// What a pipeline sends for this op. + pub(crate) fn pipelined(&self) -> Vec { + match self { + Self::Reply(_) => Vec::new(), + Self::Commands(commands) => commands.clone(), + Self::Script(call) => vec![call.script.evalsha(&call.keys, &call.args)], + Self::KeepingTtl { + increment, + key, + ttl, + .. + } => vec![increment.clone(), expire_cmd(key, *ttl)], + } + } +} + +impl Command { + fn new( + operation: Operation, + keys: Vec, + idempotent: bool, + build: impl FnOnce(Vec) -> Plan + Send + 'static, + decode: impl FnOnce(Value) -> Result + Send + 'static, + ) -> Self { + Self { + operation, + keys, + idempotent, + build: Box::new(build), + decode: Box::new(decode), + } + } + + /// Attaches a codec over the decoded reply, such as a store's value format. + pub fn map(self, decode: fn(T) -> Result) -> Command + where + T: 'static, + { + let inner = self.decode; + Command { + operation: self.operation, + keys: self.keys, + idempotent: self.idempotent, + build: self.build, + decode: Box::new(move |value| inner(value).and_then(decode)), + } + } + + pub(crate) fn prepare(self, namespace: &Namespace) -> (Op, Decode) { + let keys: Vec = self.keys.iter().map(|key| namespace.apply(key)).collect(); + let key = keys.first().cloned().unwrap_or_default(); + let op = Op { + operation: self.operation, + key, + idempotent: self.idempotent, + plan: (self.build)(keys), + }; + (op, self.decode) + } +} + +fn decode(value: Value) -> Result { + T::from_redis_value(value).map_err(|error| Error::Decode(error.into())) +} + +fn owned(keys: &[&str]) -> Vec { + keys.iter().map(|key| (*key).to_owned()).collect() +} + +fn single(keys: Vec) -> String { + keys.into_iter().next().unwrap_or_default() +} + +fn expire_cmd(key: &str, ttl: Duration) -> Cmd { + redis::cmd("EXPIRE").arg(key).arg(ttl.as_secs()).to_owned() +} + +pub(crate) fn ttl_cmd(key: &str) -> Cmd { + redis::cmd("TTL").arg(key).to_owned() +} + +/// A reply whose shape a codec over it refuses, as `what` describes. +pub fn unexpected_reply(what: &'static str) -> Error { + Error::Decode(RedisError::from((ErrorKind::UnexpectedReturnType, what))) +} + +pub fn get(key: &str) -> Command> { + Command::new( + Operation::Command("GET"), + vec![key.to_owned()], + true, + |keys| Plan::Commands(vec![redis::cmd("GET").arg(single(keys)).to_owned()]), + decode, + ) +} + +/// One `MGET` of `keys` in order, repeated keys read once, like `RedisBatch.mget`, answering one +/// value per key of `keys` in their order. No keys is no round trip, since Redis rejects an empty +/// `MGET`. Pipelined, every key must share a hash slot on a cluster, and a batch never pipelines +/// there. +pub fn mget(keys: &[&str]) -> Command>> { + let unique = keys.iter().fold(Vec::::new(), |mut unique, key| { + if !unique.iter().any(|seen| seen == key) { + unique.push((*key).to_owned()); + } + unique + }); + let positions: Vec = keys + .iter() + .filter_map(|key| unique.iter().position(|seen| seen == key)) + .collect(); + Command::new( + Operation::Command("MGET"), + unique, + true, + |keys| match keys.is_empty() { + true => Plan::Reply(Value::Array(Vec::new())), + false => Plan::Commands(vec![redis::cmd("MGET").arg(keys).to_owned()]), + }, + move |reply| { + let values: Vec = decode(reply)?; + let in_key_order = positions + .iter() + .map(|&position| values.get(position).cloned()) + .collect::>>() + .ok_or_else(|| unexpected_reply("MGET answered fewer values than keys"))?; + decode(Value::Array(in_key_order)) + }, + ) +} + +// redis-py's `set(ex=...)` takes whole seconds, as `int(ttl)` or a `timedelta`'s `int(total_seconds())` +pub fn set(key: &str, value: impl ToSingleRedisArg, ttl: Duration) -> Command<()> { + let value = value.to_redis_args().concat(); + Command::new( + Operation::Command("SET"), + vec![key.to_owned()], + true, + move |keys| { + Plan::Commands(vec![ + redis::cmd("SET") + .arg(single(keys)) + .arg(value) + .arg("EX") + .arg(ttl.as_secs()) + .to_owned(), + ]) + }, + decode, + ) +} + +/// `SET ... EX ... NX`, `EX` before `NX` as redis-py builds it. `true` when this call wrote the key. +pub fn set_nx(key: &str, value: impl ToSingleRedisArg, ttl: Duration) -> Command { + let value = value.to_redis_args().concat(); + Command::new( + Operation::Command("SET"), + vec![key.to_owned()], + false, + move |keys| { + Plan::Commands(vec![ + redis::cmd("SET") + .arg(single(keys)) + .arg(value) + .arg("EX") + .arg(ttl.as_secs()) + .arg("NX") + .to_owned(), + ]) + }, + |value| decode::>(value).map(|reply| reply.is_some()), + ) +} + +/// One multi-key `DEL`, no round trip for no keys. Pipelined, every key must share a hash slot. +pub fn del(keys: &[&str]) -> Command { + Command::new( + Operation::Command("DEL"), + owned(keys), + true, + |keys| match keys.is_empty() { + true => Plan::Reply(Value::Int(0)), + false => Plan::Commands(vec![redis::cmd("DEL").arg(keys).to_owned()]), + }, + decode, + ) +} + +pub fn expire(key: &str, ttl: Duration) -> Command { + Command::new( + Operation::Command("EXPIRE"), + vec![key.to_owned()], + true, + move |keys| Plan::Commands(vec![expire_cmd(&single(keys), ttl)]), + decode, + ) +} + +pub fn ttl(key: &str) -> Command { + Command::new( + Operation::Command("TTL"), + vec![key.to_owned()], + true, + |keys| Plan::Commands(vec![ttl_cmd(&single(keys))]), + |value| { + Ok(match decode::(value)? { + -2 => Ttl::Missing, + -1 => Ttl::Persistent, + seconds => Ttl::Expires(Duration::from_secs(seconds.unsigned_abs())), + }) + }, + ) +} + +/// `RedisCache.increment_cache`: `INCRBY`, then `TTL`, then `EXPIRE` only when the key has no TTL, +/// each its own round trip. +pub fn incr_by(key: &str, delta: i64, ttl: Duration) -> Command { + Command::new( + Operation::Command("INCRBY"), + vec![key.to_owned()], + false, + move |keys| { + let key = single(keys); + Plan::KeepingTtl { + increment: redis::cmd("INCRBY").arg(&key).arg(delta).to_owned(), + key, + ttl, + trips: Trips::Separate, + } + }, + decode, + ) +} + +/// `INCRBYFLOAT`, with the TTL handled as `expiry` says, or left alone without one. Alone it is +/// `RedisCache.async_increment`, pipelined `RedisBatch.increment`. +pub fn incr_by_float(key: &str, amount: Amount, expiry: Option) -> Command { + Command::new( + Operation::Command("INCRBYFLOAT"), + vec![key.to_owned()], + false, + move |keys| { + let key = single(keys); + let increment = redis::cmd("INCRBYFLOAT").arg(&key).arg(amount).to_owned(); + match expiry { + None => Plan::Commands(vec![increment]), + Some(Expiry::Refresh(ttl)) => { + Plan::Commands(vec![increment, expire_cmd(&key, ttl)]) + } + Some(Expiry::KeepExisting(ttl)) => Plan::KeepingTtl { + increment, + key, + ttl, + trips: Trips::Shared, + }, + } + }, + decode, + ) +} + +/// A registered script, like redis-py's `Script`. Every key it touches must be in `keys`. +pub fn eval( + script: &Script, + keys: &[&str], + args: impl ToRedisArgs, +) -> Command { + let script = script.clone(); + let args = args.to_redis_args(); + Command::new( + Operation::Script, + owned(keys), + false, + move |keys| Plan::Script(ScriptCall { script, keys, args }), + decode, + ) +} + +/// A script sent with its source as `EVAL` every time, like redis-py's `Redis.eval`, for the scripts +/// `RedisCache` runs that way rather than through `register_script`. +pub fn eval_source( + source: &'static str, + keys: &[&str], + args: impl ToRedisArgs, +) -> Command { + let args = args.to_redis_args(); + Command::new( + Operation::Script, + owned(keys), + false, + move |keys| { + Plan::Commands(vec![ + redis::cmd("EVAL") + .arg(source) + .arg(keys.len()) + .arg(keys) + .arg(args) + .to_owned(), + ]) + }, + decode, + ) +} + +/// The channel is sent as given, without the namespace, like redis-py's `publish`. +pub fn publish(channel: &str, payload: &[u8]) -> Command<()> { + let command = redis::cmd("PUBLISH").arg(channel).arg(payload).to_owned(); + Command::new( + Operation::Command("PUBLISH"), + Vec::new(), + false, + move |_| Plan::Commands(vec![command]), + |_| Ok(()), + ) +} pub(crate) fn ping() -> Cmd { redis::cmd("PING") } - -pub(crate) fn get(key: String) -> Cmd { - redis::cmd("GET").arg(key).to_owned() -} - -pub(crate) fn mget(keys: Vec) -> Cmd { - redis::cmd("MGET").arg(keys).to_owned() -} - -pub(crate) fn set(key: String, value: impl ToSingleRedisArg, ttl: Duration) -> Cmd { - redis::cmd("SET") - .arg(key) - .arg(value) - .arg("PX") - .arg(millis(ttl)) - .to_owned() -} - -pub(crate) fn set_nx(key: String, value: impl ToSingleRedisArg, ttl: Duration) -> Cmd { - redis::cmd("SET") - .arg(key) - .arg(value) - .arg("NX") - .arg("PX") - .arg(millis(ttl)) - .to_owned() -} - -pub(crate) fn del(keys: Vec) -> Cmd { - redis::cmd("DEL").arg(keys).to_owned() -} - -pub(crate) fn expire(key: String, ttl: Duration) -> Cmd { - redis::cmd("PEXPIRE").arg(key).arg(millis(ttl)).to_owned() -} - -pub(crate) fn ttl(key: String) -> Cmd { - redis::cmd("PTTL").arg(key).to_owned() -} - -pub(crate) fn incr_by(key: String, delta: i64) -> Cmd { - redis::cmd("INCRBY").arg(key).arg(delta).to_owned() -} - -pub(crate) fn incr_by_float(key: String, delta: f64) -> Cmd { - redis::cmd("INCRBYFLOAT").arg(key).arg(delta).to_owned() -} - -pub(crate) fn rpush(key: String, values: impl ToRedisArgs) -> Cmd { - redis::cmd("RPUSH").arg(key).arg(values).to_owned() -} - -fn millis(duration: Duration) -> u64 { - u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) -} diff --git a/litellm-rust/crates/redis-client/src/config.rs b/litellm-rust/crates/redis-client/src/config.rs index c40b9eb6b36..f43ab4feb85 100644 --- a/litellm-rust/crates/redis-client/src/config.rs +++ b/litellm-rust/crates/redis-client/src/config.rs @@ -1,118 +1,458 @@ -use std::time::Duration; +use std::{fmt, path::PathBuf, pin::Pin, sync::Arc, time::Duration}; -use redis::{ConnectionAddr, ConnectionInfo, IntoConnectionInfo, RedisConnectionInfo, RedisResult}; +use futures_util::{Stream, StreamExt}; + +use redis::{ + BasicAuth, ClientTlsConfig, ConnectionAddr, ConnectionInfo, ErrorKind, IntoConnectionInfo, + ProtocolVersion, RedisConnectionInfo, RedisError, RedisResult, StreamingCredentialsProvider, + TlsCertificates, TlsMode, io::tcp::TcpSettings, +}; use litellm_circuit_breaker::Breaker; use litellm_secret_value::SecretValue; -use crate::Namespace; +use pkcs8::EncryptedPrivateKeyInfo; +use crate::{ConnectError, KeyError, Namespace, RedisEvents}; +use litellm_redis_types::{Cluster, RedisUrl, Target, TlsOptions}; + +// litellm/constants.py REDIS_SOCKET_TIMEOUT, which _coerce_redis_kwargs_types sets when socket_timeout is unset pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5); +// below the idle timeouts of common cloud load balancers (AWS NLB 350s, Azure Load Balancer 240s), which the +// OS default of two hours never probes within +const KEEPALIVE_IDLE: Duration = Duration::from_secs(60); pub struct RedisConfig { - pub endpoint: Endpoint, + pub target: Target, + pub credentials: Credentials, pub namespace: Namespace, - pub timeout: Duration, + pub timeouts: Timeouts, pub breaker: Option, + pub events: Option>, + /// Ping every node before the next command once the client sat idle this long, like + /// redis-py's `health_check_interval`, so a connection the server dropped silently is + /// replaced before a real command pays for it. + pub health_check: Option, + /// Probe a connection with TCP keepalive once it sat idle, like redis-py's `socket_keepalive`. + pub keepalive: bool, } -#[derive(Clone, Debug, PartialEq)] -pub enum Endpoint { - Url(SecretValue), - Host(HostEndpoint), - Cluster(ClusterEndpoint), +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Timeouts { + pub response: Duration, + pub connect: Duration, } -#[derive(Clone, Debug, PartialEq)] -pub struct HostEndpoint { - pub host: String, - pub port: u16, - pub username: Option, - pub password: Option, - pub tls: bool, - pub db: i64, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct ClusterEndpoint { - pub nodes: Vec, - pub username: Option, - pub password: Option, - pub tls: bool, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct ClusterNode { - pub host: String, - pub port: u16, -} - -impl ClusterEndpoint { - pub(crate) fn addrs(&self) -> impl Iterator + '_ { - self.nodes - .iter() - .map(|node| addr(&node.host, node.port, self.tls)) +impl Timeouts { + pub fn new(socket_timeout: Duration, connect_timeout: Option) -> Self { + Self { + response: socket_timeout, + connect: connect_timeout.unwrap_or(socket_timeout), + } } } -impl HostEndpoint { - pub(crate) fn connection_info(&self) -> RedisResult { - let settings = RedisConnectionInfo::default().set_db(self.db); - let settings = match &self.username { - Some(username) => settings.set_username(username.expose()), - None => settings, - }; - let settings = match &self.password { - Some(password) => settings.set_password(password.expose()), - None => settings, - }; - Ok(addr(&self.host, self.port, self.tls) - .into_connection_info()? - .set_redis_settings(settings)) +impl Default for Timeouts { + fn default() -> Self { + Self::new(DEFAULT_TIMEOUT, None) } } -fn addr(host: &str, port: u16, tls: bool) -> ConnectionAddr { +#[derive(Clone, Debug)] +pub enum Credentials { + Static { + username: Option, + password: Option, + }, + Provider(CredentialsProvider), +} + +#[derive(Clone)] +pub struct CredentialsProvider(pub Arc); + +impl fmt::Debug for CredentialsProvider { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("CredentialsProvider") + .finish_non_exhaustive() + } +} + +impl StreamingCredentialsProvider for CredentialsProvider { + fn subscribe(&self) -> Pin> + Send + 'static>> { + self.0.subscribe() + } +} + +impl Credentials { + pub const NONE: Self = Self::Static { + username: None, + password: None, + }; +} + +pub(crate) fn tls_mode(tls: &TlsOptions) -> TlsMode { + match tls.verify_certificate { + true => TlsMode::Secure, + false => TlsMode::Insecure, + } +} + +pub(crate) fn tls_certificates(tls: &TlsOptions) -> Result { + let client_tls = match (&tls.certfile, &tls.keyfile) { + (Some(certfile), keyfile) => Some(ClientTlsConfig { + client_cert: read(certfile)?, + client_key: client_key( + keyfile.as_ref().unwrap_or(certfile), + tls.key_password.as_ref(), + )?, + }), + (None, _) => None, + }; + let from_file: Option> = tls.ca_certs.as_ref().map(read).transpose()?; + let from_directory: Vec> = tls + .ca_path + .as_ref() + .map(read_directory) + .transpose()? + .unwrap_or_default(); + let configured: Vec> = from_file + .into_iter() + .chain(tls.ca_data.iter().map(|pem| pem.clone().into_bytes())) + .chain(from_directory) + .collect(); + Ok(TlsCertificates { + client_tls, + root_cert: (!configured.is_empty()).then(|| with_system_roots(configured.join(&b'\n'))), + }) +} + +/// Every regular file in `path`, in name order, as redis-py hands `ssl_ca_path` to +/// `load_verify_locations(capath=...)` +fn read_directory(path: &PathBuf) -> Result>, ConnectError> { + let tls_file = |source| ConnectError::TlsFile { + path: path.display().to_string(), + source, + }; + let mut files: Vec = std::fs::read_dir(path) + .map_err(tls_file)? + .map(|entry| entry.map(|entry| entry.path())) + .collect::>() + .map_err(tls_file)?; + files.sort(); + files + .iter() + .filter(|file| file.is_file()) + .map(read) + .collect() +} + +pub(crate) fn apply_tls( + tls: &TlsOptions, + info: ConnectionInfo, +) -> Result { + let tls_info = redis::Client::build_with_tls(info, tls_certificates(tls)?)? + .get_connection_info() + .clone(); + let mut addr = match tls_info.addr().clone() { + ConnectionAddr::TcpTls { + host, + port, + tls_params, + .. + } => ConnectionAddr::TcpTls { + host, + port, + insecure: !tls.verify_certificate, + tls_params, + }, + addr => addr, + }; + addr.set_danger_accept_invalid_hostnames(!tls.check_hostname); + Ok(tls_info.set_addr(addr)) +} + +fn with_system_roots(configured: Vec) -> Vec { + let system = rustls_native_certs::load_native_certs(); + let system_pem = system + .certs + .iter() + .map(|cert| pem::encode(&pem::Pem::new("CERTIFICATE", cert.to_vec()))); + configured + .into_iter() + .chain(b"\n".iter().copied()) + .chain(system_pem.flat_map(String::into_bytes)) + .collect() +} + +/// The key as unencrypted PEM, decrypting a PKCS#8 `ENCRYPTED PRIVATE KEY` with `ssl_password` the way +/// `load_cert_chain(password=...)` does, since rustls only reads unencrypted keys. Like OpenSSL, a password next to +/// an unencrypted key is unused +fn client_key(path: &PathBuf, password: Option<&SecretValue>) -> Result, ConnectError> { + let contents = read(path)?; + let key_error = |reason| ConnectError::TlsKey { + path: path.display().to_string(), + reason, + }; + let Ok(sections) = pem::parse_many(&contents) else { + return Ok(contents); + }; + let encrypted = sections + .iter() + .find(|section| section.tag() == "ENCRYPTED PRIVATE KEY"); + match (encrypted, password) { + (Some(_), None) => Err(key_error(KeyError::MissingPassword)), + (Some(section), Some(password)) => { + let decrypted = EncryptedPrivateKeyInfo::try_from(section.contents()) + .and_then(|info| info.decrypt(password.expose())) + .map_err(|source| key_error(KeyError::Decrypt(source)))?; + Ok(pem::encode(&pem::Pem::new("PRIVATE KEY", decrypted.as_bytes())).into_bytes()) + } + (None, _) if sections.iter().any(is_legacy_encrypted) => { + Err(key_error(KeyError::LegacyEncryption)) + } + (None, _) => Ok(contents), + } +} + +fn is_legacy_encrypted(section: &pem::Pem) -> bool { + section + .headers() + .get("Proc-Type") + .is_some_and(|proc_type| proc_type.contains("ENCRYPTED")) +} + +fn read(path: &PathBuf) -> Result, ConnectError> { + std::fs::read(path).map_err(|source| ConnectError::TlsFile { + path: path.display().to_string(), + source, + }) +} + +pub(crate) fn addr(host: &str, port: u16, tls: Option<&TlsOptions>) -> ConnectionAddr { match tls { - true => ConnectionAddr::TcpTls { + Some(tls) => ConnectionAddr::TcpTls { host: host.to_owned(), port, - insecure: false, + insecure: !tls.verify_certificate, tls_params: None, }, - false => ConnectionAddr::Tcp(host.to_owned(), port), + None => ConnectionAddr::Tcp(host.to_owned(), port), } } +pub(crate) fn cluster_addrs(cluster: &Cluster) -> impl Iterator + '_ { + cluster + .nodes + .iter() + .map(|node| addr(&node.host, node.port, cluster.tls.as_ref())) +} + +pub(crate) fn tcp_settings(keepalive: bool) -> TcpSettings { + let settings = TcpSettings::default().set_nodelay(true); + match keepalive { + true => settings.set_keepalive(socket2::TcpKeepalive::new().with_time(KEEPALIVE_IDLE)), + false => settings, + } +} + +pub(crate) fn connection_info( + target: &Target, + credentials: &Credentials, + keepalive: bool, +) -> Result { + let (info, tls) = match target { + Target::Url(url) => (url_info(url, credentials)?, url.tls.as_ref()), + Target::Standalone(host) => { + let info = addr(&host.host, host.port, host.tls.as_ref()).into_connection_info()?; + let settings = info.redis_settings().clone().set_db(host.db); + (info.set_redis_settings(settings), host.tls.as_ref()) + } + Target::Unix(socket) => { + let info = ConnectionAddr::Unix(socket.path.clone()).into_connection_info()?; + let settings = info.redis_settings().clone().set_db(socket.db); + (info.set_redis_settings(settings), None) + } + Target::Cluster(_) | Target::Sentinel(_) => { + unreachable!("cluster and sentinel endpoints build their own connection info") + } + }; + let info = with_static_credentials(info, credentials).set_tcp_settings(tcp_settings(keepalive)); + match (tls, info.addr()) { + (Some(tls), ConnectionAddr::TcpTls { .. }) => apply_tls(tls, info), + _ => Ok(info), + } +} + +// parse_url reads ?db= over the path, and the url's userinfo over ?username= and ?password=. A credential +// provider supersedes both, as it does in redis-py. ?protocol=3 would switch redis-rs to RESP3, which no +// reply decode here expects, so the protocol is ignored as the resolver warns +fn url_info(url: &RedisUrl, credentials: &Credentials) -> Result { + let info = url.url.expose().into_connection_info()?; + let settings = info + .redis_settings() + .clone() + .set_protocol(ProtocolVersion::RESP2); + let settings = match url.db { + Some(db) => settings.set_db(db), + None => settings, + }; + let info = info.set_redis_settings(settings); + Ok(match credentials { + Credentials::Static { .. } => with_static_credentials( + info, + &Credentials::Static { + username: url.username.clone(), + password: url.password.clone(), + }, + ), + Credentials::Provider(_) => info, + }) +} + +fn with_static_credentials(info: ConnectionInfo, credentials: &Credentials) -> ConnectionInfo { + let settings = static_credentials(info.redis_settings().clone(), credentials); + info.set_redis_settings(settings) +} + +fn static_credentials( + settings: RedisConnectionInfo, + credentials: &Credentials, +) -> RedisConnectionInfo { + let Credentials::Static { username, password } = credentials else { + return settings; + }; + let settings = match (username, settings.username()) { + (Some(username), None) => settings.set_username(username.expose()), + _ => settings, + }; + match (password, settings.password()) { + (Some(password), None) => settings.set_password(password.expose()), + _ => settings, + } +} + +/// `settings` with the credentials a connection opened now sends. A provider is asked once for its +/// current pair, which is when redis-py asks its `credential_provider`. +pub(crate) async fn authenticated( + settings: RedisConnectionInfo, + credentials: &Credentials, +) -> RedisResult { + let Credentials::Provider(provider) = credentials else { + return Ok(static_credentials(settings, credentials)); + }; + let current = provider.subscribe().next().await.ok_or_else(|| { + RedisError::from(( + ErrorKind::AuthenticationFailed, + "the Redis credential provider ended before giving credentials", + )) + })??; + Ok(settings + .set_username(current.username()) + .set_password(current.password())) +} + #[cfg(test)] mod tests { - use litellm_secret_value::SecretValue; + use litellm_redis_types::{RedisUrl, UnixSocket}; use rstest::rstest; - use super::{ClusterEndpoint, Endpoint, HostEndpoint}; + use super::*; #[rstest] - #[case::url(Endpoint::Url(SecretValue::new("redis://credential-value@cache:6379/0")))] - #[case::host( - Endpoint::Host(HostEndpoint { - host: "cache".to_owned(), - port: 6379, - username: Some(SecretValue::new("credential-value")), - password: Some(SecretValue::new("credential-value")), - tls: false, - db: 0, - }) - )] - #[case::cluster(Endpoint::Cluster(ClusterEndpoint { - nodes: Vec::new(), - username: Some(SecretValue::new("credential-value")), - password: Some(SecretValue::new("credential-value")), - tls: false, + #[case::host(Target::Standalone(crate::Standalone { host: "redis".into(), port: 6379, tls: None, db: 0 }))] + #[case::url(Target::Url(RedisUrl { + url: SecretValue::new("redis://cache:6379".to_owned()), + db: None, + username: None, + password: None, + tls: None, }))] - fn debug_output_never_shows_the_secret(#[case] endpoint: Endpoint) { - let debug = format!("{endpoint:?}"); + fn keepalive_probes_an_idle_socket_only_when_enabled(#[case] target: Target) { + let probed = |keepalive| { + connection_info(&target, &Credentials::NONE, keepalive) + .unwrap() + .tcp_settings() + .keepalive() + .is_some() + }; - assert!(!debug.contains("credential-value"), "{debug}"); - assert!(debug.contains("[REDACTED]"), "{debug}"); + assert!(probed(true)); + assert!(!probed(false)); + } + + #[test] + fn a_unix_socket_replaces_host_port_and_tls_like_redis_py() { + let target = Target::Unix(UnixSocket { + path: "/tmp/redis.sock".into(), + db: 3, + }); + + let info = connection_info(&target, &Credentials::NONE, true).unwrap(); + + assert_eq!(*info.addr(), ConnectionAddr::Unix("/tmp/redis.sock".into())); + assert_eq!(info.redis_settings().db(), 3); + } + + #[rstest] + #[case::the_query_db_wins_over_the_path("redis://cache:6379/1", Some(3), None, 3, None)] + #[case::the_path_db_without_a_query_db("redis://cache:6379/1", None, None, 1, None)] + #[case::the_query_password_fills_a_missing_userinfo( + "redis://cache:6379", + None, + Some("query"), + 0, + Some("query") + )] + #[case::the_userinfo_wins_over_the_query_password( + "redis://:userinfo@cache:6379", + None, + Some("query"), + 0, + Some("userinfo") + )] + fn a_url_reads_its_query_like_parse_url( + #[case] url: &str, + #[case] db: Option, + #[case] password: Option<&str>, + #[case] expected_db: i64, + #[case] expected_password: Option<&str>, + ) { + let target = Target::Url(RedisUrl { + url: SecretValue::new(url), + tls: None, + db, + username: None, + password: password.map(SecretValue::new), + }); + let static_password = Credentials::Static { + username: None, + password: Some(SecretValue::new("static")), + }; + + let info = connection_info(&target, &static_password, true).unwrap(); + + assert_eq!(info.redis_settings().db(), expected_db); + assert_eq!( + info.redis_settings().password(), + expected_password.or(Some("static")) + ); + } + + #[rstest] + #[case::by_number("redis://cache:6379/?protocol=3")] + #[case::by_name("redis://cache:6379/?protocol=resp3")] + fn a_url_never_switches_the_protocol_from_resp2(#[case] url: &str) { + let target = Target::Url(RedisUrl { + url: SecretValue::new(url), + tls: None, + db: None, + username: None, + password: None, + }); + + let info = connection_info(&target, &Credentials::NONE, true).unwrap(); + + assert_eq!(info.redis_settings().protocol(), ProtocolVersion::RESP2); } } diff --git a/litellm-rust/crates/redis-client/src/connection.rs b/litellm-rust/crates/redis-client/src/connection.rs index 41729bfd107..c1f82490b3b 100644 --- a/litellm-rust/crates/redis-client/src/connection.rs +++ b/litellm-rust/crates/redis-client/src/connection.rs @@ -1,53 +1,161 @@ -use std::time::Duration; - -use redis::{ - Cmd, ConnectionInfo, Pipeline, RedisFuture, RedisResult, Value, - aio::{ConnectionLike, ConnectionManager, ConnectionManagerConfig}, - cluster::{ClusterClientBuilder, ClusterConfig}, - cluster_async::ClusterConnection, +use std::{ + sync::{ + Arc, Mutex, PoisonError, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, }; -use crate::ClusterEndpoint; +use redis::{ + AsyncConnectionConfig, Cmd, ConnectionAddr, ConnectionInfo, ErrorKind, Pipeline, + RedisConnectionInfo, RedisError, RedisFuture, RedisResult, ServerErrorKind, TlsCertificates, + TlsMode, Value, + aio::{ConnectionLike, ConnectionManager, ConnectionManagerConfig, MultiplexedConnection}, + cluster::{ClusterClientBuilder, ClusterConfig}, + cluster_async::ClusterConnection, + cluster_routing::{MultipleNodeRoutingInfo, ResponsePolicy, RoutingInfo}, + sentinel::{SentinelClient, SentinelClientBuilder, SentinelServerType}, +}; -const RETRIES: u32 = 3; +use crate::{ + Cluster, ConnectError, Credentials, Sentinel, Target, Timeouts, + config::{self, addr}, + error::Deployment, +}; + +// redis-py 5.3.1 redis/asyncio/connection.py: without retry options a connection gets Retry(NoBackoff(), 0) +const RECONNECT_RETRIES: usize = 0; +// redis-py 5.3.1 redis/asyncio/cluster.py: cluster_error_retry_attempts=3 +const CLUSTER_RETRIES: u32 = 3; +// redis-py 5.3.1 redis/asyncio/cluster.py sleeps a flat 0.25s before retrying after CLUSTERDOWN. +// redis-rs draws the wait from [min, max), so a 1ms window pins it to 250ms +const CLUSTER_RETRY_WAIT_MS: u64 = 250; #[derive(Clone)] pub(crate) enum Connection { - Standalone(ConnectionManager), - Cluster(ClusterConnection), + Standalone(Renewing), + Cluster(Renewing), + Sentinel(SentinelConnection), } impl Connection { - pub(crate) fn standalone(info: ConnectionInfo, timeout: Duration) -> RedisResult { - let config = ConnectionManagerConfig::new() - .set_connection_timeout(Some(timeout)) - .set_response_timeout(Some(timeout)) - .set_number_of_retries(RETRIES as usize); - let manager = ConnectionManager::new_lazy_with_config(redis::Client::open(info)?, config)?; - Ok(Self::Standalone(manager)) + pub(crate) fn new( + target: &Target, + credentials: &Credentials, + timeouts: Timeouts, + keepalive: bool, + ) -> Result { + match target { + Target::Url(_) | Target::Standalone(_) | Target::Unix(_) => Self::standalone( + config::connection_info(target, credentials, keepalive)?, + credentials, + timeouts, + ), + Target::Cluster(cluster) => Self::cluster(cluster, credentials, timeouts, keepalive), + Target::Sentinel(sentinel) => Ok(Self::Sentinel(SentinelConnection::new( + sentinel, + credentials, + timeouts, + keepalive, + )?)), + } } - pub(crate) fn cluster(endpoint: &ClusterEndpoint, timeout: Duration) -> RedisResult { - let builder = ClusterClientBuilder::new(endpoint.addrs()) - .connection_timeout(timeout) - .response_timeout(timeout) - .retries(RETRIES); - let builder = match &endpoint.username { - Some(username) => builder.username(username.expose()), + pub(crate) fn deployment(&self) -> Deployment { + match self { + Self::Standalone(_) => Deployment::Standalone, + Self::Cluster(_) => Deployment::Cluster, + Self::Sentinel(_) => Deployment::Sentinel, + } + } + + fn standalone( + info: ConnectionInfo, + credentials: &Credentials, + timeouts: Timeouts, + ) -> Result { + let config = ConnectionManagerConfig::new() + .set_connection_timeout(Some(timeouts.connect)) + .set_response_timeout(Some(timeouts.response)) + .set_number_of_retries(RECONNECT_RETRIES); + let config = match credentials { + Credentials::Provider(provider) => config.set_credentials_provider(provider.clone()), + Credentials::Static { .. } => config, + }; + let client = redis::Client::open(info)?; + Ok(Self::Standalone(Renewing::new(move || { + ConnectionManager::new_lazy_with_config(client.clone(), config.clone()) + })?)) + } + + fn cluster( + endpoint: &Cluster, + credentials: &Credentials, + timeouts: Timeouts, + keepalive: bool, + ) -> Result { + let builder = ClusterClientBuilder::new(config::cluster_addrs(endpoint)) + .tcp_settings(config::tcp_settings(keepalive)) + .connection_timeout(timeouts.connect) + .response_timeout(timeouts.response) + .retries(CLUSTER_RETRIES) + .retry_wait_formula(0, 2) + .min_retry_wait(CLUSTER_RETRY_WAIT_MS) + .max_retry_wait(CLUSTER_RETRY_WAIT_MS + 1); + let builder = match &endpoint.tls { + Some(tls) => builder + .tls(config::tls_mode(tls)) + .certs(config::tls_certificates(tls)?) + .danger_accept_invalid_hostnames(!tls.check_hostname), None => builder, }; - let builder = match &endpoint.password { - Some(password) => builder.password(password.expose()), - None => builder, + let builder = match credentials { + Credentials::Static { username, password } => { + let builder = match username { + Some(username) => builder.username(username.expose()), + None => builder, + }; + match password { + Some(password) => builder.password(password.expose()), + None => builder, + } + } + Credentials::Provider(provider) => builder.set_credentials_provider(provider.clone()), }; - let config = ClusterConfig::new() - .set_connection_timeout(timeout) - .set_response_timeout(timeout); - Ok(Self::Cluster( - builder - .build()? - .get_pending_async_connection_with_config(config), - )) + // The builder's response timeout bounds the whole request, but with_config overwrites each node's + // response timeout with this one even when unset, so it has to be repeated here + let config = ClusterConfig::new().set_response_timeout(timeouts.response); + let client = builder.build()?; + Ok(Self::Cluster(Renewing::new(move || { + Ok(client.get_pending_async_connection_with_config(config.clone())) + })?)) + } +} + +impl Connection { + /// `PING` on the one multiplexed connection, or on every cluster node, so a dead socket is + /// replaced before a real command uses it. + pub(crate) async fn ping_every_node(&self) -> RedisResult<()> { + let ping = redis::cmd("PING"); + match self { + Self::Cluster(renewing) => { + let (generation, mut connection) = renewing.current(); + let reply = connection + .route_command( + ping, + RoutingInfo::MultiNode(( + MultipleNodeRoutingInfo::AllNodes, + Some(ResponsePolicy::AllSucceeded), + )), + ) + .await; + renewing.settle(generation, &reply); + reply.map(drop) + } + Self::Standalone(_) | Self::Sentinel(_) => { + ping.query_async::(&mut self.clone()).await.map(drop) + } + } } } @@ -56,6 +164,7 @@ impl ConnectionLike for Connection { match self { Self::Standalone(connection) => connection.req_packed_command(cmd), Self::Cluster(connection) => connection.req_packed_command(cmd), + Self::Sentinel(connection) => connection.req_packed_command(cmd), } } @@ -68,6 +177,7 @@ impl ConnectionLike for Connection { match self { Self::Standalone(connection) => connection.req_packed_commands(pipeline, offset, count), Self::Cluster(connection) => connection.req_packed_commands(pipeline, offset, count), + Self::Sentinel(connection) => connection.req_packed_commands(pipeline, offset, count), } } @@ -75,6 +185,441 @@ impl ConnectionLike for Connection { match self { Self::Standalone(connection) => connection.get_db(), Self::Cluster(connection) => connection.get_db(), + Self::Sentinel(connection) => connection.get_db(), } } } + +/// A connection swapped for a fresh one once a command on it times out, like redis-py disconnecting +/// a connection whose read timed out. redis-rs keeps the socket after a timeout and the cluster +/// client retries on it, so a socket that went silent would time out every command until TCP gives +/// up on it. +#[derive(Clone)] +pub(crate) struct Renewing(Arc>); + +type Open = Box RedisResult + Send + Sync>; + +struct RenewingState { + open: Open, + current: Mutex<(u64, C)>, +} + +impl Renewing { + fn new(open: impl Fn() -> RedisResult + Send + Sync + 'static) -> RedisResult { + let first = open()?; + Ok(Self(Arc::new(RenewingState { + open: Box::new(open), + current: Mutex::new((0, first)), + }))) + } + + fn current(&self) -> (u64, C) { + self.0 + .current + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + /// Replaces the connection of `generation` when its reply timed out, unless another timeout + /// already replaced it. + fn settle(&self, generation: u64, reply: &RedisResult) { + if !reply.as_ref().is_err_and(RedisError::is_timeout) { + return; + } + let mut current = self + .0 + .current + .lock() + .unwrap_or_else(PoisonError::into_inner); + if current.0 != generation { + return; + } + match (self.0.open)() { + Ok(fresh) => *current = (generation + 1, fresh), + Err(error) => { + tracing::warn!(%error, "replacing a timed-out Redis connection failed") + } + } + } +} + +impl ConnectionLike for Renewing { + fn req_packed_command<'a>(&'a mut self, cmd: &'a Cmd) -> RedisFuture<'a, Value> { + Box::pin(async move { + let (generation, mut connection) = self.current(); + let reply = connection.req_packed_command(cmd).await; + self.settle(generation, &reply); + reply + }) + } + + fn req_packed_commands<'a>( + &'a mut self, + pipeline: &'a Pipeline, + offset: usize, + count: usize, + ) -> RedisFuture<'a, Vec> { + Box::pin(async move { + let (generation, mut connection) = self.current(); + let replies = connection + .req_packed_commands(pipeline, offset, count) + .await; + self.settle(generation, &replies); + replies + }) + } + + fn get_db(&self) -> i64 { + self.current().1.get_db() + } +} + +#[derive(Clone)] +pub(crate) struct SentinelConnection(Arc); + +struct SentinelState { + discovery: tokio::sync::Mutex<()>, + current: Mutex>, + generations: AtomicU64, + endpoint: Sentinel, + credentials: Credentials, + tls: Option<(TlsMode, TlsCertificates)>, + config: AsyncConnectionConfig, + connect_timeout: Duration, + keepalive: bool, +} + +impl SentinelConnection { + fn new( + endpoint: &Sentinel, + credentials: &Credentials, + timeouts: Timeouts, + keepalive: bool, + ) -> Result { + let tls = match &endpoint.tls { + Some(tls) => Some((config::tls_mode(tls), config::tls_certificates(tls)?)), + None => None, + }; + let state = SentinelState { + discovery: tokio::sync::Mutex::new(()), + current: Mutex::new(None), + generations: AtomicU64::new(0), + endpoint: endpoint.clone(), + credentials: credentials.clone(), + tls, + config: async_config(credentials, timeouts), + connect_timeout: timeouts.connect, + keepalive, + }; + state.sentinel_client(&RedisConnectionInfo::default())?; + Ok(Self(Arc::new(state))) + } + + fn cached(&self) -> Option<(u64, MultiplexedConnection)> { + self.0 + .current + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone() + } + + async fn connection(&self) -> RedisResult<(u64, MultiplexedConnection)> { + if let Some(current) = self.cached() { + return Ok(current); + } + let _discovering = self.0.discovery.lock().await; + if let Some(current) = self.cached() { + return Ok(current); + } + let connection = self + .discover() + .await? + .get_multiplexed_async_connection_with_config(&self.0.config) + .await?; + let mut current = self + .0 + .current + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let generation = self.0.generations.fetch_add(1, Ordering::Relaxed); + *current = Some((generation, connection.clone())); + Ok((generation, connection)) + } + + pub(crate) async fn master(&self) -> RedisResult { + self.discover().await + } + + async fn discover(&self) -> RedisResult { + let master = tokio::time::timeout(self.0.connect_timeout, self.ask_sentinels()) + .await + .map_err(|_| timed_out("asking the Sentinels for the master timed out"))??; + self.master_client(master.get_connection_info().addr()) + } + + /// redis-rs sends `ROLE` to the master a Sentinel names before trusting it, so that probe gets + /// the data credentials and TLS a connection opened now would use. The Sentinel client is built + /// per discovery since a provider's credentials change after `new`. + async fn ask_sentinels(&self) -> RedisResult { + let data_auth = + config::authenticated(RedisConnectionInfo::default(), &self.0.credentials).await?; + self.0.sentinel_client(&data_auth)?.async_get_client().await + } + + fn master_client(&self, master: &ConnectionAddr) -> Result { + let (host, port) = match master { + ConnectionAddr::Tcp(host, port) | ConnectionAddr::TcpTls { host, port, .. } => { + (host.as_str(), *port) + } + _ => { + return Err(RedisError::from(( + ErrorKind::InvalidClientConfig, + "Sentinel named a master that is not a TCP address", + ))); + } + }; + let endpoint = Target::Standalone(crate::Standalone { + host: host.to_owned(), + port, + tls: self.0.endpoint.tls.clone(), + db: self.0.endpoint.db, + }); + let info = config::connection_info(&endpoint, &self.0.credentials, self.0.keepalive) + .map_err(|error| match error { + ConnectError::Redis(error) => error, + ConnectError::TlsFile { path, .. } => RedisError::from(( + ErrorKind::InvalidClientConfig, + "reading a Redis TLS file", + path, + )), + ConnectError::TlsKey { path, reason } => RedisError::from(( + ErrorKind::InvalidClientConfig, + "reading the Redis TLS key", + format!("{path}: {reason}"), + )), + })?; + redis::Client::open(info) + } + + fn forget(&self, generation: u64) { + let mut current = self + .0 + .current + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if current + .as_ref() + .is_some_and(|(cached, _)| *cached == generation) + { + *current = None; + } + } +} + +impl SentinelState { + fn sentinel_client(&self, data_auth: &RedisConnectionInfo) -> RedisResult { + let endpoint = &self.endpoint; + let sentinels = endpoint + .monitors + .iter() + .map(|node| addr(&node.host, node.port, endpoint.tls.as_ref())); + let builder = SentinelClientBuilder::new( + sentinels, + &endpoint.service_name, + SentinelServerType::Master, + )?; + let (username, password) = match &endpoint.monitor_auth { + Some(auth) => (auth.username.as_ref(), auth.password.as_ref()), + None => (None, None), + }; + let builder = match username { + Some(username) => builder.set_client_to_sentinel_username(username.expose()), + None => builder, + }; + let builder = match password { + Some(password) => builder.set_client_to_sentinel_password(password.expose()), + None => builder, + }; + let builder = match data_auth.username() { + Some(username) => builder.set_client_to_redis_username(username), + None => builder, + }; + let builder = match data_auth.password() { + Some(password) => builder.set_client_to_redis_password(password), + None => builder, + }; + let builder = match &self.tls { + Some((mode, certificates)) => builder + .set_client_to_sentinel_tls_mode(*mode) + .set_client_to_sentinel_certificates(certificates.clone()) + .set_client_to_redis_tls_mode(*mode) + .set_client_to_redis_certificates(certificates.clone()), + None => builder, + }; + builder.build() + } +} + +pub(crate) fn async_config(credentials: &Credentials, timeouts: Timeouts) -> AsyncConnectionConfig { + let config = AsyncConnectionConfig::new() + .set_connection_timeout(Some(timeouts.connect)) + .set_response_timeout(Some(timeouts.response)); + match credentials { + Credentials::Provider(provider) => config.set_credentials_provider(provider.clone()), + Credentials::Static { .. } => config, + } +} + +fn timed_out(message: &'static str) -> RedisError { + RedisError::from(std::io::Error::new(std::io::ErrorKind::TimedOut, message)) +} + +fn needs_rediscovery(error: &RedisError) -> bool { + matches!( + crate::Error::classify(error.clone(), Deployment::Sentinel), + crate::Error::Transport(_) + ) +} + +fn replied_read_only(values: &[Value]) -> bool { + values.iter().any(|value| { + matches!(value, Value::ServerError(error) if error.kind() == Some(ServerErrorKind::ReadOnly)) + }) +} + +impl ConnectionLike for SentinelConnection { + fn req_packed_command<'a>(&'a mut self, cmd: &'a Cmd) -> RedisFuture<'a, Value> { + Box::pin(async move { + let (generation, mut connection) = self.connection().await?; + let reply = connection.req_packed_command(cmd).await; + let rediscover = match &reply { + Ok(value) => replied_read_only(std::slice::from_ref(value)), + Err(error) => needs_rediscovery(error), + }; + if rediscover { + self.forget(generation); + } + reply + }) + } + + fn req_packed_commands<'a>( + &'a mut self, + pipeline: &'a Pipeline, + offset: usize, + count: usize, + ) -> RedisFuture<'a, Vec> { + Box::pin(async move { + let (generation, mut connection) = self.connection().await?; + let replies = connection + .req_packed_commands(pipeline, offset, count) + .await; + let rediscover = match &replies { + Ok(values) => replied_read_only(values), + Err(error) => needs_rediscovery(error), + }; + if rediscover { + self.forget(generation); + } + replies + }) + } + + fn get_db(&self) -> i64 { + self.0.endpoint.db + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Node; + use litellm_redis_testing::{RedisSentinel, SERVICE_NAME}; + + fn endpoint(server: &RedisSentinel) -> Sentinel { + let (host, port) = server.sentinel(); + Sentinel { + monitors: vec![Node { + host: host.to_owned(), + port, + }], + service_name: SERVICE_NAME.to_owned(), + monitor_auth: None, + tls: None, + db: 0, + } + } + + #[rstest::rstest] + #[tokio::test] + async fn a_late_failure_does_not_discard_the_reconnected_sentinel_master() { + let server = RedisSentinel::start().await.unwrap(); + let mut client = SentinelConnection::new( + &endpoint(&server), + &Credentials::NONE, + Timeouts::default(), + true, + ) + .unwrap(); + let (old_generation, _) = client.connection().await.unwrap(); + let original: i64 = redis::cmd("CLIENT") + .arg("ID") + .query_async(&mut client) + .await + .unwrap(); + client.forget(old_generation); + let replacement: i64 = redis::cmd("CLIENT") + .arg("ID") + .query_async(&mut client) + .await + .unwrap(); + assert_ne!(original, replacement); + + client.forget(old_generation); + let after_late_failure: i64 = redis::cmd("CLIENT") + .arg("ID") + .query_async(&mut client) + .await + .unwrap(); + + assert_eq!(replacement, after_late_failure); + } + + #[rstest::rstest] + #[tokio::test] + async fn a_read_only_reply_to_one_command_rediscovers_the_master() { + let server = RedisSentinel::start().await.unwrap(); + let mut client = SentinelConnection::new( + &endpoint(&server), + &Credentials::NONE, + Timeouts::default(), + true, + ) + .unwrap(); + let replica = redis::Client::open(format!("redis://127.0.0.1:{}", server.replica_port())) + .unwrap() + .get_multiplexed_async_connection() + .await + .unwrap(); + let stale = client.0.generations.fetch_add(1, Ordering::Relaxed); + *client.0.current.lock().unwrap() = Some((stale, replica)); + + let on_replica = redis::cmd("SET") + .arg("k") + .arg("1") + .exec_async(&mut client) + .await; + let on_master = redis::cmd("SET") + .arg("k") + .arg("1") + .exec_async(&mut client) + .await; + + assert_eq!( + on_replica.unwrap_err().kind(), + ErrorKind::Server(ServerErrorKind::ReadOnly) + ); + on_master.unwrap(); + } +} diff --git a/litellm-rust/crates/redis-client/src/error.rs b/litellm-rust/crates/redis-client/src/error.rs index ed3b2c74004..54e55615e60 100644 --- a/litellm-rust/crates/redis-client/src/error.rs +++ b/litellm-rust/crates/redis-client/src/error.rs @@ -13,32 +13,93 @@ pub enum Error { } #[derive(Debug, thiserror::Error)] -#[error("connecting to Redis")] -pub struct ConnectError(#[from] RedisError); +pub enum ConnectError { + #[error("connecting to Redis")] + Redis(#[from] RedisError), + #[error("reading the Redis TLS file {path}")] + TlsFile { + path: String, + #[source] + source: std::io::Error, + }, + #[error("reading the Redis TLS key {path}")] + TlsKey { + path: String, + #[source] + reason: KeyError, + }, +} -impl From for Error { - fn from(error: RedisError) -> Self { - match error.kind() { - ErrorKind::Server( - ServerErrorKind::BusyLoading - | ServerErrorKind::ClusterDown - | ServerErrorKind::MasterDown - | ServerErrorKind::TryAgain - | ServerErrorKind::ReadOnly, - ) => Self::Transport(error), - ErrorKind::Server(_) | ErrorKind::Extension => Self::Server(error), - ErrorKind::Parse | ErrorKind::UnexpectedReturnType => Self::Decode(error), +#[derive(Debug, thiserror::Error)] +pub enum KeyError { + #[error("the key is encrypted, so ssl_password is required")] + MissingPassword, + #[error("ssl_password does not decrypt the key, or the key uses an unsupported cipher")] + Decrypt(#[source] pkcs8::Error), + #[error( + "the key uses legacy OpenSSL PEM encryption, which Rust cannot read. Convert it to PKCS#8 with `openssl pkcs8 -topk8`" + )] + LegacyEncryption, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Deployment { + Standalone, + Cluster, + Sentinel, +} + +// redis-py 5.3.1 redis/_parsers/base.py EXCEPTION_CLASSES["ERR"] entries that subclass ConnectionError +const CONNECTION_ERR_MESSAGES: &[&str] = &[ + "max number of clients reached", + "invalid password", + "Client sent AUTH, but no password is set", + "AUTH called without any password configured for the default user. Are you sure your configuration is correct?", +]; + +impl Error { + pub(crate) fn classify(error: RedisError, deployment: Deployment) -> Self { + match (error.kind(), deployment) { + (ErrorKind::Parse | ErrorKind::UnexpectedReturnType, _) => Self::Decode(error), + (ErrorKind::Server(kind), _) if is_unavailable(kind, deployment) => { + Self::Transport(error) + } + (ErrorKind::Server(ServerErrorKind::ResponseError), _) + if error + .detail() + .is_some_and(|detail| CONNECTION_ERR_MESSAGES.contains(&detail)) => + { + Self::Transport(error) + } + (ErrorKind::Extension, _) if matches!(error.code(), Some("NOAUTH" | "WRONGPASS")) => { + Self::Transport(error) + } + (ErrorKind::Server(_) | ErrorKind::Extension | ErrorKind::InvalidClientConfig, _) => { + Self::Server(error) + } _ => Self::Transport(error), } } } +fn is_unavailable(kind: ServerErrorKind, deployment: Deployment) -> bool { + matches!( + (kind, deployment), + (ServerErrorKind::BusyLoading, _) + | ( + ServerErrorKind::ClusterDown | ServerErrorKind::MasterDown, + Deployment::Cluster + ) + | (ServerErrorKind::ReadOnly, Deployment::Sentinel) + ) +} + #[cfg(test)] mod tests { use redis::{RedisError, Value}; use rstest::rstest; - use super::Error; + use super::{Deployment, Error}; fn reply(resp: &str) -> RedisError { match redis::parse_redis_value(resp.as_bytes()).unwrap() { @@ -47,23 +108,101 @@ mod tests { } } + fn is_transport(resp: &str, deployment: Deployment) -> bool { + match Error::classify(reply(resp), deployment) { + Error::Transport(_) => true, + Error::Server(_) => false, + other => panic!("{resp:?} classified as {other:?}"), + } + } + #[rstest] #[case::wrong_type("-WRONGTYPE Operation against a key holding the wrong kind of value\r\n")] #[case::no_script("-NOSCRIPT No matching script\r\n")] #[case::cross_slot("-CROSSSLOT Keys in request don't hash to the same slot\r\n")] #[case::generic("-ERR value is not an integer or out of range\r\n")] #[case::no_perm("-NOPERM this user has no permissions to access one of the keys\r\n")] - fn rejections_are_server_errors(#[case] resp: &str) { - assert!(matches!(Error::from(reply(resp)), Error::Server(_))); + #[case::try_again("-TRYAGAIN Multiple keys request during rehashing of slot\r\n")] + #[case::out_of_memory("-OOM command not allowed when used memory > 'maxmemory'.\r\n")] + #[case::auth_arity("-ERR wrong number of arguments for 'auth' command\r\n")] + #[case::max_clients_with_suffix("-ERR max number of clients reached, try later\r\n")] + fn rejections_are_server_errors_on_every_deployment( + #[case] resp: &str, + #[values(Deployment::Standalone, Deployment::Cluster, Deployment::Sentinel)] + deployment: Deployment, + ) { + assert!(!is_transport(resp, deployment)); } #[rstest] #[case::loading("-LOADING Redis is loading the dataset in memory\r\n")] - #[case::cluster_down("-CLUSTERDOWN The cluster is down\r\n")] - #[case::master_down("-MASTERDOWN Link with MASTER is down\r\n")] - #[case::try_again("-TRYAGAIN Multiple keys request during rehashing of slot\r\n")] - #[case::read_only("-READONLY You can't write against a read only replica.\r\n")] - fn unavailable_nodes_are_transport_errors(#[case] resp: &str) { - assert!(matches!(Error::from(reply(resp)), Error::Transport(_))); + #[case::no_auth("-NOAUTH Authentication required.\r\n")] + #[case::wrong_pass("-WRONGPASS invalid username-password pair or user is disabled.\r\n")] + #[case::max_clients("-ERR max number of clients reached\r\n")] + #[case::invalid_password("-ERR invalid password\r\n")] + #[case::no_password_set("-ERR Client sent AUTH, but no password is set\r\n")] + #[case::no_default_password( + "-ERR AUTH called without any password configured for the default user. Are you sure your configuration is correct?\r\n" + )] + fn unavailable_nodes_are_transport_errors_on_every_deployment( + #[case] resp: &str, + #[values(Deployment::Standalone, Deployment::Cluster, Deployment::Sentinel)] + deployment: Deployment, + ) { + assert!(is_transport(resp, deployment)); + } + + #[rstest] + #[case::master_down_on_standalone( + "-MASTERDOWN Link with MASTER is down\r\n", + Deployment::Standalone, + false + )] + #[case::master_down_on_cluster( + "-MASTERDOWN Link with MASTER is down\r\n", + Deployment::Cluster, + true + )] + #[case::cluster_down_on_standalone( + "-CLUSTERDOWN The cluster is down\r\n", + Deployment::Standalone, + false + )] + #[case::cluster_down_on_cluster( + "-CLUSTERDOWN The cluster is down\r\n", + Deployment::Cluster, + true + )] + #[case::read_only_on_standalone( + "-READONLY You can't write against a read only replica.\r\n", + Deployment::Standalone, + false + )] + #[case::read_only_on_cluster( + "-READONLY You can't write against a read only replica.\r\n", + Deployment::Cluster, + false + )] + #[case::master_down_on_sentinel( + "-MASTERDOWN Link with MASTER is down\r\n", + Deployment::Sentinel, + false + )] + #[case::cluster_down_on_sentinel( + "-CLUSTERDOWN The cluster is down\r\n", + Deployment::Sentinel, + false + )] + #[case::read_only_on_sentinel( + "-READONLY You can't write against a read only replica.\r\n", + Deployment::Sentinel, + true + )] + fn topology_decides_like_redis_py( + #[case] resp: &str, + #[case] deployment: Deployment, + #[case] transport: bool, + ) { + assert_eq!(is_transport(resp, deployment), transport); } } diff --git a/litellm-rust/crates/redis-client/src/events.rs b/litellm-rust/crates/redis-client/src/events.rs new file mode 100644 index 00000000000..9ea9ff4defa --- /dev/null +++ b/litellm-rust/crates/redis-client/src/events.rs @@ -0,0 +1,78 @@ +use std::{any::Any, future::Future, sync::Arc, time::Duration}; + +use litellm_circuit_breaker::{State, Transition}; + +use crate::Error; + +/// What the host attached to the calls made inside `tagged`, opaque to the client. +pub type CallTag = Arc; + +tokio::task_local! { + static TAG: CallTag; +} + +pub async fn tagged(tag: CallTag, operation: F) -> F::Output { + TAG.scope(tag, operation).await +} + +/// The tag of the `tagged` scope this task runs in. `RedisEvents::call` runs on the calling task, +/// so a sink reads its call's tag here. +pub fn current_tag() -> Option { + TAG.try_with(Arc::clone).ok() +} + +pub trait RedisEvents: Send + Sync { + fn call(&self, event: &CallEvent); + + fn breaker(&self, transition: &Transition); +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Operation { + Command(&'static str), + Script, + Pipeline { ops: usize }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Outcome { + Succeeded, + Failed(FailureKind), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FailureKind { + Timeout, + Transport, + Server, + Decode, + BreakerOpen, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CallEvent { + pub operation: Operation, + pub duration: Duration, + pub outcome: Outcome, +} + +impl FailureKind { + pub fn of(error: &Error) -> Self { + match error { + Error::Transport(error) if error.is_timeout() => Self::Timeout, + Error::Transport(_) => Self::Transport, + Error::Server(_) => Self::Server, + Error::Decode(_) => Self::Decode, + Error::BreakerOpen => Self::BreakerOpen, + } + } +} + +impl Outcome { + pub(crate) fn of(result: &Result) -> Self { + match result { + Ok(_) => Self::Succeeded, + Err(error) => Self::Failed(FailureKind::of(error)), + } + } +} diff --git a/litellm-rust/crates/redis-client/src/lib.rs b/litellm-rust/crates/redis-client/src/lib.rs index 47ff8677e06..2754666c329 100644 --- a/litellm-rust/crates/redis-client/src/lib.rs +++ b/litellm-rust/crates/redis-client/src/lib.rs @@ -1,20 +1,30 @@ +mod batch; mod client; -mod commands; +pub mod commands; mod config; mod connection; mod error; +mod events; mod namespace; mod pipeline; +mod pubsub; mod script; -#[cfg(test)] -mod test_support; -pub use client::{RedisClient, Ttl}; -pub use config::{ - ClusterEndpoint, ClusterNode, DEFAULT_TIMEOUT, Endpoint, HostEndpoint, RedisConfig, +pub use batch::{Declared, RedisBatch}; +pub use client::{RedisClient, Ttl, WeakRedisClient}; +pub use commands::{Amount, Command, Expiry}; +pub use config::{Credentials, CredentialsProvider, DEFAULT_TIMEOUT, RedisConfig, Timeouts}; +pub use error::{ConnectError, Error, KeyError}; +pub use events::{ + CallEvent, CallTag, FailureKind, Operation, Outcome, RedisEvents, current_tag, tagged, +}; +pub use litellm_redis_types::{ + Cluster, Node, RedisUrl, Sentinel, Standalone, StaticAuth, Target, TlsOptions, UnixSocket, }; -pub use error::{ConnectError, Error}; pub use namespace::Namespace; pub use pipeline::Pipeline; -pub use redis::Value; +pub use redis::{ + BasicAuth, ErrorKind as RedisErrorKind, RedisError, StreamingCredentialsProvider, Value, + cluster_routing::Slot, +}; pub use script::Script; diff --git a/litellm-rust/crates/redis-client/src/pipeline.rs b/litellm-rust/crates/redis-client/src/pipeline.rs index e54c65e9605..0b6bb6a6673 100644 --- a/litellm-rust/crates/redis-client/src/pipeline.rs +++ b/litellm-rust/crates/redis-client/src/pipeline.rs @@ -1,30 +1,20 @@ -use std::{collections::HashMap, time::Duration}; +use std::collections::HashMap; use futures_util::future::join_all; -use redis::{ - Cmd, ErrorKind, RedisResult, ServerErrorKind, ToRedisArgs, ToSingleRedisArg, Value, - cluster_routing::Slot, -}; +use redis::{ErrorKind, RedisError, RedisResult, ServerErrorKind, Value, cluster_routing::Slot}; -use crate::{Error, Namespace, Script, commands, connection::Connection}; +use crate::{ + Command, Error, Namespace, + commands::{Op, Plan}, + connection::Connection, + script::ScriptCall, +}; pub struct Pipeline { namespace: Namespace, ops: Vec, } -struct Op { - command: Cmd, - key: String, - script: Option, -} - -struct ScriptCall { - script: Script, - keys: Vec, - args: Vec>, -} - type Settled = (usize, Result); impl Pipeline { @@ -35,60 +25,13 @@ impl Pipeline { } } - pub fn get(self, key: &str) -> Self { - let key = self.namespace.apply(key); - self.command(commands::get(key.clone()), key) + pub(crate) fn of(namespace: Namespace, ops: Vec) -> Self { + Self { namespace, ops } } - pub fn set(self, key: &str, value: impl ToSingleRedisArg, ttl: Duration) -> Self { - let key = self.namespace.apply(key); - self.command(commands::set(key.clone(), value, ttl), key) - } - - pub fn del(self, key: &str) -> Self { - let key = self.namespace.apply(key); - self.command(commands::del(vec![key.clone()]), key) - } - - pub fn expire(self, key: &str, ttl: Duration) -> Self { - let key = self.namespace.apply(key); - self.command(commands::expire(key.clone(), ttl), key) - } - - pub fn incr_by(self, key: &str, delta: i64) -> Self { - let key = self.namespace.apply(key); - self.command(commands::incr_by(key.clone(), delta), key) - } - - pub fn incr_by_float(self, key: &str, delta: f64) -> Self { - let key = self.namespace.apply(key); - self.command(commands::incr_by_float(key.clone(), delta), key) - } - - pub fn eval(self, script: &Script, keys: &[&str], args: impl ToRedisArgs) -> Self { - let keys: Vec = keys.iter().map(|key| self.namespace.apply(key)).collect(); - let args = args.to_redis_args(); - let op = Op { - command: script.evalsha(&keys, &args), - key: keys.first().cloned().unwrap_or_default(), - script: Some(ScriptCall { - script: script.clone(), - keys, - args, - }), - }; - self.push(op) - } - - fn command(self, command: Cmd, key: String) -> Self { - self.push(Op { - command, - key, - script: None, - }) - } - - fn push(mut self, op: Op) -> Self { + /// Appends one command in its pipelined form; `RedisClient::run` answers it in order, undecoded. + pub fn with(mut self, command: Command) -> Self { + let (op, _) = command.prepare(&self.namespace); self.ops.push(op); self } @@ -102,7 +45,7 @@ impl Pipeline { let settled = join_all( groups .iter() - .map(|group| self.run_group(connection.clone(), group)), + .map(|batches| self.run_in_order(connection, batches)), ) .await; let mut settled: Vec = settled.into_iter().flatten().collect(); @@ -110,183 +53,131 @@ impl Pipeline { settled.into_iter().map(|(_, result)| result).collect() } - fn groups(&self, connection: &Connection) -> Vec> { + async fn run_in_order(&self, connection: &Connection, batches: &[Vec]) -> Vec { + let mut settled = Vec::with_capacity(batches.iter().map(Vec::len).sum()); + for batch in batches { + settled.extend(self.run_group(connection.clone(), batch).await); + } + settled + } + + fn groups(&self, connection: &Connection) -> Vec>> { match connection { _ if self.ops.is_empty() => Vec::new(), - Connection::Standalone(_) => vec![(0..self.ops.len()).collect()], + Connection::Standalone(_) | Connection::Sentinel(_) => { + vec![vec![(0..self.ops.len()).collect()]] + } Connection::Cluster(_) => { let mut by_slot: HashMap> = HashMap::new(); - for (index, op) in self.ops.iter().enumerate() { + for index in 0..self.ops.len() { by_slot - .entry(Slot::for_key(&op.key)) + .entry(Slot::for_key(&self.ops[index].key)) .or_default() .push(index); } - by_slot.into_values().collect() + by_slot + .into_values() + .map(|indices| self.retry_safe_batches(indices)) + .collect() } } } + fn retry_safe_batches(&self, indices: Vec) -> Vec> { + indices + .chunk_by(|&left, &right| self.ops[left].idempotent && self.ops[right].idempotent) + .map(<[usize]>::to_vec) + .collect() + } + + /// A script the server had not cached ran nothing, so it alone runs again, the way litellm's + /// `RedisBatch` reruns it through redis-py's `Script`: `EVALSHA`, then `SCRIPT LOAD` and + /// `EVALSHA` on `NOSCRIPT`. + async fn settle_script( + &self, + connection: Connection, + index: usize, + reply: RedisResult, + ) -> (usize, RedisResult) { + match (&self.ops[index].plan, reply) { + (Plan::Script(call), Err(error)) if is_missing_script(&error) => { + (index, rerun_missing_script(connection, call).await) + } + (_, reply) => (index, reply), + } + } + async fn run_group(&self, mut connection: Connection, indices: &[usize]) -> Vec { + let deployment = connection.deployment(); let mut pipeline = redis::pipe(); pipeline.ignore_errors(); - for &index in indices { - pipeline.add_command(self.ops[index].command.clone()); - } - let replies: RedisResult>> = - pipeline.query_async(&mut connection).await; + let widths: Vec = indices + .iter() + .map(|&index| { + let commands = self.ops[index].plan.pipelined(); + let width = commands.len(); + for command in commands { + pipeline.add_command(command); + } + width + }) + .collect(); + let replies: RedisResult>> = match pipeline.is_empty() { + true => Ok(Vec::new()), + false => pipeline.query_async(&mut connection).await, + }; let replies = match replies { Ok(replies) => replies, Err(error) => { return indices .iter() - .map(|&index| (index, Err(Error::from(error.clone())))) + .map(|&index| (index, Err(Error::classify(error.clone(), deployment)))) .collect(); } }; - let mut settled = Vec::with_capacity(indices.len()); - for (&index, reply) in indices.iter().zip(replies) { - settled.push((index, self.settle(&mut connection, index, reply).await)); - } - settled - } - - async fn settle( - &self, - connection: &mut Connection, - index: usize, - reply: RedisResult, - ) -> Result { - match (reply, &self.ops[index].script) { - (Err(error), Some(call)) - if error.kind() == ErrorKind::Server(ServerErrorKind::NoScript) => - { - Ok(call - .script - .invocation(&call.keys, &call.args) - .invoke_async(connection) - .await?) - } - (reply, _) => Ok(reply?), - } + let mut replies = replies.into_iter(); + let per_op: Vec<(usize, RedisResult)> = indices + .iter() + .zip(widths) + .map(|(&index, width)| match &self.ops[index].plan { + Plan::Reply(value) => (index, Ok(value.clone())), + _ => (index, first_reply(replies.by_ref().take(width))), + }) + .collect(); + join_all( + per_op + .into_iter() + .map(|(index, reply)| self.settle_script(connection.clone(), index, reply)), + ) + .await + .into_iter() + .map(|(index, reply)| { + ( + index, + reply.map_err(|error| Error::classify(error, deployment)), + ) + }) + .collect() } } -#[cfg(test)] -mod tests { - use std::time::{Duration, Instant}; - - use redis::Value; - use rstest::rstest; - - use crate::{ - DEFAULT_TIMEOUT, Error, Script, - test_support::{Server, Topology, flush_scripts}, - }; - - #[rstest] - #[tokio::test] - async fn each_op_settles_on_its_own_in_input_order( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", DEFAULT_TIMEOUT).await; - client - .set("text", "not a number", Duration::from_secs(60)) - .await - .unwrap(); - let pipeline = client - .pipeline() - .incr_by("a", 1) - .incr_by("text", 1) - .incr_by("c", 5) - .set("b", "2", Duration::from_secs(60)) - .get("b") - .incr_by("a", 1) - .get("missing"); - - let results = client.run(&pipeline).await; - - assert_eq!(results.len(), 7); - assert_eq!(results[0].as_ref().unwrap(), &Value::Int(1)); - assert!( - matches!(results[1], Err(Error::Server(_))), - "{:?}", - results[1] - ); - assert_eq!(results[2].as_ref().unwrap(), &Value::Int(5)); - assert_eq!(results[3].as_ref().unwrap(), &Value::Okay); - assert_eq!( - results[4].as_ref().unwrap(), - &Value::BulkString(b"2".to_vec()) - ); - assert_eq!(results[5].as_ref().unwrap(), &Value::Int(2)); - assert_eq!(results[6].as_ref().unwrap(), &Value::Nil); - } - - #[rstest] - #[tokio::test] - async fn a_flushed_script_in_a_pipeline_reruns_alone( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("ns", DEFAULT_TIMEOUT).await; - let incr = Script::new("return redis.call('INCRBY', KEYS[1], ARGV[1])"); - client.eval::(&incr, &["counter"], 1).await.unwrap(); - flush_scripts(&client).await; - let pipeline = client - .pipeline() - .eval(&incr, &["counter"], 2) - .incr_by("other", 7); - - let results = client.run(&pipeline).await; - - assert_eq!(results[0].as_ref().unwrap(), &Value::Int(3)); - assert_eq!(results[1].as_ref().unwrap(), &Value::Int(7)); - assert_eq!(client.get::("counter").await.unwrap(), Some(3)); - } - - #[rstest] - #[tokio::test] - async fn an_unreachable_server_fails_every_op_as_transport( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", Duration::from_millis(500)).await; - let pipeline = client.pipeline().incr_by("a", 1).incr_by("c", 1); - - server.freeze().await; - let results = client.run(&pipeline).await; - server.thaw().await; - - assert_eq!(results.len(), 2); - assert!( - matches!(results[0], Err(Error::Transport(_))), - "{:?}", - results[0] - ); - assert!( - matches!(results[1], Err(Error::Transport(_))), - "{:?}", - results[1] - ); - } - - #[rstest] - #[tokio::test] - async fn an_empty_pipeline_returns_without_asking_redis( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", Duration::from_secs(2)).await; - - server.freeze().await; - let started = Instant::now(); - let results = client.run(&client.pipeline()).await; - let elapsed = started.elapsed(); - server.thaw().await; - - assert!(results.is_empty()); - assert!(elapsed < Duration::from_secs(1), "{elapsed:?}"); - } +async fn rerun_missing_script(mut connection: Connection, call: &ScriptCall) -> RedisResult { + call.script + .invocation(&call.keys, &call.args) + .invoke_async(&mut connection) + .await +} + +fn is_missing_script(error: &RedisError) -> bool { + error.kind() == ErrorKind::Server(ServerErrorKind::NoScript) +} + +// RedisBatch settles an op from its first reply and fails it on any reply that failed +fn first_reply(replies: impl Iterator>) -> RedisResult { + replies + .reduce(|first, next| match (first, next) { + (Err(error), _) | (Ok(_), Err(error)) => Err(error), + (first, Ok(_)) => first, + }) + .unwrap_or(Ok(Value::Nil)) } diff --git a/litellm-rust/crates/redis-client/src/pubsub.rs b/litellm-rust/crates/redis-client/src/pubsub.rs new file mode 100644 index 00000000000..b6b18d3e481 --- /dev/null +++ b/litellm-rust/crates/redis-client/src/pubsub.rs @@ -0,0 +1,225 @@ +use std::{sync::Arc, time::Duration}; + +use futures_util::{StreamExt, stream::BoxStream}; +use redis::{ConnectionInfo, RedisResult, aio::PubSubStream}; + +use crate::{ + ConnectError, Credentials, Standalone, Target, Timeouts, config, + connection::{Connection, SentinelConnection}, +}; + +// litellm/proxy/common_utils/auth_cache_invalidation_pubsub.py: _BACKOFF_INITIAL_SECONDS, _BACKOFF_MAX_SECONDS +const BACKOFF_INITIAL: Duration = Duration::from_secs(5); +const BACKOFF_MAX: Duration = Duration::from_secs(60); + +/// A subscription the way redis-py's `PubSub` holds one: a dedicated RESP2 connection that sends +/// `SUBSCRIBE`, authenticated once when it connects. +#[derive(Clone)] +pub(crate) struct Subscriber { + source: Source, + credentials: Credentials, + connect_timeout: Duration, +} + +#[derive(Clone)] +enum Source { + Nodes(Arc<[ConnectionInfo]>), + Sentinel(SentinelConnection), +} + +impl Subscriber { + pub(crate) fn new( + endpoint: &Target, + credentials: &Credentials, + timeouts: Timeouts, + keepalive: bool, + connection: &Connection, + ) -> Result { + let source = match (endpoint, connection) { + (_, Connection::Sentinel(sentinel)) => Source::Sentinel(sentinel.clone()), + (Target::Cluster(cluster), _) => Source::Nodes( + cluster + .nodes + .iter() + .map(|node| { + let node = Target::Standalone(Standalone { + host: node.host.clone(), + port: node.port, + tls: cluster.tls.clone(), + db: 0, + }); + config::connection_info(&node, credentials, keepalive) + }) + .collect::>()?, + ), + _ => Source::Nodes(Arc::new([config::connection_info( + endpoint, + credentials, + keepalive, + )?])), + }; + Ok(Self { + source, + credentials: credentials.clone(), + connect_timeout: timeouts.connect, + }) + } + + pub(crate) fn messages(self, channel: String) -> BoxStream<'static, Vec> { + futures_util::stream::unfold( + (self, channel, None::, BACKOFF_INITIAL), + |(subscriber, channel, live, backoff)| async move { + let mut live = live; + let mut backoff = backoff; + loop { + let Some(current) = live.as_mut() else { + match subscriber.subscribe(&channel).await { + Ok(subscribed) => { + live = Some(subscribed); + backoff = BACKOFF_INITIAL; + } + Err(error) => { + tracing::warn!(%error, ?backoff, "Redis subscribe failed, retrying"); + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(BACKOFF_MAX); + } + } + continue; + }; + match current.next().await { + Some(message) => { + let payload = message.get_payload_bytes().to_vec(); + return Some((payload, (subscriber, channel, live, backoff))); + } + None => live = None, + } + tracing::warn!(?backoff, "Redis subscription connection lost, resubscribing"); + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(BACKOFF_MAX); + } + }, + ) + .boxed() + } + + async fn subscribe(&self, channel: &str) -> RedisResult { + match &self.source { + Source::Sentinel(sentinel) => { + let master = sentinel.master().await?; + self.subscribe_on(master.get_connection_info().clone(), channel) + .await + } + Source::Nodes(nodes) => { + let mut last = None; + for node in nodes.iter() { + match self.subscribe_on(node.clone(), channel).await { + Ok(live) => return Ok(live), + Err(error) => last = Some(error), + } + } + Err(last.expect("an endpoint always has at least one node")) + } + } + } + + async fn subscribe_on(&self, info: ConnectionInfo, channel: &str) -> RedisResult { + let settings = + config::authenticated(info.redis_settings().clone(), &self.credentials).await?; + let client = redis::Client::open(info.set_redis_settings(settings))?; + let mut pubsub = tokio::time::timeout(self.connect_timeout, client.get_async_pubsub()) + .await + .map_err(|_| { + redis::RedisError::from(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "connecting the Redis subscription timed out", + )) + })??; + subscribe_connected(&mut pubsub, channel, self.connect_timeout).await?; + Ok(pubsub.into_on_message()) + } +} + +async fn subscribe_connected( + pubsub: &mut redis::aio::PubSub, + channel: &str, + connect_timeout: Duration, +) -> RedisResult<()> { + tokio::time::timeout(connect_timeout, pubsub.subscribe(channel)) + .await + .map_err(|_| { + redis::RedisError::from(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "subscribing to the Redis channel timed out", + )) + })? +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use futures_util::StreamExt; + use litellm_redis_testing::{RedisServer, TcpProxy}; + use rstest::rstest; + + use super::subscribe_connected; + + #[rstest] + #[tokio::test] + async fn a_stalled_pubsub_subscribe_times_out_and_a_fresh_pubsub_recovers() { + let server = RedisServer::start().await.unwrap(); + let proxy = TcpProxy::start(server.port()).await.unwrap(); + let client = redis::Client::open(format!("redis://127.0.0.1:{}", proxy.port())).unwrap(); + let mut pubsub = tokio::time::timeout(Duration::from_secs(2), client.get_async_pubsub()) + .await + .expect("connecting the initial pubsub should finish") + .unwrap(); + tokio::time::timeout( + Duration::from_secs(2), + subscribe_connected(&mut pubsub, "existing", Duration::from_secs(1)), + ) + .await + .expect("the initial subscription acknowledgement should arrive") + .unwrap(); + + proxy.blackhole_open_connections(); + let result = tokio::time::timeout( + Duration::from_secs(1), + subscribe_connected(&mut pubsub, "stalled", Duration::from_millis(100)), + ) + .await + .expect("the stalled subscription should return before the safety deadline"); + assert!( + matches!(&result, Err(error) if error.is_timeout()), + "{result:?}" + ); + drop(pubsub); + + let mut recovered = tokio::time::timeout(Duration::from_secs(2), client.get_async_pubsub()) + .await + .expect("a fresh pubsub connection should finish") + .unwrap(); + tokio::time::timeout( + Duration::from_secs(2), + subscribe_connected(&mut recovered, "recovered", Duration::from_secs(1)), + ) + .await + .expect("the fresh subscription acknowledgement should arrive") + .unwrap(); + + let publisher = redis::Client::open(server.url()).unwrap(); + let mut connection = publisher.get_multiplexed_async_connection().await.unwrap(); + let subscriber_count: i64 = redis::cmd("PUBLISH") + .arg("recovered") + .arg("payload") + .query_async(&mut connection) + .await + .unwrap(); + assert_eq!(subscriber_count, 1); + let message = tokio::time::timeout(Duration::from_secs(2), recovered.on_message().next()) + .await + .expect("the fresh subscription should receive the published payload") + .unwrap(); + assert_eq!(message.get_payload::().unwrap(), "payload"); + } +} diff --git a/litellm-rust/crates/redis-client/src/script.rs b/litellm-rust/crates/redis-client/src/script.rs index c5d73133ecc..6dbdbafe9c9 100644 --- a/litellm-rust/crates/redis-client/src/script.rs +++ b/litellm-rust/crates/redis-client/src/script.rs @@ -1,13 +1,18 @@ +use std::sync::Arc; + use redis::{Cmd, ScriptInvocation}; +/// A Lua script, cheap to clone into every call that runs it. #[derive(Clone)] -pub struct Script(redis::Script); +pub struct Script(Arc); impl Script { pub fn new(source: &str) -> Self { - Self(redis::Script::new(source)) + Self(Arc::new(redis::Script::new(source))) } + /// The `EVALSHA` a pipeline sends, like redis-py. A `NOSCRIPT` reply is settled by + /// [`Script::invocation`] alone, which loads the script and sends it again. pub(crate) fn evalsha(&self, keys: &[String], args: &[Vec]) -> Cmd { redis::cmd("EVALSHA") .arg(self.0.get_hash()) @@ -24,65 +29,10 @@ impl Script { } } -#[cfg(test)] -mod tests { - use std::time::Duration; - - use rstest::rstest; - - use super::Script; - use crate::{ - DEFAULT_TIMEOUT, Error, - test_support::{Server, Topology, flush_scripts}, - }; - - #[rstest] - #[tokio::test] - async fn script_keys_are_namespaced( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("a", DEFAULT_TIMEOUT).await; - let bare = server.client("", DEFAULT_TIMEOUT).await; - let read = Script::new("return redis.call('GET', KEYS[1])"); - bare.set("a:x", "1", Duration::from_secs(60)).await.unwrap(); - - let value: Option = client - .eval(&read, &["x"], Vec::::new()) - .await - .unwrap(); - - assert_eq!(value.as_deref(), Some("1")); - } - - #[rstest] - #[tokio::test] - async fn a_failing_script_is_a_server_error_and_keeps_the_connection( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", DEFAULT_TIMEOUT).await; - let incr = Script::new("return redis.call('INCRBY', KEYS[1], ARGV[1])"); - client.rpush("list", "item").await.unwrap(); - - let failed = client.eval::(&incr, &["list"], 1).await; - - assert!(matches!(failed, Err(Error::Server(_))), "{failed:?}"); - assert_eq!(client.eval::(&incr, &["counter"], 3).await.unwrap(), 3); - } - - #[rstest] - #[tokio::test] - async fn a_flushed_script_is_loaded_again( - #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, - ) { - let server = Server::start(topology).await; - let client = server.client("", DEFAULT_TIMEOUT).await; - let incr = Script::new("return redis.call('INCRBY', KEYS[1], ARGV[1])"); - client.eval::(&incr, &["counter"], 1).await.unwrap(); - - flush_scripts(&client).await; - - assert_eq!(client.eval::(&incr, &["counter"], 1).await.unwrap(), 2); - } +/// A script call in a pipeline, kept apart from plain commands because it is the one entry a +/// pipeline may send again. +pub(crate) struct ScriptCall { + pub(crate) script: Script, + pub(crate) keys: Vec, + pub(crate) args: Vec>, } diff --git a/litellm-rust/crates/redis-client/src/test_support.rs b/litellm-rust/crates/redis-client/src/test_support.rs deleted file mode 100644 index e503739a942..00000000000 --- a/litellm-rust/crates/redis-client/src/test_support.rs +++ /dev/null @@ -1,126 +0,0 @@ -use std::time::Duration; - -use litellm_circuit_breaker::Breaker; -use litellm_redis_testing::{RedisCluster, RedisServer}; -use litellm_secret_value::SecretValue; -use redis::cluster_routing::{Route, RoutingInfo, SingleNodeRoutingInfo, SlotAddr}; - -use crate::{ - ClusterEndpoint, ClusterNode, Endpoint, Namespace, RedisClient, RedisConfig, - connection::Connection, -}; - -#[derive(Clone, Copy, Debug)] -pub(crate) enum Topology { - Standalone, - Cluster, -} - -pub(crate) enum Server { - Standalone(RedisServer), - Cluster(RedisCluster), -} - -impl Server { - pub(crate) async fn start(topology: Topology) -> Self { - match topology { - Topology::Standalone => Self::Standalone(RedisServer::start().await.unwrap()), - Topology::Cluster => Self::Cluster(RedisCluster::start().await.unwrap()), - } - } - - pub(crate) fn endpoint(&self) -> Endpoint { - match self { - Self::Standalone(server) => Endpoint::Url(SecretValue::new(server.url().to_owned())), - Self::Cluster(cluster) => Endpoint::Cluster(ClusterEndpoint { - nodes: cluster - .nodes() - .map(|(host, port)| ClusterNode { - host: host.to_owned(), - port, - }) - .collect(), - username: None, - password: None, - tls: false, - }), - } - } - - pub(crate) async fn client(&self, namespace: &str, timeout: Duration) -> RedisClient { - self.guarded_client(namespace, timeout, None).await - } - - pub(crate) async fn guarded_client( - &self, - namespace: &str, - timeout: Duration, - breaker: Option, - ) -> RedisClient { - RedisClient::new(RedisConfig { - endpoint: self.endpoint(), - namespace: Namespace::new(namespace), - timeout, - breaker, - }) - .unwrap() - } - - pub(crate) async fn restart(&self) { - match self { - Self::Standalone(server) => server.restart().await.unwrap(), - Self::Cluster(cluster) => cluster.restart().await.unwrap(), - } - } - - pub(crate) async fn stop(&self) { - match self { - Self::Standalone(server) => server.stop().await.unwrap(), - Self::Cluster(cluster) => cluster.stop().await.unwrap(), - } - } - - pub(crate) async fn resume(&self) { - match self { - Self::Standalone(server) => server.resume().await.unwrap(), - Self::Cluster(cluster) => cluster.resume().await.unwrap(), - } - } - - pub(crate) async fn freeze(&self) { - match self { - Self::Standalone(server) => server.freeze().await.unwrap(), - Self::Cluster(cluster) => cluster.freeze().await.unwrap(), - } - } - - pub(crate) async fn thaw(&self) { - match self { - Self::Standalone(server) => server.thaw().await.unwrap(), - Self::Cluster(cluster) => cluster.thaw().await.unwrap(), - } - } -} - -pub(crate) async fn client_id(client: &RedisClient, key: &str) -> i64 { - let command = redis::cmd("CLIENT").arg("ID").to_owned(); - match client.connection.clone() { - Connection::Standalone(mut connection) => { - command.query_async(&mut connection).await.unwrap() - } - Connection::Cluster(mut connection) => { - let node = Route::with_key(key, SlotAddr::Master); - let routing = RoutingInfo::SingleNode(SingleNodeRoutingInfo::SpecificNode(node)); - redis::from_redis_value(connection.route_command(command, routing).await.unwrap()) - .unwrap() - } - } -} - -pub(crate) async fn flush_scripts(client: &RedisClient) { - redis::cmd("SCRIPT") - .arg("FLUSH") - .query_async::<()>(&mut client.connection.clone()) - .await - .unwrap(); -} diff --git a/litellm-rust/crates/redis-client/tests/batch.rs b/litellm-rust/crates/redis-client/tests/batch.rs new file mode 100644 index 00000000000..45ca419951e --- /dev/null +++ b/litellm-rust/crates/redis-client/tests/batch.rs @@ -0,0 +1,330 @@ +mod support; + +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; + +use litellm_circuit_breaker::{State, Transition}; +use litellm_redis_client::commands; +use litellm_redis_client::{ + Amount, CallEvent, Credentials, DEFAULT_TIMEOUT, Declared, Error, Expiry, Namespace, Operation, + RedisBatch, RedisClient, RedisConfig, RedisEvents, Script, Timeouts, Ttl, +}; +use litellm_redis_testing::{RedisDeployment, Topology}; +use rstest::rstest; +use support::{connect, endpoint}; + +const MINUTE: Option = Some(Expiry::KeepExisting(Duration::from_secs(60))); + +// RedisBatch pipelines a standalone flush and runs each op on its own on a cluster +fn expected_trips(topology: Topology, alone: &[Operation]) -> Vec { + match topology { + Topology::Standalone => vec![Operation::Pipeline { ops: alone.len() }], + Topology::Cluster => alone.to_vec(), + } +} + +#[derive(Default)] +struct RoundTrips(Mutex>); + +impl RoundTrips { + fn seen(&self) -> Vec { + self.0.lock().unwrap().clone() + } +} + +impl RedisEvents for RoundTrips { + fn call(&self, event: &CallEvent) { + self.0.lock().unwrap().push(event.operation); + } + + fn breaker(&self, _transition: &Transition) {} +} + +fn observed(server: &RedisDeployment) -> (RedisClient, Arc) { + let trips = Arc::new(RoundTrips::default()); + let client = RedisClient::new(RedisConfig { + target: endpoint(server), + credentials: Credentials::NONE, + namespace: Namespace::new("ns"), + timeouts: Timeouts::default(), + breaker: None, + events: Some(trips.clone()), + health_check: None, + keepalive: true, + }) + .unwrap(); + (client, trips) +} + +#[rstest] +#[tokio::test] +async fn declared_ops_wait_for_the_flush_and_share_one_round_trip( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let (client, trips) = observed(&server); + let batch = client.batch(); + let counter = batch.add(commands::incr_by_float("counter", Amount::Int(3), MINUTE)); + let total = batch.add(commands::incr_by_float("total", Amount::Float(1.5), MINUTE)); + let written = batch.add(commands::set("other", "v", Duration::from_secs(60))); + let read = batch.add(commands::get::("value")); + + let outside = connect(&server, "ns", DEFAULT_TIMEOUT); + assert_eq!( + outside.send(commands::get::("counter")).await.unwrap(), + None + ); + assert!(trips.seen().is_empty()); + + batch.flush().await; + + let expected = expected_trips( + topology, + &[ + Operation::Command("INCRBYFLOAT"), + Operation::Command("INCRBYFLOAT"), + Operation::Command("SET"), + Operation::Command("GET"), + ], + ); + let mut seen = trips.seen(); + seen.sort_by_key(|operation| format!("{operation:?}")); + let mut sorted = expected.clone(); + sorted.sort_by_key(|operation| format!("{operation:?}")); + assert_eq!(seen, sorted); + assert_eq!(counter.resolve().await.unwrap(), 3.0); + assert_eq!(total.resolve().await.unwrap(), 1.5); + written.resolve().await.unwrap(); + assert_eq!(read.resolve().await.unwrap(), None); + assert_eq!(trips.seen().len(), expected.len()); +} + +#[rstest] +#[tokio::test] +async fn resolving_before_a_flush_sends_everything_declared_so_far( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let (client, trips) = observed(&server); + let batch = client.batch(); + let first = batch.add(commands::incr_by_float("a", Amount::Int(1), MINUTE)); + let second = batch.add(commands::incr_by_float("b", Amount::Int(2), MINUTE)); + let increment = Operation::Command("INCRBYFLOAT"); + + assert_eq!(second.resolve().await.unwrap(), 2.0); + let late = batch.add(commands::incr_by_float("a", Amount::Int(10), MINUTE)); + assert_eq!(first.resolve().await.unwrap(), 1.0); + assert_eq!( + trips.seen(), + expected_trips(topology, &[increment, increment]) + ); + + assert_eq!(late.resolve().await.unwrap(), 11.0); + let both: Vec = expected_trips(topology, &[increment, increment]) + .into_iter() + .chain(expected_trips(topology, &[increment])) + .collect(); + assert_eq!(trips.seen(), both); + batch.flush().await; + assert_eq!(trips.seen().len(), both.len()); +} + +#[rstest] +#[tokio::test] +async fn a_pipelined_batch_is_one_pipeline_on_every_topology( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let (client, trips) = observed(&server); + let batch = RedisBatch::pipelined(client); + let declared = + ["a", "b", "c"].map(|key| batch.add(commands::incr_by_float(key, Amount::Int(2), MINUTE))); + + let totals = Declared::all(declared).await; + + assert_eq!( + totals.into_iter().map(Result::unwrap).collect::>(), + [2.0; 3] + ); + assert_eq!(trips.seen(), [Operation::Pipeline { ops: 3 }]); +} + +#[rstest] +#[tokio::test] +async fn each_declared_op_gets_its_own_typed_result( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + client + .send(commands::set( + "text", + "not a number", + Duration::from_secs(60), + )) + .await + .unwrap(); + let batch = client.batch(); + let rejected = batch.add(commands::incr_by_float("text", Amount::Int(1), MINUTE)); + let undecodable = batch.add(commands::get::("text")); + let missing = batch.add(commands::get::("missing")); + let script = batch.add(commands::eval::>( + &Script::new("return {KEYS[1], ARGV[1]}"), + &["k"], + "arg", + )); + let removed = batch.add(commands::del(&["text"])); + + batch.flush().await; + + assert!(matches!(rejected.resolve().await, Err(Error::Server(_)))); + assert!(matches!(undecodable.resolve().await, Err(Error::Decode(_)))); + assert_eq!(missing.resolve().await.unwrap(), None); + assert_eq!(script.resolve().await.unwrap(), ["k", "arg"]); + assert_eq!(removed.resolve().await.unwrap(), 1); +} + +#[rstest] +#[tokio::test] +async fn an_op_whose_flush_was_cancelled_resolves_as_transport( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", Duration::from_secs(5)); + let batch = client.batch(); + let counter = batch.add(commands::incr_by_float("counter", Amount::Int(1), MINUTE)); + + server.freeze().await.unwrap(); + let cancelled = tokio::time::timeout(Duration::from_millis(200), batch.flush()).await; + server.thaw().await.unwrap(); + + assert!(cancelled.is_err()); + assert!(matches!(counter.resolve().await, Err(Error::Transport(_)))); +} + +#[rstest] +#[tokio::test] +async fn a_settled_op_waits_for_someone_else_to_flush_and_sends_nothing_itself( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let (client, trips) = observed(&server); + let batch = client.batch(); + let counter = tokio::spawn( + batch + .add(commands::incr_by_float("counter", Amount::Int(4), MINUTE)) + .settled(), + ); + + tokio::time::sleep(Duration::from_millis(200)).await; + assert!(!counter.is_finished()); + assert!(trips.seen().is_empty()); + + batch.flush().await; + + assert_eq!(counter.await.unwrap().unwrap(), 4.0); + assert_eq!( + trips.seen(), + expected_trips(topology, &[Operation::Command("INCRBYFLOAT")]) + ); +} + +#[rstest] +#[tokio::test] +async fn a_settled_op_whose_flush_was_cancelled_resolves_as_transport() { + let server = RedisDeployment::start(Topology::Standalone).await.unwrap(); + let client = connect(&server, "", Duration::from_secs(5)); + let batch = client.batch(); + let counter = batch.add(commands::incr_by_float("counter", Amount::Int(1), MINUTE)); + + server.freeze().await.unwrap(); + let cancelled = tokio::time::timeout(Duration::from_millis(200), batch.flush()).await; + server.thaw().await.unwrap(); + + assert!(cancelled.is_err()); + assert!(matches!(counter.settled().await, Err(Error::Transport(_)))); +} + +#[rstest] +#[tokio::test] +async fn clones_are_the_same_client_and_a_second_connection_is_not() { + let server = RedisDeployment::start(Topology::Standalone).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + let twin = connect(&server, "", DEFAULT_TIMEOUT); + + assert!(client.same_client(&client.clone())); + assert!(!client.same_client(&twin)); +} + +#[rstest] +#[tokio::test] +async fn mget_reads_each_key_once_and_answers_every_key_in_order( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + client + .send(commands::set("a", "1", Duration::from_secs(60))) + .await + .unwrap(); + client + .send(commands::set("c", "3", Duration::from_secs(60))) + .await + .unwrap(); + server + .cli_on_every_node(&["CONFIG", "RESETSTAT"]) + .await + .unwrap(); + let batch = client.batch(); + + let values = batch.add(commands::mget::(&["c", "missing", "a", "c"])); + batch.flush().await; + + assert_eq!( + values.resolve().await.unwrap(), + [ + Some("3".to_owned()), + None, + Some("1".to_owned()), + Some("3".to_owned()) + ] + ); + let stats = support::command_stats(&server).await; + assert!(support::calls(&stats, "mget") > 0, "{stats}"); + assert_eq!(support::calls(&stats, "get"), 0, "{stats}"); + assert_eq!(support::keys_read(&server).await, 3); +} + +// RedisBatch's _Increment refreshes the TTL in a pipeline, while its cluster path calls +// async_increment, which keeps a TTL the key already has +#[rstest] +#[case::pipelined(Topology::Standalone, false)] +#[case::alone_on_a_cluster(Topology::Cluster, true)] +#[tokio::test] +async fn a_batched_increment_treats_the_ttl_like_redis_batch( + #[case] topology: Topology, + #[case] keeps_the_ttl: bool, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + client + .send(commands::set("counter", "1", Duration::from_secs(30))) + .await + .unwrap(); + let batch = client.batch(); + + let counter = batch.add(commands::incr_by_float( + "counter", + Amount::Int(1), + Some(Expiry::KeepExisting(Duration::from_secs(600))), + )); + batch.flush().await; + + assert_eq!(counter.resolve().await.unwrap(), 2.0); + let Ttl::Expires(left) = client.send(commands::ttl("counter")).await.unwrap() else { + panic!("expected an expiring counter") + }; + assert_eq!(left <= Duration::from_secs(30), keeps_the_ttl, "{left:?}"); +} diff --git a/litellm-rust/crates/redis-client/tests/client.rs b/litellm-rust/crates/redis-client/tests/client.rs new file mode 100644 index 00000000000..e9c6cd7e712 --- /dev/null +++ b/litellm-rust/crates/redis-client/tests/client.rs @@ -0,0 +1,1114 @@ +mod support; + +use std::{ + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; + +use litellm_redis_testing::{RedisCluster, RedisDeployment, RedisServer, Topology, eventually}; +use rstest::rstest; + +use litellm_circuit_breaker::{Breaker, Config, State}; + +use litellm_redis_client::commands; +use litellm_redis_client::{ + Amount, CallEvent, Credentials, DEFAULT_TIMEOUT, Error, Expiry, FailureKind, Namespace, + Operation, Outcome, RedisClient, RedisConfig, RedisEvents, RedisUrl, Script, Standalone, + Target, Timeouts, Ttl, Value, +}; +use litellm_secret_value::SecretValue; +use support::{KEEP, connect, connect_guarded, connections, endpoint}; + +async fn persist(client: &RedisClient, key: &str) { + let set = Script::new("return redis.call('SET', KEYS[1], '0')"); + client + .send(commands::eval::<()>(&set, &[key], Vec::::new())) + .await + .unwrap(); +} + +#[derive(Clone, Copy, Debug)] +enum Counter { + IncrBy, + IncrByFloat, +} + +const MINUTE: Duration = Duration::from_secs(60); + +async fn write(client: &RedisClient, counter: Counter, key: &str, expiry: Expiry) { + match (counter, expiry) { + (Counter::IncrBy, Expiry::KeepExisting(ttl)) => { + client.send(commands::incr_by(key, 1, ttl)).await.unwrap(); + } + (Counter::IncrBy, Expiry::Refresh(_)) => unreachable!("increment_cache never refreshes"), + (Counter::IncrByFloat, expiry) => { + client + .send(commands::incr_by_float( + key, + Amount::Float(0.5), + Some(expiry), + )) + .await + .unwrap(); + } + } +} + +async fn persist_as(client: &RedisClient, key: &str) { + let source = "return redis.call('SET', KEYS[1], '1')"; + client + .send(commands::eval::( + &Script::new(source), + &[key], + Vec::::new(), + )) + .await + .unwrap(); +} + +fn remaining(ttl: Ttl) -> Duration { + match ttl { + Ttl::Expires(left) => left, + other => panic!("expected an expiring key, got {other:?}"), + } +} + +#[rstest] +#[case::increment_cache(Counter::IncrBy, Expiry::KeepExisting(MINUTE))] +#[case::async_increment(Counter::IncrByFloat, Expiry::KeepExisting(MINUTE))] +#[case::async_increment_refreshing(Counter::IncrByFloat, Expiry::Refresh(MINUTE))] +#[tokio::test] +async fn a_counter_write_expires_a_new_key( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + #[case] counter: Counter, + #[case] expiry: Expiry, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + + write(&client, counter, "fresh", expiry).await; + + let left = remaining(client.send(commands::ttl("fresh")).await.unwrap()); + assert!(left > Duration::from_secs(50) && left <= MINUTE, "{left:?}"); +} + +#[rstest] +#[tokio::test] +async fn keep_existing_leaves_a_running_ttl_and_refresh_resets_it( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + let short = Expiry::Refresh(Duration::from_secs(30)); + let long = Duration::from_secs(600); + write(&client, Counter::IncrByFloat, "kept", short).await; + write(&client, Counter::IncrByFloat, "refreshed", short).await; + write( + &client, + Counter::IncrBy, + "kept_int", + Expiry::KeepExisting(Duration::from_secs(30)), + ) + .await; + + write( + &client, + Counter::IncrByFloat, + "kept", + Expiry::KeepExisting(long), + ) + .await; + write( + &client, + Counter::IncrByFloat, + "refreshed", + Expiry::Refresh(long), + ) + .await; + write( + &client, + Counter::IncrBy, + "kept_int", + Expiry::KeepExisting(long), + ) + .await; + + assert!( + remaining(client.send(commands::ttl("kept")).await.unwrap()) <= Duration::from_secs(30) + ); + assert!( + remaining(client.send(commands::ttl("kept_int")).await.unwrap()) <= Duration::from_secs(30) + ); + assert!( + remaining(client.send(commands::ttl("refreshed")).await.unwrap()) + > Duration::from_secs(500) + ); +} + +#[rstest] +#[tokio::test] +async fn keep_existing_expires_a_key_that_had_no_ttl( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + #[values(Counter::IncrBy, Counter::IncrByFloat)] counter: Counter, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "ns", DEFAULT_TIMEOUT); + persist_as(&client, "persistent").await; + + write(&client, counter, "persistent", KEEP).await; + + let left = remaining(client.send(commands::ttl("persistent")).await.unwrap()); + assert!(left > Duration::from_secs(50), "{left:?}"); +} + +#[rstest] +#[tokio::test] +async fn counters_send_the_commands_redis_cache_sends( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + server + .cli_on_every_node(&["CONFIG", "RESETSTAT"]) + .await + .unwrap(); + + client + .send(commands::incr_by("int", 1, MINUTE)) + .await + .unwrap(); + client + .send(commands::incr_by_float("kept", Amount::Int(1), Some(KEEP))) + .await + .unwrap(); + client + .send(commands::incr_by_float( + "refreshed", + Amount::Float(0.5), + Some(Expiry::Refresh(MINUTE)), + )) + .await + .unwrap(); + client + .run( + &client + .pipeline() + .with(commands::incr_by_float( + "piped", + Amount::Float(0.25), + Some(Expiry::Refresh(MINUTE)), + )) + .with(commands::incr_by_float("bare", Amount::Float(0.25), None)), + ) + .await; + + let stats = support::command_stats(&server).await; + let calls = |command: &str| support::calls(&stats, command); + assert_eq!(calls("incrby"), 1); + assert_eq!(calls("incrbyfloat"), 4); + assert_eq!(calls("ttl"), 2); + assert_eq!(calls("expire"), 4); + assert_eq!( + calls("eval") + calls("evalsha") + calls("pttl") + calls("pexpire"), + 0 + ); +} + +#[derive(Default)] +struct Recorded(Mutex>); + +#[rstest] +#[case::beyond_double_precision(9_007_199_254_740_992_i64, 1_i64)] +#[case::negative_beyond_double_precision(-9_007_199_254_740_992_i64, -1_i64)] +#[case::maximum_integer(i64::MAX - 1, 1_i64)] +#[case::minimum_integer(i64::MIN + 1, -1_i64)] +#[tokio::test] +async fn integer_increments_preserve_every_bit( + #[case] initial: i64, + #[case] delta: i64, + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + client + .send(commands::set("integer", initial, Duration::from_secs(60))) + .await + .unwrap(); + let expected = initial + delta; + let result = client + .send(commands::incr_by("integer", delta, MINUTE)) + .await + .unwrap(); + assert_eq!(result, expected); + assert_eq!( + client.send(commands::get::("integer")).await.unwrap(), + Some(expected) + ); +} + +#[rstest] +#[tokio::test] +async fn outage_events_report_the_failure_kind( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + #[values(false, true)] timeout: bool, + #[values(false, true)] pipeline: bool, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let recorded = Arc::new(Recorded::default()); + let client = RedisClient::new(RedisConfig { + target: endpoint(&server), + namespace: Namespace::default(), + timeouts: Timeouts::new(Duration::from_millis(300), None), + breaker: None, + events: Some(recorded.clone()), + credentials: Credentials::NONE, + health_check: None, + keepalive: true, + }) + .unwrap(); + client.ping().await.unwrap(); + if timeout { + server.freeze().await.unwrap(); + } else { + server.stop().await.unwrap(); + } + let result = if pipeline { + client + .run(&client.pipeline().with(commands::get::("missing"))) + .await + .remove(0) + .map(drop) + } else { + client + .send(commands::get::("missing")) + .await + .map(drop) + }; + if timeout { + server.thaw().await.unwrap(); + } else { + server.resume().await.unwrap(); + } + assert!(matches!(result, Err(Error::Transport(_)))); + let expected = if timeout { + FailureKind::Timeout + } else { + FailureKind::Transport + }; + assert_eq!( + recorded.0.lock().unwrap().last().unwrap().outcome, + Outcome::Failed(expected) + ); +} + +impl RedisEvents for Recorded { + fn call(&self, event: &CallEvent) { + self.0.lock().unwrap().push(*event); + } + + fn breaker(&self, _transition: &litellm_circuit_breaker::Transition) {} +} + +#[rstest] +#[tokio::test] +async fn every_call_reports_its_operation_and_outcome( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let recorded = Arc::new(Recorded::default()); + let client = RedisClient::new(RedisConfig { + target: endpoint(&server), + credentials: Credentials::NONE, + namespace: Namespace::default(), + timeouts: Timeouts::default(), + breaker: None, + events: Some(recorded.clone()), + health_check: None, + keepalive: true, + }) + .unwrap(); + client + .send(commands::set("list", "item", Duration::from_secs(60))) + .await + .unwrap(); + + let _ = client + .send(commands::incr_by("list", 1, Duration::from_secs(60))) + .await; + let _ = client.send(commands::get::("missing")).await; + let _ = client + .run( + &client + .pipeline() + .with(commands::get::("a")) + .with(commands::get::("b")), + ) + .await; + let failures = client + .run(&client.pipeline().with(commands::incr_by_float( + "list", + Amount::Int(1), + Some(Expiry::Refresh(Duration::from_secs(60))), + ))) + .await; + assert!(matches!(failures[0], Err(Error::Server(_)))); + + let seen: Vec<(Operation, Outcome)> = recorded + .0 + .lock() + .unwrap() + .iter() + .map(|event| (event.operation, event.outcome)) + .collect(); + assert_eq!( + seen, + vec![ + (Operation::Command("SET"), Outcome::Succeeded), + ( + Operation::Command("INCRBY"), + Outcome::Failed(FailureKind::Server) + ), + (Operation::Command("GET"), Outcome::Succeeded), + (Operation::Pipeline { ops: 2 }, Outcome::Succeeded), + ( + Operation::Pipeline { ops: 1 }, + Outcome::Failed(FailureKind::Server) + ), + ] + ); +} + +#[rstest] +#[tokio::test] +async fn namespaces_isolate_keys_on_the_server( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let a = connect(&server, "a", DEFAULT_TIMEOUT); + let b = connect(&server, "b", DEFAULT_TIMEOUT); + let bare = connect(&server, "", DEFAULT_TIMEOUT); + + a.send(commands::set("x", "1", Duration::from_secs(60))) + .await + .unwrap(); + + assert_eq!( + a.send(commands::get::("x")) + .await + .unwrap() + .as_deref(), + Some("1") + ); + assert_eq!( + a.send(commands::get::("a:x")) + .await + .unwrap() + .as_deref(), + Some("1") + ); + assert_eq!( + bare.send(commands::get::("a:x")) + .await + .unwrap() + .as_deref(), + Some("1") + ); + assert_eq!(b.send(commands::get::("x")).await.unwrap(), None); +} + +#[rstest] +#[tokio::test] +async fn a_rejected_command_keeps_the_connection( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + client + .send(commands::set("list", "item", Duration::from_secs(60))) + .await + .unwrap(); + let before = connections(&server).await; + + let rejected = client + .send(commands::incr_by("list", 1, Duration::from_secs(60))) + .await; + + assert!(matches!(rejected, Err(Error::Server(_))), "{rejected:?}"); + assert_eq!( + client + .send(commands::get::("list")) + .await + .unwrap() + .as_deref(), + Some("item") + ); + assert_eq!(connections(&server).await, before); +} + +#[rstest] +#[tokio::test] +async fn a_reply_of_another_type_is_a_decode_error_and_a_missing_key_is_none( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + client + .send(commands::set("float", "15.0", Duration::from_secs(60))) + .await + .unwrap(); + + let decoded = client.send(commands::get::("float")).await; + + assert!(matches!(decoded, Err(Error::Decode(_))), "{decoded:?}"); + assert_eq!( + client.send(commands::get::("missing")).await.unwrap(), + None + ); +} + +#[rstest] +#[tokio::test] +async fn a_timed_out_command_does_not_leak_its_reply_into_the_next_one( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", Duration::from_millis(500)); + client + .send(commands::set("first", "1", Duration::from_secs(60))) + .await + .unwrap(); + client + .send(commands::set("second", "2", Duration::from_secs(60))) + .await + .unwrap(); + + server.freeze().await.unwrap(); + let timed_out = client.send(commands::get::("first")).await; + server.thaw().await.unwrap(); + + assert!( + matches!(timed_out, Err(Error::Transport(_))), + "{timed_out:?}" + ); + assert_eq!( + client + .send(commands::get::("second")) + .await + .unwrap() + .as_deref(), + Some("2") + ); +} + +#[rstest] +#[tokio::test] +async fn the_same_client_recovers_after_redis_restarts( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", Duration::from_millis(500)); + + server.restart().await.unwrap(); + + let recovered = eventually(async || { + client + .send(commands::incr_by( + "after_restart", + 1, + Duration::from_secs(60), + )) + .await + .ok() + }) + .await; + assert_eq!(recovered, 1); +} + +#[rstest] +#[tokio::test] +async fn a_client_created_while_redis_is_down_works_once_redis_is_up( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + server.stop().await.unwrap(); + + let client = connect(&server, "", Duration::from_millis(500)); + let started = Instant::now(); + let while_down = client.ping().await; + let failed_in = started.elapsed(); + server.resume().await.unwrap(); + + assert!(failed_in < Duration::from_secs(2), "{failed_in:?}"); + + assert!( + matches!(while_down, Err(Error::Transport(_))), + "{while_down:?}" + ); + let recovered = eventually(async || client.ping().await.ok()).await; + assert_eq!(recovered, ()); +} + +#[rstest] +#[tokio::test] +async fn empty_batches_succeed_without_asking_redis( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + + assert_eq!( + client.send(commands::mget::(&[])).await.unwrap(), + Vec::>::new() + ); + assert_eq!(client.send(commands::del(&[])).await.unwrap(), 0); +} + +#[rstest] +#[tokio::test] +async fn mget_and_del_span_slots_in_input_order( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + client + .send(commands::set("a", "1", Duration::from_secs(60))) + .await + .unwrap(); + client + .send(commands::set("c", "3", Duration::from_secs(60))) + .await + .unwrap(); + + let values = client + .send(commands::mget::(&["c", "b", "a"])) + .await + .unwrap(); + let deleted = client.send(commands::del(&["a", "b", "c"])).await.unwrap(); + + assert_eq!( + values, + vec![Some("3".to_owned()), None, Some("1".to_owned())] + ); + assert_eq!(deleted, 2); + assert_eq!( + client + .send(commands::mget::(&["a", "c"])) + .await + .unwrap(), + vec![None, None] + ); +} + +#[rstest] +#[tokio::test] +async fn set_nx_only_writes_a_missing_key( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + + let first = client + .send(commands::set_nx("lock", "owner-1", Duration::from_secs(60))) + .await + .unwrap(); + let second = client + .send(commands::set_nx("lock", "owner-2", Duration::from_secs(60))) + .await + .unwrap(); + + assert!(first); + assert!(!second); + assert_eq!( + client + .send(commands::get::("lock")) + .await + .unwrap() + .as_deref(), + Some("owner-1") + ); +} + +#[rstest] +#[tokio::test] +async fn ttl_reports_missing_persistent_and_remaining_time( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + persist(&client, "persistent").await; + client + .send(commands::set("expiring", "1", Duration::from_secs(60))) + .await + .unwrap(); + client + .send(commands::incr_by_float( + "expired_later", + Amount::Float(0.5), + Some(KEEP), + )) + .await + .unwrap(); + assert!( + client + .send(commands::expire("expired_later", Duration::from_secs(30))) + .await + .unwrap() + ); + + assert_eq!( + client.send(commands::ttl("missing")).await.unwrap(), + Ttl::Missing + ); + assert_eq!( + client.send(commands::ttl("persistent")).await.unwrap(), + Ttl::Persistent + ); + assert!(matches!( + client.send(commands::ttl("expiring")).await.unwrap(), + Ttl::Expires(left) if left > Duration::from_secs(50) && left <= Duration::from_secs(60) + )); + assert!(matches!( + client.send(commands::ttl("expired_later")).await.unwrap(), + Ttl::Expires(left) if left > Duration::from_secs(20) && left <= Duration::from_secs(30) + )); + assert!( + !client + .send(commands::expire("missing", Duration::from_secs(30))) + .await + .unwrap() + ); +} + +#[rstest] +#[tokio::test] +async fn incr_by_float_accumulates_fractions( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + + client + .send(commands::incr_by_float( + "spend", + Amount::Float(0.25), + Some(KEEP), + )) + .await + .unwrap(); + + assert_eq!( + client + .send(commands::incr_by_float( + "spend", + Amount::Float(0.5), + Some(KEEP) + )) + .await + .unwrap(), + 0.75 + ); +} + +#[rstest] +#[tokio::test] +async fn concurrent_commands_on_one_cloned_client_all_apply( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + + let tasks: Vec<_> = (0..100) + .map(|_| { + let client = client.clone(); + tokio::spawn(async move { + for _ in 0..10 { + client + .send(commands::incr_by("shared", 1, Duration::from_secs(60))) + .await + .unwrap(); + } + }) + }) + .collect(); + for task in tasks { + task.await.unwrap(); + } + + assert_eq!( + client.send(commands::get::("shared")).await.unwrap(), + Some(1000) + ); +} + +fn breaker(failure_threshold: u32) -> Breaker { + Breaker::new( + Config { + failure_threshold, + recovery_timeout: Duration::from_millis(200), + timeout_min_duration: Duration::ZERO, + }, + Box::new(|_| {}), + ) +} + +#[rstest] +#[tokio::test] +async fn an_open_breaker_rejects_without_waiting_on_redis_and_a_probe_closes_it( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let breaker = breaker(2); + let client = connect_guarded( + &server, + "", + Duration::from_millis(300), + Some(breaker.clone()), + ); + + server.freeze().await.unwrap(); + let first = client.send(commands::get::("k")).await; + let second = client.send(commands::get::("k")).await; + let started = Instant::now(); + let third = client.send(commands::get::("k")).await; + let rejected_in = started.elapsed(); + server.thaw().await.unwrap(); + + assert!(matches!(first, Err(Error::Transport(_))), "{first:?}"); + assert!(matches!(second, Err(Error::Transport(_))), "{second:?}"); + assert!(matches!(third, Err(Error::BreakerOpen)), "{third:?}"); + assert!(rejected_in < Duration::from_millis(50), "{rejected_in:?}"); + + let recovered = eventually(async || client.send(commands::get::("k")).await.ok()).await; + assert_eq!(recovered, None); + assert!( + matches!(breaker.state(), State::Closed { .. }), + "{:?}", + breaker.state() + ); +} + +#[rstest] +#[tokio::test] +async fn rejected_commands_never_open_the_breaker( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let breaker = breaker(2); + let client = connect_guarded(&server, "", DEFAULT_TIMEOUT, Some(breaker.clone())); + client + .send(commands::set("list", "item", Duration::from_secs(60))) + .await + .unwrap(); + + let rejected = [ + client + .send(commands::incr_by("list", 1, Duration::from_secs(60))) + .await, + client + .send(commands::incr_by("list", 1, Duration::from_secs(60))) + .await, + client + .send(commands::incr_by("list", 1, Duration::from_secs(60))) + .await, + ]; + + assert!( + matches!(rejected[2], Err(Error::Server(_))), + "{:?}", + rejected[2] + ); + assert!( + matches!(breaker.state(), State::Closed { hard: 0, .. }), + "{:?}", + breaker.state() + ); +} + +#[rstest] +#[tokio::test] +async fn a_probe_redis_rejects_still_proves_redis_answers_and_closes_the_breaker( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let breaker = breaker(2); + let client = connect_guarded( + &server, + "", + Duration::from_millis(300), + Some(breaker.clone()), + ); + client + .send(commands::set("list", "item", Duration::from_secs(60))) + .await + .unwrap(); + server.freeze().await.unwrap(); + let _ = client.send(commands::get::("k")).await; + let _ = client.send(commands::get::("k")).await; + server.thaw().await.unwrap(); + assert!( + matches!(breaker.state(), State::Open { .. }), + "{:?}", + breaker.state() + ); + tokio::time::sleep(Duration::from_millis(250)).await; + + let probe = client + .send(commands::incr_by("list", 1, Duration::from_secs(60))) + .await; + + assert!(matches!(probe, Err(Error::Server(_))), "{probe:?}"); + assert!( + matches!(breaker.state(), State::Closed { .. }), + "{:?}", + breaker.state() + ); +} + +#[rstest] +#[tokio::test] +async fn a_replica_refusing_a_write_does_not_reset_a_run_of_failures() { + let server = RedisDeployment::start(Topology::Standalone).await.unwrap(); + let breaker = breaker(2); + let client = connect_guarded( + &server, + "", + Duration::from_millis(300), + Some(breaker.clone()), + ); + client.send(commands::get::("k")).await.unwrap(); + server.freeze().await.unwrap(); + let _ = client.send(commands::get::("k")).await; + server.thaw().await.unwrap(); + server + .cli_on_every_node(&["REPLICAOF", "127.0.0.1", "1"]) + .await + .unwrap(); + + let refused = client + .send(commands::set("k", "v", Duration::from_secs(60))) + .await; + server.freeze().await.unwrap(); + let _ = client.send(commands::get::("k")).await; + server.thaw().await.unwrap(); + + assert!(matches!(refused, Err(Error::Server(_))), "{refused:?}"); + assert!( + matches!(breaker.state(), State::Open { .. }), + "{:?}", + breaker.state() + ); +} + +#[rstest] +#[tokio::test] +async fn an_open_breaker_fails_every_pipeline_op_without_sending( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect_guarded(&server, "", Duration::from_millis(300), Some(breaker(1))); + let pipeline = client + .pipeline() + .with(commands::incr_by_float( + "a", + Amount::Int(1), + Some(Expiry::Refresh(Duration::from_secs(60))), + )) + .with(commands::incr_by_float( + "c", + Amount::Int(1), + Some(Expiry::Refresh(Duration::from_secs(60))), + )); + + server.freeze().await.unwrap(); + let failed = client.run(&pipeline).await; + let rejected = client.run(&pipeline).await; + server.thaw().await.unwrap(); + + assert!( + matches!(failed[0], Err(Error::Transport(_))), + "{:?}", + failed[0] + ); + assert!( + matches!(rejected[0], Err(Error::BreakerOpen)), + "{:?}", + rejected[0] + ); + assert!( + matches!(rejected[1], Err(Error::BreakerOpen)), + "{:?}", + rejected[1] + ); +} + +#[rstest] +#[tokio::test] +async fn a_host_endpoint_connects_to_its_database() { + let server = RedisServer::start().await.unwrap(); + let port = server.url().rsplit(':').next().unwrap().parse().unwrap(); + let connect = |target| { + RedisClient::new(RedisConfig { + target, + credentials: Credentials::NONE, + namespace: Namespace::default(), + timeouts: Timeouts::default(), + breaker: None, + events: None, + health_check: None, + keepalive: true, + }) + }; + let url = |url: String| { + Target::Url(RedisUrl { + url: SecretValue::new(url), + tls: None, + db: None, + username: None, + password: None, + }) + }; + let in_db_3 = connect(Target::Standalone(Standalone { + host: "127.0.0.1".to_owned(), + port, + tls: None, + db: 3, + })) + .unwrap(); + let in_db_0 = connect(url(server.url().to_owned())).unwrap(); + let via_url = connect(url(format!("{}/3", server.url()))).unwrap(); + + in_db_3 + .send(commands::set("k", "3", Duration::from_secs(60))) + .await + .unwrap(); + + assert_eq!( + in_db_0.send(commands::get::("k")).await.unwrap(), + None + ); + assert_eq!( + via_url + .send(commands::get::("k")) + .await + .unwrap() + .as_deref(), + Some("3") + ); +} + +async fn ping_calls(cluster: &RedisCluster) -> Vec { + cluster + .cli_outputs(&["INFO", "commandstats"]) + .await + .unwrap() + .iter() + .map(|info| { + info.lines() + .find_map(|line| line.strip_prefix("cmdstat_ping:calls=")) + .and_then(|rest| rest.split(',').next()) + .and_then(|calls| calls.trim().parse().ok()) + .unwrap_or(0) + }) + .collect() +} + +fn health_checked( + server: &RedisDeployment, + interval: Duration, + timeout: Duration, + breaker: Option, + events: Option>, +) -> RedisClient { + RedisClient::new(RedisConfig { + target: endpoint(server), + credentials: Credentials::NONE, + namespace: Namespace::default(), + timeouts: Timeouts::new(timeout, None), + breaker, + events: events.map(|events| events as Arc), + health_check: Some(interval), + keepalive: true, + }) + .unwrap() +} + +// redis-py 5.3.1 redis/asyncio/connection.py check_health: a connection idle longer than +// health_check_interval gets a PING before the next command, a busy one does not +#[rstest] +#[tokio::test] +async fn an_idle_client_pings_every_node_before_the_next_command_like_redis_py() { + let server = RedisDeployment::start(Topology::Cluster).await.unwrap(); + let RedisDeployment::Cluster(cluster) = &server else { + unreachable!("started a cluster") + }; + let client = health_checked(&server, Duration::from_secs(3), DEFAULT_TIMEOUT, None, None); + client + .send(commands::set("k", "1", Duration::from_secs(60))) + .await + .unwrap(); + let before = ping_calls(cluster).await; + + client + .send(commands::set("k", "2", Duration::from_secs(60))) + .await + .unwrap(); + let busy = ping_calls(cluster).await; + tokio::time::sleep(Duration::from_millis(3500)).await; + client + .send(commands::set("k", "3", Duration::from_secs(60))) + .await + .unwrap(); + let idle = ping_calls(cluster).await; + + assert_eq!(busy, before); + assert_eq!( + idle, + before.iter().map(|calls| calls + 1).collect::>(), + "before {before:?}" + ); +} + +#[rstest] +#[tokio::test] +async fn a_failed_health_check_is_neither_a_call_nor_a_breaker_failure() { + let server = RedisDeployment::start(Topology::Cluster).await.unwrap(); + let recorded = Arc::new(Recorded::default()); + let breaker = breaker(2); + let client = health_checked( + &server, + Duration::from_millis(200), + Duration::from_millis(300), + Some(breaker.clone()), + Some(recorded.clone()), + ); + client + .send(commands::set("k", "1", Duration::from_secs(60))) + .await + .unwrap(); + + server.freeze().await.unwrap(); + tokio::time::sleep(Duration::from_millis(300)).await; + let failed = client.send(commands::get::("k")).await; + server.thaw().await.unwrap(); + + assert!(matches!(failed, Err(Error::Transport(_))), "{failed:?}"); + assert!( + matches!(breaker.state(), State::Closed { timeouts: 1, .. }), + "{:?}", + breaker.state() + ); + assert_eq!(recorded.0.lock().unwrap().len(), 2); +} + +#[rstest] +#[case::int(Amount::Int(1), "1")] +#[case::float_keeps_its_point(Amount::Float(1.0), "1.0")] +#[case::small_float(Amount::Float(1e-5), "1e-05")] +#[case::large_float(Amount::Float(1e16), "1e+16")] +#[case::fraction(Amount::Float(0.1 + 0.2), "0.30000000000000004")] +fn amounts_are_written_like_redis_py_encodes_them(#[case] amount: Amount, #[case] expected: &str) { + assert_eq!( + redis::ToRedisArgs::to_redis_args(&amount), + vec![expected.as_bytes().to_vec()] + ); +} diff --git a/litellm-rust/crates/redis-client/tests/config.rs b/litellm-rust/crates/redis-client/tests/config.rs new file mode 100644 index 00000000000..101c6529e94 --- /dev/null +++ b/litellm-rust/crates/redis-client/tests/config.rs @@ -0,0 +1,290 @@ +use rstest::rstest; + +use litellm_redis_client::{Credentials, RedisUrl, Sentinel, StaticAuth, Target, TlsOptions}; +use litellm_secret_value::SecretValue; + +#[rstest] +#[case::url(format!("{:?}", Target::Url(RedisUrl { url: SecretValue::new("redis://user:credential-value@cache:6379/0"), tls: None, db: None, username: None, password: None })))] +#[case::static_password(format!("{:?}", Credentials::Static { username: None, password: Some(SecretValue::new("credential-value")) }))] +#[case::sentinel_password(format!("{:?}", Target::Sentinel(Sentinel { + monitors: Vec::new(), + service_name: "primary".to_owned(), + monitor_auth: Some(StaticAuth { username: None, password: Some(SecretValue::new("credential-value")) }), + tls: None, + db: 0, +})))] +fn debug_output_never_shows_the_secret(#[case] debug: String) { + assert!(!debug.contains("credential-value"), "{debug}"); + assert!(debug.contains("[REDACTED]"), "{debug}"); +} + +#[derive(Clone, Copy, Debug)] +enum Ca { + Trusted, + TrustedData, + TrustedDirectory, + SystemRoots, +} + +fn directory_holding(ca: &std::path::Path) -> std::path::PathBuf { + let directory = std::env::temp_dir().join(format!("litellm-ca-dir-{}", std::process::id())); + std::fs::create_dir_all(&directory).unwrap(); + std::fs::copy(ca, directory.join("ca.pem")).unwrap(); + directory +} + +#[derive(Clone, Copy, Debug)] +enum ClientKey { + Absent, + Plain, + Encrypted { password: Option<&'static str> }, +} + +async fn tls_server(require_client_certificate: bool) -> litellm_redis_testing::RedisTlsServer { + if rustls::crypto::CryptoProvider::get_default().is_none() { + let _ = rustls::crypto::ring::default_provider().install_default(); + } + litellm_redis_testing::RedisTlsServer::start(require_client_certificate) + .await + .unwrap() +} + +fn tls_client( + server: &litellm_redis_testing::RedisTlsServer, + host: &str, + tls: TlsOptions, +) -> Result { + litellm_redis_client::RedisClient::new(litellm_redis_client::RedisConfig { + target: Target::Standalone(litellm_redis_client::Standalone { + host: host.to_owned(), + port: server.port(), + tls: Some(tls), + db: 0, + }), + credentials: Credentials::NONE, + namespace: litellm_redis_client::Namespace::default(), + timeouts: litellm_redis_client::Timeouts::default(), + breaker: None, + events: None, + health_check: None, + keepalive: true, + }) +} + +fn tls_options( + files: &litellm_redis_testing::TlsFiles, + ca: Ca, + verify_certificate: bool, + check_hostname: bool, + client_key: ClientKey, +) -> TlsOptions { + let (keyfile, key_password) = match client_key { + ClientKey::Absent => (None, None), + ClientKey::Plain => (Some(files.client_key.clone()), None), + ClientKey::Encrypted { password } => ( + Some(files.encrypted_client_key.clone()), + password.map(SecretValue::new), + ), + }; + TlsOptions { + ca_certs: matches!(ca, Ca::Trusted).then(|| files.ca.clone()), + ca_data: matches!(ca, Ca::TrustedData).then(|| std::fs::read_to_string(&files.ca).unwrap()), + ca_path: matches!(ca, Ca::TrustedDirectory).then(|| directory_holding(&files.ca)), + certfile: keyfile.is_some().then(|| files.client_cert.clone()), + keyfile, + key_password, + verify_certificate, + check_hostname, + } +} + +async fn ping_over_tls( + require_client_certificate: bool, + host: &str, + ca: Ca, + verify_certificate: bool, + check_hostname: bool, + client_key: ClientKey, +) -> Result<(), litellm_redis_client::Error> { + let server = tls_server(require_client_certificate).await; + let tls = tls_options( + server.files(), + ca, + verify_certificate, + check_hostname, + client_key, + ); + tls_client(&server, host, tls).unwrap().ping().await +} + +#[rstest] +#[case::trusted_ca_without_hostname_check_like_redis_py( + "127.0.0.1", + Ca::Trusted, + true, + false, + true +)] +#[case::trusted_ca_as_pem_text_like_ssl_ca_data("127.0.0.1", Ca::TrustedData, true, false, true)] +#[case::trusted_ca_in_a_directory_like_ssl_ca_path( + "127.0.0.1", + Ca::TrustedDirectory, + true, + false, + true +)] +#[case::hostname_check_with_a_matching_name("localhost", Ca::Trusted, true, true, true)] +#[case::hostname_check_against_an_unlisted_address("127.0.0.1", Ca::Trusted, true, true, false)] +#[case::an_untrusted_ca_is_refused("127.0.0.1", Ca::SystemRoots, true, false, false)] +#[case::cert_reqs_none_skips_verification("127.0.0.1", Ca::SystemRoots, false, false, true)] +#[tokio::test] +async fn tls_options_decide_which_servers_are_trusted( + #[case] host: &str, + #[case] ca: Ca, + #[case] verify_certificate: bool, + #[case] check_hostname: bool, + #[case] connects: bool, +) { + let result = ping_over_tls( + false, + host, + ca, + verify_certificate, + check_hostname, + ClientKey::Absent, + ) + .await; + + assert_eq!(result.is_ok(), connects, "{result:?}"); + if let Err(error) = result { + assert!( + matches!(error, litellm_redis_client::Error::Transport(_)), + "{error:?}" + ); + } +} + +#[rstest] +#[case::with_the_client_certificate(ClientKey::Plain, true)] +#[case::with_an_encrypted_key_and_its_ssl_password( + ClientKey::Encrypted { password: Some(litellm_redis_testing::CLIENT_KEY_PASSWORD) }, + true +)] +#[case::without_a_client_certificate(ClientKey::Absent, false)] +#[tokio::test] +async fn mutual_tls_sends_the_configured_client_certificate( + #[case] client_key: ClientKey, + #[case] connects: bool, +) { + let result = ping_over_tls(true, "127.0.0.1", Ca::Trusted, true, false, client_key).await; + + assert_eq!(result.is_ok(), connects, "{result:?}"); +} + +#[rstest] +#[case::without_ssl_password(None, "ssl_password is required")] +#[case::with_the_wrong_ssl_password(Some("wrong-password"), "does not decrypt the key")] +#[tokio::test] +async fn an_encrypted_key_that_cannot_be_decrypted_fails_before_connecting( + #[case] password: Option<&'static str>, + #[case] reason: &str, +) { + let server = tls_server(true).await; + let tls = tls_options( + server.files(), + Ca::Trusted, + true, + false, + ClientKey::Encrypted { password }, + ); + + let error = tls_client(&server, "127.0.0.1", tls).err().unwrap(); + + let litellm_redis_client::ConnectError::TlsKey { + path, + reason: cause, + } = &error + else { + panic!("expected a TLS key error, got {error:?}"); + }; + assert_eq!( + *path, + server.files().encrypted_client_key.display().to_string() + ); + assert!(cause.to_string().contains(reason), "{cause}"); +} + +#[rstest] +fn a_missing_tls_file_names_the_path() { + let tls = TlsOptions { + ca_certs: Some("/nonexistent/ca.pem".into()), + ..TlsOptions::verified() + }; + + let error = litellm_redis_client::RedisClient::new(litellm_redis_client::RedisConfig { + target: Target::Standalone(litellm_redis_client::Standalone { + host: "127.0.0.1".to_owned(), + port: 6379, + tls: Some(tls), + db: 0, + }), + credentials: Credentials::NONE, + namespace: litellm_redis_client::Namespace::default(), + timeouts: litellm_redis_client::Timeouts::default(), + breaker: None, + events: None, + health_check: None, + keepalive: true, + }) + .err() + .unwrap(); + + assert!(error.to_string().contains("/nonexistent/ca.pem"), "{error}"); +} + +#[rstest] +fn a_legacy_encrypted_pem_key_is_refused_with_a_conversion_hint() { + let directory = std::env::temp_dir().join(format!("litellm-legacy-key-{}", std::process::id())); + std::fs::create_dir_all(&directory).unwrap(); + let keyfile = directory.join("client.key"); + std::fs::write( + &keyfile, + "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-256-CBC,00112233445566778899AABBCCDDEEFF\n\nAAAA\n-----END RSA PRIVATE KEY-----\n", + ) + .unwrap(); + let tls = TlsOptions { + certfile: Some(keyfile.clone()), + keyfile: Some(keyfile), + key_password: Some(SecretValue::new("legacy-password")), + ..TlsOptions::verified() + }; + + let error = litellm_redis_client::RedisClient::new(litellm_redis_client::RedisConfig { + target: Target::Standalone(litellm_redis_client::Standalone { + host: "127.0.0.1".to_owned(), + port: 6379, + tls: Some(tls), + db: 0, + }), + credentials: Credentials::NONE, + namespace: litellm_redis_client::Namespace::default(), + timeouts: litellm_redis_client::Timeouts::default(), + breaker: None, + events: None, + health_check: None, + keepalive: true, + }) + .err() + .unwrap(); + + assert!( + matches!( + &error, + litellm_redis_client::ConnectError::TlsKey { + reason: litellm_redis_client::KeyError::LegacyEncryption, + .. + } + ), + "{error:?}" + ); +} diff --git a/litellm-rust/crates/redis-client/tests/connection.rs b/litellm-rust/crates/redis-client/tests/connection.rs new file mode 100644 index 00000000000..4efa6038cf0 --- /dev/null +++ b/litellm-rust/crates/redis-client/tests/connection.rs @@ -0,0 +1,329 @@ +mod support; + +use std::{collections::BTreeSet, sync::Arc, time::Duration}; + +use litellm_redis_testing::{ + ClientConnection, RedisDeployment, RedisSentinel, RedisServer, SERVICE_NAME, Topology, + eventually, +}; +use rstest::rstest; + +use litellm_redis_client::commands; +use litellm_redis_client::{ + Credentials, CredentialsProvider, Error, Namespace, Node, RedisClient, RedisConfig, Sentinel, + Standalone, Target, Timeouts, +}; +use litellm_secret_value::SecretValue; +use support::{Rotating, connect_as, connections, proxied}; + +#[rstest] +#[tokio::test] +async fn a_pushed_credential_reauthenticates_the_live_connection_and_a_reconnect_uses_the_latest( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + server + .cli_on_every_node(&[ + "ACL", "SETUSER", "alice", "on", ">first", "~*", "&*", "+@all", + ]) + .await + .unwrap(); + server + .cli_on_every_node(&[ + "ACL", "SETUSER", "bob", "on", ">second", "~*", "&*", "+@all", + ]) + .await + .unwrap(); + let provider = Arc::new(Rotating::new("alice", "first")); + let client = connect_as( + &server, + Credentials::Provider(CredentialsProvider(provider.clone())), + ); + client + .send(commands::set("k", "1", Duration::from_secs(60))) + .await + .unwrap(); + let before = connections(&server).await; + + provider.rotate("bob", "second"); + let rotated = eventually(async || { + let now = connections(&server).await; + (users(&now) == BTreeSet::from(["bob"])).then_some(now) + }) + .await; + server + .cli_on_every_node(&["ACL", "DELUSER", "alice"]) + .await + .unwrap(); + server + .cli_on_every_node(&["CLIENT", "KILL", "USER", "bob"]) + .await + .unwrap(); + let after_reconnect = eventually(async || { + client + .send(commands::get::("k")) + .await + .ok() + .flatten() + }) + .await; + + let reconnected = connections(&server).await; + + assert_eq!(users(&before), BTreeSet::from(["alice"])); + assert_eq!(ids(&rotated), ids(&before)); + assert_eq!(after_reconnect, "1"); + assert!(!reconnected.is_empty()); + assert_eq!(users(&reconnected), BTreeSet::from(["bob"])); +} + +fn users(connections: &[ClientConnection]) -> BTreeSet<&str> { + connections + .iter() + .map(|connection| connection.user.as_str()) + .collect() +} + +fn ids(connections: &[ClientConnection]) -> Vec<(u16, u64)> { + connections + .iter() + .map(|connection| (connection.node, connection.id)) + .collect() +} + +#[rstest] +#[tokio::test] +async fn an_out_of_range_database_is_a_server_rejection() { + let server = RedisServer::start().await.unwrap(); + let port = server.url().rsplit(':').next().unwrap().parse().unwrap(); + let client = RedisClient::new(RedisConfig { + target: Target::Standalone(Standalone { + host: "127.0.0.1".into(), + port, + tls: None, + db: 999, + }), + credentials: Credentials::NONE, + namespace: Namespace::default(), + timeouts: Timeouts::new(Duration::from_secs(2), None), + breaker: None, + events: None, + health_check: None, + keepalive: true, + }) + .unwrap(); + let result = client.ping().await; + assert!(matches!(result, Err(Error::Server(_))), "{result:?}"); +} + +#[rstest] +#[tokio::test] +async fn a_rejected_credential_is_a_transport_failure() { + let server = RedisDeployment::start(Topology::Standalone).await.unwrap(); + server + .cli_on_every_node(&["ACL", "SETUSER", "default", "on", ">right"]) + .await + .unwrap(); + let with_password = |password: &str| { + connect_as( + &server, + Credentials::Static { + username: None, + password: Some(SecretValue::new(password)), + }, + ) + }; + with_password("right").ping().await.unwrap(); + + let rejected = with_password("wrong").ping().await; + + assert!(matches!(rejected, Err(Error::Transport(_))), "{rejected:?}"); +} + +fn sentinel_client(sentinel: &RedisSentinel, credentials: Credentials) -> RedisClient { + let (host, port) = sentinel.sentinel(); + RedisClient::new(RedisConfig { + target: Target::Sentinel(Sentinel { + monitors: vec![Node { + host: host.to_owned(), + port, + }], + service_name: SERVICE_NAME.to_owned(), + monitor_auth: None, + tls: None, + db: 0, + }), + credentials, + namespace: Namespace::default(), + timeouts: Timeouts::new(Duration::from_secs(1), None), + breaker: None, + events: None, + health_check: None, + keepalive: true, + }) + .unwrap() +} + +#[rstest] +#[tokio::test] +async fn sentinel_writes_follow_the_master_through_a_failover() { + let sentinel = RedisSentinel::start().await.unwrap(); + let client = sentinel_client(&sentinel, Credentials::NONE); + client + .send(commands::set("before", "1", Duration::from_secs(600))) + .await + .unwrap(); + eventually(async || { + let replica = + redis::Client::open(format!("redis://127.0.0.1:{}", sentinel.replica_port())).ok()?; + let mut replica = replica.get_multiplexed_async_connection().await.ok()?; + let value: Option = redis::cmd("GET") + .arg("before") + .query_async(&mut replica) + .await + .ok()?; + value + }) + .await; + + let promoted = sentinel.failover().await.unwrap(); + eventually(async || { + client + .send(commands::set("after", "2", Duration::from_secs(600))) + .await + .ok() + }) + .await; + + assert_eq!(promoted, sentinel.replica_port()); + assert_eq!( + client + .send(commands::get::("before")) + .await + .unwrap() + .as_deref(), + Some("1") + ); + let promoted = redis::Client::open(format!("redis://127.0.0.1:{promoted}")).unwrap(); + let mut promoted = promoted.get_multiplexed_async_connection().await.unwrap(); + let after: Option = redis::cmd("GET") + .arg("after") + .query_async(&mut promoted) + .await + .unwrap(); + assert_eq!(after.as_deref(), Some("2")); +} + +const DATA_PASSWORD: &str = "data-secret"; + +#[rstest] +#[tokio::test] +async fn sentinel_reaches_a_master_that_requires_a_password() { + let sentinel = RedisSentinel::start_with_password(DATA_PASSWORD) + .await + .unwrap(); + let client = sentinel_client( + &sentinel, + Credentials::Static { + username: None, + password: Some(SecretValue::new(DATA_PASSWORD)), + }, + ); + + client + .send(commands::set("k", "1", Duration::from_secs(60))) + .await + .unwrap(); + + assert_eq!( + client + .send(commands::get::("k")) + .await + .unwrap() + .as_deref(), + Some("1") + ); +} + +#[rstest] +#[tokio::test] +async fn sentinel_rediscovers_with_the_credentials_the_provider_holds_now() { + let sentinel = RedisSentinel::start_with_password(DATA_PASSWORD) + .await + .unwrap(); + sentinel + .cli_on_data_nodes(&[ + "ACL", "SETUSER", "alice", "on", ">first", "~*", "&*", "+@all", + ]) + .await + .unwrap(); + sentinel + .cli_on_data_nodes(&[ + "ACL", "SETUSER", "bob", "on", ">second", "~*", "&*", "+@all", + ]) + .await + .unwrap(); + let provider = Arc::new(Rotating::new("alice", "first")); + let client = sentinel_client( + &sentinel, + Credentials::Provider(CredentialsProvider(provider.clone())), + ); + client + .send(commands::set("k", "1", Duration::from_secs(600))) + .await + .unwrap(); + + provider.rotate("bob", "second"); + sentinel + .cli_on_data_nodes(&["ACL", "DELUSER", "alice"]) + .await + .unwrap(); + sentinel + .cli_on_data_nodes(&["CLIENT", "KILL", "USER", "bob"]) + .await + .unwrap(); + let after_rediscovery = eventually(async || { + client + .send(commands::get::("k")) + .await + .ok() + .flatten() + }) + .await; + + assert_eq!(after_rediscovery, "1"); +} + +#[rstest] +#[tokio::test] +async fn a_timed_out_command_replaces_a_connection_that_stopped_answering( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let proxies = proxied(&server).await; + let client = RedisClient::new(RedisConfig { + target: proxies.target.clone(), + credentials: Credentials::NONE, + namespace: Namespace::default(), + timeouts: Timeouts::new(Duration::from_millis(500), None), + breaker: None, + events: None, + health_check: None, + keepalive: true, + }) + .unwrap(); + client + .send(commands::set("k", "1", Duration::from_secs(600))) + .await + .unwrap(); + + proxies.blackhole_open_connections(); + let stalled = client.send(commands::get::("k")).await; + let next = client.send(commands::get::("k")).await; + + assert!( + matches!(&stalled, Err(Error::Transport(error)) if error.is_timeout()), + "{stalled:?}" + ); + assert_eq!(next.unwrap().as_deref(), Some("1")); +} diff --git a/litellm-rust/crates/redis-client/tests/pipeline.rs b/litellm-rust/crates/redis-client/tests/pipeline.rs new file mode 100644 index 00000000000..5aed70b42dd --- /dev/null +++ b/litellm-rust/crates/redis-client/tests/pipeline.rs @@ -0,0 +1,362 @@ +mod support; + +use std::time::{Duration, Instant}; + +use litellm_redis_client::Value; +use litellm_redis_client::commands; +use litellm_redis_testing::{RedisDeployment, Topology}; +use rstest::rstest; + +use litellm_redis_client::{Amount, DEFAULT_TIMEOUT, Error, Expiry, Script}; +use support::connect; + +async fn begin_migration(cluster: &litellm_redis_testing::RedisCluster, key: &str) { + let (_, first) = cluster.nodes().next().unwrap(); + let slot = cluster + .cli_on_node(first, &["CLUSTER", "KEYSLOT", key]) + .await + .unwrap(); + let slot = slot.trim(); + let number: u16 = slot.parse().unwrap(); + let nodes = cluster + .cli_on_node(first, &["CLUSTER", "NODES"]) + .await + .unwrap(); + let owner = nodes.lines().find(|line| line.split_whitespace().skip(8).any(|range| { + let (start, end) = range.split_once('-').unwrap_or((range, range)); + matches!((start.parse::(), end.parse::()), (Ok(start), Ok(end)) if (start..=end).contains(&number)) + })).unwrap(); + let target = nodes.lines().find(|line| *line != owner).unwrap(); + let address = |line: &str| { + let value = line.split_whitespace().nth(1).unwrap(); + value + .split_once('@') + .unwrap() + .0 + .rsplit_once(':') + .unwrap() + .1 + .parse::() + .unwrap() + }; + let source_port = address(owner); + let target_port = address(target); + let source_id = owner.split_whitespace().next().unwrap(); + let target_id = target.split_whitespace().next().unwrap(); + assert_eq!( + cluster + .cli_on_node(source_port, &["GET", key]) + .await + .unwrap() + .trim(), + "0" + ); + assert_eq!( + cluster + .cli_on_node( + target_port, + &["CLUSTER", "SETSLOT", slot, "IMPORTING", source_id] + ) + .await + .unwrap() + .trim(), + "OK" + ); + assert_eq!( + cluster + .cli_on_node( + source_port, + &["CLUSTER", "SETSLOT", slot, "MIGRATING", target_id] + ) + .await + .unwrap() + .trim(), + "OK" + ); + let state = cluster + .cli_on_node(source_port, &["CLUSTER", "NODES"]) + .await + .unwrap(); + assert!( + state.contains(&format!("[{slot}->-{target_id}]")), + "{state}" + ); +} + +#[rstest] +#[tokio::test] +async fn a_retrying_neighbor_cannot_reapply_a_cluster_counter() { + let cluster = litellm_redis_testing::RedisCluster::start().await.unwrap(); + let server = RedisDeployment::Cluster(cluster); + let client = connect(&server, "", DEFAULT_TIMEOUT); + let counter = "{migration}:counter"; + client + .send(commands::set(counter, 0, Duration::from_secs(60))) + .await + .unwrap(); + let RedisDeployment::Cluster(cluster) = &server else { + unreachable!() + }; + begin_migration(cluster, counter).await; + let both_keys = Script::new("return redis.call('MGET', KEYS[1], KEYS[2])"); + + let results = client + .run( + &client + .pipeline() + .with(commands::incr_by_float( + counter, + Amount::Int(1), + Some(Expiry::Refresh(Duration::from_secs(60))), + )) + .with(commands::eval::( + &both_keys, + &[counter, "{migration}:missing"], + Vec::::new(), + )), + ) + .await; + + assert_eq!( + client.send(commands::get::(counter)).await.unwrap(), + Some(1) + ); + assert_eq!( + results[0].as_ref().unwrap(), + &Value::BulkString(b"1".to_vec()) + ); + assert!( + matches!(&results[1], Err(Error::Server(error)) if error.code() == Some("TRYAGAIN")), + "{:?}", + results[1] + ); +} + +#[rstest] +#[tokio::test] +async fn each_op_settles_on_its_own_in_input_order( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + client + .send(commands::set( + "text", + "not a number", + Duration::from_secs(60), + )) + .await + .unwrap(); + let pipeline = client + .pipeline() + .with(commands::incr_by_float( + "a", + Amount::Int(1), + Some(Expiry::Refresh(Duration::from_secs(60))), + )) + .with(commands::incr_by_float( + "text", + Amount::Int(1), + Some(Expiry::Refresh(Duration::from_secs(60))), + )) + .with(commands::incr_by_float( + "c", + Amount::Int(5), + Some(Expiry::Refresh(Duration::from_secs(60))), + )) + .with(commands::set("b", "2", Duration::from_secs(60))) + .with(commands::get::("b")) + .with(commands::incr_by_float( + "a", + Amount::Int(1), + Some(Expiry::Refresh(Duration::from_secs(60))), + )) + .with(commands::get::("missing")); + + let results = client.run(&pipeline).await; + + assert_eq!(results.len(), 7); + assert_eq!( + results[0].as_ref().unwrap(), + &Value::BulkString(b"1".to_vec()) + ); + assert!( + matches!(results[1], Err(Error::Server(_))), + "{:?}", + results[1] + ); + assert_eq!( + results[2].as_ref().unwrap(), + &Value::BulkString(b"5".to_vec()) + ); + assert_eq!(results[3].as_ref().unwrap(), &Value::Okay); + assert_eq!( + results[4].as_ref().unwrap(), + &Value::BulkString(b"2".to_vec()) + ); + assert_eq!( + results[5].as_ref().unwrap(), + &Value::BulkString(b"2".to_vec()) + ); + assert_eq!(results[6].as_ref().unwrap(), &Value::Nil); +} + +#[rstest] +#[case::cached(true, Value::BulkString(b"2".to_vec()))] +#[case::lost_by_the_server(false, Value::Nil)] +#[tokio::test] +async fn a_lost_script_reruns_after_the_rest_of_its_pipeline_like_redis_py( + #[case] cached: bool, + #[case] read_in_the_same_pipeline: Value, +) { + let server = RedisDeployment::start(Topology::Standalone).await.unwrap(); + let client = connect(&server, "ns", DEFAULT_TIMEOUT); + let incr = Script::new("return redis.call('INCRBY', KEYS[1], ARGV[1])"); + client + .send(commands::eval::(&incr, &["warmup"], 1)) + .await + .unwrap(); + if !cached { + server + .cli_on_every_node(&["SCRIPT", "FLUSH"]) + .await + .unwrap(); + } + let pipeline = client + .pipeline() + .with(commands::eval::(&incr, &["counter"], 2)) + .with(commands::get::("counter")); + + let results = client.run(&pipeline).await; + + assert_eq!(results[0].as_ref().unwrap(), &Value::Int(2)); + assert_eq!(results[1].as_ref().unwrap(), &read_in_the_same_pipeline); + assert_eq!( + client.send(commands::get::("counter")).await.unwrap(), + Some(2) + ); +} + +#[rstest] +#[tokio::test] +async fn an_unreachable_server_fails_every_op_as_transport( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", Duration::from_millis(500)); + let pipeline = client + .pipeline() + .with(commands::incr_by_float( + "a", + Amount::Int(1), + Some(Expiry::Refresh(Duration::from_secs(60))), + )) + .with(commands::incr_by_float( + "c", + Amount::Int(1), + Some(Expiry::Refresh(Duration::from_secs(60))), + )); + + server.freeze().await.unwrap(); + let results = client.run(&pipeline).await; + server.thaw().await.unwrap(); + + assert_eq!(results.len(), 2); + assert!( + matches!(results[0], Err(Error::Transport(_))), + "{:?}", + results[0] + ); + assert!( + matches!(results[1], Err(Error::Transport(_))), + "{:?}", + results[1] + ); +} + +#[rstest] +#[tokio::test] +async fn an_empty_pipeline_returns_without_asking_redis( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", Duration::from_secs(2)); + + server.freeze().await.unwrap(); + let started = Instant::now(); + let results = client.run(&client.pipeline()).await; + let elapsed = started.elapsed(); + server.thaw().await.unwrap(); + + assert!(results.is_empty()); + assert!(elapsed < Duration::from_secs(1), "{elapsed:?}"); +} + +#[rstest] +#[tokio::test] +async fn a_pipeline_sends_scripts_by_sha_and_reruns_only_the_ones_the_server_lost( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let deployment = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&deployment, "scripts", DEFAULT_TIMEOUT); + let count = Script::new("return redis.call('INCR', KEYS[1])"); + deployment + .cli_on_every_node(&["SCRIPT", "FLUSH"]) + .await + .unwrap(); + deployment + .cli_on_every_node(&["CONFIG", "RESETSTAT"]) + .await + .unwrap(); + + let first = client + .run( + &client + .pipeline() + .with(commands::incr_by_float( + "{a}spend", + Amount::Float(0.5), + Some(Expiry::Refresh(Duration::from_secs(60))), + )) + .with(commands::eval::( + &count, + &["{a}"], + Vec::::new(), + )) + .with(commands::eval::( + &count, + &["b"], + Vec::::new(), + )), + ) + .await; + let second = client + .run(&client.pipeline().with(commands::eval::( + &count, + &["{a}"], + Vec::::new(), + ))) + .await; + + let replies: Vec = first + .into_iter() + .chain(second) + .map(Result::unwrap) + .collect(); + assert_eq!( + replies, + [ + Value::BulkString(b"0.5".to_vec()), + Value::Int(1), + Value::Int(1), + Value::Int(2) + ] + ); + assert_eq!( + client.send(commands::get::("{a}spend")).await.unwrap(), + Some(0.5) + ); + let stats = support::command_stats(&deployment).await; + assert!(support::calls(&stats, "evalsha") > 0, "{stats}"); + assert_eq!(support::calls(&stats, "eval"), 0, "{stats}"); +} diff --git a/litellm-rust/crates/redis-client/tests/pubsub.rs b/litellm-rust/crates/redis-client/tests/pubsub.rs new file mode 100644 index 00000000000..2ad7cb38a5a --- /dev/null +++ b/litellm-rust/crates/redis-client/tests/pubsub.rs @@ -0,0 +1,189 @@ +mod support; + +use std::time::{Duration, Instant}; + +use futures_util::StreamExt; +use litellm_circuit_breaker::{Breaker, Config}; +use litellm_redis_client::commands; +use litellm_redis_client::{ + Credentials, DEFAULT_TIMEOUT, Error, Namespace, Node, RedisClient, RedisConfig, Sentinel, + Target, Timeouts, +}; +use litellm_redis_testing::{RedisDeployment, RedisSentinel, SERVICE_NAME, Topology, eventually}; +use rstest::rstest; +use support::{connect, connect_guarded}; +use tokio::sync::mpsc::{UnboundedReceiver, unbounded_channel}; + +const CHANNEL: &str = "litellm_proxy.auth_cache_invalidation"; + +fn listen(client: &RedisClient, channel: &str) -> UnboundedReceiver> { + let (sender, receiver) = unbounded_channel(); + let mut messages = client.subscribe(channel); + tokio::spawn(async move { + while let Some(message) = messages.next().await { + if sender.send(message).is_err() { + return; + } + } + }); + receiver +} + +async fn delivered( + publisher: &RedisClient, + channel: &str, + received: &mut UnboundedReceiver>, + payload: &[u8], +) { + eventually(async || { + publisher + .send(commands::publish(channel, payload)) + .await + .ok()?; + let deadline = tokio::time::Instant::now() + Duration::from_millis(200); + while let Ok(Some(message)) = tokio::time::timeout_at(deadline, received.recv()).await { + if message == payload { + return Some(()); + } + } + None + }) + .await; +} + +#[rstest] +#[tokio::test] +async fn a_subscriber_receives_raw_bytes_on_the_channel_name_without_the_namespace( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let publisher = connect(&server, "ns", DEFAULT_TIMEOUT); + let plain = connect(&server, "", DEFAULT_TIMEOUT); + let mut received = listen(&plain, CHANNEL); + + delivered( + &publisher, + CHANNEL, + &mut received, + b"{\"cache_key\":\"k\"}\xff", + ) + .await; +} + +#[rstest] +#[tokio::test] +async fn a_subscriber_resubscribes_after_redis_restarts( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", Duration::from_millis(500)); + let mut received = listen(&client, CHANNEL); + delivered(&client, CHANNEL, &mut received, b"before").await; + + server.restart().await.unwrap(); + + delivered(&client, CHANNEL, &mut received, b"after").await; +} + +#[rstest] +#[tokio::test] +async fn a_sentinel_subscriber_receives_from_the_master() { + let server = RedisSentinel::start().await.unwrap(); + let (host, port) = server.sentinel(); + let client = RedisClient::new(RedisConfig { + target: Target::Sentinel(Sentinel { + monitors: vec![Node { + host: host.to_owned(), + port, + }], + service_name: SERVICE_NAME.to_owned(), + monitor_auth: None, + tls: None, + db: 0, + }), + credentials: Credentials::NONE, + namespace: Namespace::default(), + timeouts: Timeouts::default(), + breaker: None, + events: None, + health_check: None, + keepalive: true, + }) + .unwrap(); + let mut received = listen(&client, CHANNEL); + + delivered(&client, CHANNEL, &mut received, b"payload").await; +} + +async fn publish_calls(server: &RedisDeployment) -> Vec { + let replies = match server { + RedisDeployment::Standalone(server) => { + vec![server.cli(&["INFO", "commandstats"]).await.unwrap()] + } + RedisDeployment::Cluster(cluster) => cluster + .cli_outputs(&["INFO", "commandstats"]) + .await + .unwrap(), + }; + replies + .iter() + .flat_map(|reply| reply.lines()) + .filter(|line| line.starts_with("cmdstat_publish:")) + .map(str::to_owned) + .collect() +} + +#[rstest] +#[tokio::test] +async fn an_open_breaker_rejects_a_publish_without_sending_it( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let breaker = Breaker::new( + Config { + failure_threshold: 2, + recovery_timeout: Duration::from_secs(60), + timeout_min_duration: Duration::ZERO, + }, + Box::new(|_| {}), + ); + let client = connect_guarded(&server, "", Duration::from_millis(300), Some(breaker)); + + server.freeze().await.unwrap(); + let opened = [client.ping().await, client.ping().await]; + let started = Instant::now(); + let rejected = client.send(commands::publish(CHANNEL, b"payload")).await; + let rejected_in = started.elapsed(); + server.thaw().await.unwrap(); + + assert!( + opened + .iter() + .all(|ping| matches!(ping, Err(Error::Transport(_)))), + "{opened:?}" + ); + assert!(matches!(rejected, Err(Error::BreakerOpen)), "{rejected:?}"); + assert!(rejected_in < Duration::from_millis(50), "{rejected_in:?}"); + assert_eq!(publish_calls(&server).await, Vec::::new()); +} + +// redis-py's PubSub subscribes on a RESP2 connection, so it never sends HELLO +#[rstest] +#[tokio::test] +async fn a_subscription_speaks_resp2_like_redis_py( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + server + .cli_on_every_node(&["CONFIG", "RESETSTAT"]) + .await + .unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + let mut received = listen(&client, CHANNEL); + + delivered(&client, CHANNEL, &mut received, b"evict").await; + + let stats = support::command_stats(&server).await; + assert!(support::calls(&stats, "subscribe") > 0, "{stats}"); + assert_eq!(support::calls(&stats, "hello"), 0, "{stats}"); +} diff --git a/litellm-rust/crates/redis-client/tests/script.rs b/litellm-rust/crates/redis-client/tests/script.rs new file mode 100644 index 00000000000..ea6d1c5ff78 --- /dev/null +++ b/litellm-rust/crates/redis-client/tests/script.rs @@ -0,0 +1,133 @@ +mod support; + +use std::time::Duration; + +use litellm_redis_testing::{RedisDeployment, Topology}; +use rstest::rstest; + +use litellm_redis_client::Script; +use litellm_redis_client::commands; +use litellm_redis_client::{DEFAULT_TIMEOUT, Error, Ttl}; +use support::connect; + +#[rstest] +#[tokio::test] +async fn script_keys_are_namespaced( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "a", DEFAULT_TIMEOUT); + let bare = connect(&server, "", DEFAULT_TIMEOUT); + let read = Script::new("return redis.call('GET', KEYS[1])"); + bare.send(commands::set("a:x", "1", Duration::from_secs(60))) + .await + .unwrap(); + + let value: Option = client + .send(commands::eval(&read, &["x"], Vec::::new())) + .await + .unwrap(); + + assert_eq!(value.as_deref(), Some("1")); +} + +#[rstest] +#[tokio::test] +async fn a_failing_script_is_a_server_error_and_keeps_the_connection( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + let incr = Script::new("return redis.call('INCRBY', KEYS[1], ARGV[1])"); + client + .send(commands::set("list", "item", Duration::from_secs(60))) + .await + .unwrap(); + + let failed = client + .send(commands::eval::(&incr, &["list"], 1)) + .await; + + assert!(matches!(failed, Err(Error::Server(_))), "{failed:?}"); + assert_eq!( + client + .send(commands::eval::(&incr, &["counter"], 3)) + .await + .unwrap(), + 3 + ); +} + +#[rstest] +#[tokio::test] +async fn a_flushed_script_is_loaded_again( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "", DEFAULT_TIMEOUT); + let incr = Script::new("return redis.call('INCRBY', KEYS[1], ARGV[1])"); + client + .send(commands::eval::(&incr, &["counter"], 1)) + .await + .unwrap(); + + server + .cli_on_every_node(&["SCRIPT", "FLUSH"]) + .await + .unwrap(); + + assert_eq!( + client + .send(commands::eval::(&incr, &["counter"], 1)) + .await + .unwrap(), + 2 + ); +} + +const INCREMENT_WITH_FLOOR: &str = "local count = redis.call('INCRBY', KEYS[1], ARGV[1]) \ +if count < 0 then count = redis.call('INCRBY', KEYS[1], -count) end \ +if redis.call('TTL', KEYS[1]) < 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]) end \ +return count"; + +#[rstest] +#[tokio::test] +async fn pythons_increment_with_floor_script_floors_at_zero_and_expires_in_one_call( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + #[values(false, true)] pipelined: bool, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + let client = connect(&server, "ns", DEFAULT_TIMEOUT); + let floor = Script::new(INCREMENT_WITH_FLOOR); + server + .cli_on_every_node(&["SCRIPT", "FLUSH"]) + .await + .unwrap(); + + let count = match pipelined { + false => client + .send(commands::eval::(&floor, &["slots"], (-5, 60))) + .await + .unwrap(), + true => { + let results = client + .run(&client.pipeline().with(commands::eval::( + &floor, + &["slots"], + (-5, 60), + ))) + .await; + redis::from_redis_value(results[0].as_ref().unwrap().clone()).unwrap() + } + }; + + assert_eq!(count, 0); + assert_eq!( + client.send(commands::get::("slots")).await.unwrap(), + Some(0) + ); + assert!(matches!( + client.send(commands::ttl("slots")).await.unwrap(), + Ttl::Expires(left) if left > Duration::from_secs(50) && left <= Duration::from_secs(60) + )); +} diff --git a/litellm-rust/crates/redis-client/tests/support/mod.rs b/litellm-rust/crates/redis-client/tests/support/mod.rs new file mode 100644 index 00000000000..9cf209dce4a --- /dev/null +++ b/litellm-rust/crates/redis-client/tests/support/mod.rs @@ -0,0 +1,206 @@ +#![allow(dead_code)] // each test binary compiles this module on its own and uses a different subset + +use std::{pin::Pin, time::Duration}; + +use futures_util::Stream; +use litellm_circuit_breaker::Breaker; +use litellm_redis_client::{ + BasicAuth, Cluster, Credentials, Expiry, Namespace, Node, RedisClient, RedisConfig, RedisError, + RedisUrl, Standalone, StreamingCredentialsProvider, Target, Timeouts, +}; +use litellm_redis_testing::{ClientConnection, RedisDeployment, TcpProxy}; +use litellm_secret_value::SecretValue; +use tokio::sync::watch; + +pub const KEEP: Expiry = Expiry::KeepExisting(Duration::from_secs(60)); + +pub fn endpoint(deployment: &RedisDeployment) -> Target { + match deployment { + RedisDeployment::Standalone(server) => Target::Url(RedisUrl { + url: SecretValue::new(server.url()), + tls: None, + db: None, + username: None, + password: None, + }), + RedisDeployment::Cluster(cluster) => Target::Cluster(Cluster { + nodes: cluster + .nodes() + .map(|(host, port)| Node { + host: host.to_owned(), + port, + }) + .collect(), + tls: None, + }), + } +} + +pub fn connect(deployment: &RedisDeployment, namespace: &str, timeout: Duration) -> RedisClient { + connect_guarded(deployment, namespace, timeout, None) +} + +pub fn connect_guarded( + deployment: &RedisDeployment, + namespace: &str, + timeout: Duration, + breaker: Option, +) -> RedisClient { + RedisClient::new(RedisConfig { + target: endpoint(deployment), + credentials: Credentials::NONE, + namespace: Namespace::new(namespace), + timeouts: Timeouts::new(timeout, None), + breaker, + events: None, + health_check: None, + keepalive: true, + }) + .unwrap() +} + +pub fn connect_as(deployment: &RedisDeployment, credentials: Credentials) -> RedisClient { + RedisClient::new(RedisConfig { + target: endpoint(deployment), + credentials, + namespace: Namespace::default(), + timeouts: Timeouts::new(Duration::from_secs(2), None), + breaker: None, + events: None, + health_check: None, + keepalive: true, + }) + .unwrap() +} + +pub async fn connections(deployment: &RedisDeployment) -> Vec { + deployment.clients().await.unwrap() +} + +/// A `TcpProxy` in front of every node, and a target that reaches the nodes only through them. A +/// cluster announces the proxy ports, so its slot map and redirects lead there too. +pub struct Proxied { + proxies: Vec, + pub target: Target, +} + +impl Proxied { + pub fn blackhole_open_connections(&self) { + self.proxies + .iter() + .for_each(TcpProxy::blackhole_open_connections); + } +} + +pub async fn proxied(deployment: &RedisDeployment) -> Proxied { + match deployment { + RedisDeployment::Standalone(server) => { + let proxy = TcpProxy::start(server.port()).await.unwrap(); + let target = Target::Standalone(Standalone { + host: "127.0.0.1".to_owned(), + port: proxy.port(), + tls: None, + db: 0, + }); + Proxied { + proxies: vec![proxy], + target, + } + } + RedisDeployment::Cluster(cluster) => { + let mut proxies = Vec::new(); + for (_, port) in cluster.nodes() { + proxies.push(TcpProxy::start(port).await.unwrap()); + } + let ports: [u16; 3] = std::array::from_fn(|node| proxies[node].port()); + cluster.announce_ports(ports).await.unwrap(); + let target = Target::Cluster(Cluster { + nodes: ports + .iter() + .map(|&port| Node { + host: "127.0.0.1".to_owned(), + port, + }) + .collect(), + tls: None, + }); + Proxied { proxies, target } + } + } +} + +/// `INFO

` from every node, concatenated. +async fn info(deployment: &RedisDeployment, section: &str) -> String { + match deployment { + RedisDeployment::Standalone(server) => server.cli(&["INFO", section]).await.unwrap(), + RedisDeployment::Cluster(cluster) => { + let mut stats = String::new(); + for (_, port) in cluster.nodes() { + stats.push_str(&cluster.cli_on_node(port, &["INFO", section]).await.unwrap()); + } + stats + } + } +} + +/// `INFO commandstats` from every node, so a test can count what was actually sent. +pub async fn command_stats(deployment: &RedisDeployment) -> String { + info(deployment, "commandstats").await +} + +/// Keys looked up on every node since the last `CONFIG RESETSTAT`, found or not. +pub async fn keys_read(deployment: &RedisDeployment) -> u64 { + let stats = info(deployment, "stats").await; + ["keyspace_hits:", "keyspace_misses:"] + .iter() + .flat_map(|field| { + stats + .lines() + .filter_map(move |line| line.trim().strip_prefix(field)?.parse::().ok()) + }) + .sum() +} + +/// How many times `command` reached a server, summed across the nodes in `stats`. +pub fn calls(stats: &str, command: &str) -> u64 { + let prefix = format!("cmdstat_{command}:calls="); + stats + .lines() + .filter_map(|line| line.trim().strip_prefix(&prefix)) + .filter_map(|rest| rest.split(',').next()?.parse::().ok()) + .sum() +} + +pub struct Rotating(watch::Sender); + +impl Rotating { + pub fn new(username: &str, password: &str) -> Self { + Self(watch::Sender::new(BasicAuth::new( + username.to_owned(), + password.to_owned(), + ))) + } + + pub fn rotate(&self, username: &str, password: &str) { + self.0 + .send_replace(BasicAuth::new(username.to_owned(), password.to_owned())); + } +} + +impl StreamingCredentialsProvider for Rotating { + fn subscribe( + &self, + ) -> Pin> + Send + 'static>> { + let receiver = self.0.subscribe(); + Box::pin(futures_util::stream::unfold( + (receiver, true), + |(mut receiver, first)| async move { + if !first { + receiver.changed().await.ok()?; + } + let credentials = receiver.borrow_and_update().clone(); + Some((Ok(credentials), (receiver, false))) + }, + )) + } +} diff --git a/litellm-rust/crates/redis-connect/AGENTS.md b/litellm-rust/crates/redis-connect/AGENTS.md new file mode 100644 index 00000000000..8e9d1302eaf --- /dev/null +++ b/litellm-rust/crates/redis-connect/AGENTS.md @@ -0,0 +1,11 @@ +- Turns a `litellm_redis_types::Resolution` into a `RedisClient`, the same way for every host. It depends on `litellm-redis-types`, never on `litellm-config`, so a host that resolves Redis some other way connects the same way. `redis-client` stays cloud-agnostic, so credential providers and cloud token sources are built here +- The host passes only what it alone can build in `HostServices`: the event sink, the HTTP client and the environment lookup. The breaker config comes from `Resolution::breaker`, which `config` reads from `REDIS_CIRCUIT_BREAKER_*` with Python's fallback, so a bad value keeps the default instead of failing connect. Each cloud source asks its auth crate for one explicit identity, the way the Python SDK builds it: `AwsAuthService::default_chain_credentials` (botocore's default chain), `CredentialSource::application_default` (`google.auth.default`) and `AzureIdentity` (the class `_build_azure_credential` picks). LiteLLM's provider settings such as `AWS_ROLE_NAME`, `AZURE_AD_TOKEN` or `VERTEXAI_CREDENTIALS` belong to Bedrock, Azure OpenAI and Vertex and never reach those constructors. `Auth::Provider` (a handle for a Python `credential_provider` object) fails at connect with an error naming the key, since no host can supply one yet +- `RedisHosts` is the one place a host shares Redis state. Breakers are keyed by `Endpoint`, so every caller on a server sees one health signal whatever its db or credentials. A client takes only breaker settings and admits through its server's breaker, so no caller can pair a client with a breaker of its own, and clients are keyed by `Resolution` alone. Both are held weakly, so a breaker or client is freed with its last user, and the lock is held while a client connects so each is built once. A caller holding the Python GIL releases it first. Hosts own one instance (python-bridge per process, the gateway per `connect`) and never keep a registry of their own +- Every expiring credential goes through the one refresh loop in `refresh.rs`. A cloud source only fetches. Never add a second loop or let a source sleep. The expiry deadline lives only in the published `Latest::Ready`, so expiring it leaves `Ready` and cannot fire again. Never keep a copy of the deadline in the loop +- AWS SDK identity settings must be in the process environment. `aws-config` cannot take the host lookup, so config-only overrides for profiles, role/web-identity settings, credentials, files and metadata sources fail before connecting rather than silently using another identity. Equal process/config values are allowed. Bedrock-only settings remain unrelated to Redis +- redis-rs subscribes once for the connection handshake and again for re-authentication, so a new connection sends a redundant initial AUTH. Unchanged refreshes do not trigger another AUTH. Do not skip a provider's first or alternating subscriptions, because concurrent connections and retries each need the current token. Removing the initial duplicate requires an upstream connection-level fix +- The loop starts on the first `subscribe()`, which redis-rs calls inside its runtime, so `connect` is safe off a runtime +- A token-only credential (Azure without `REDIS_USERNAME`, GCP) is sent as `AUTH default `, because redis-rs always sends a username with a credentials provider. On Redis 6+ that is the same as Python's `AUTH ` +- Tests for the refresh loop use a fake source and real Redis ACL users. Cloud sources are proven against the live service, unit tests only pin request shapes we own +- Rotation tests must retain the same server-side connection IDs across re-AUTH, then force a reconnect after removing the old user and verify the new identity still works +- Live cloud tests live in `tests/live/`. Set `LITELLM_REDIS_AWS_LIVE_CONFIG`, `LITELLM_REDIS_AZURE_LIVE_CONFIG` or `LITELLM_REDIS_GCP_LIVE_CONFIG` to a JSON file containing `redis` settings and `hold_seconds`. Set the waiting period from the deployment's token lifetime when checking expiry. Optional fixtures return early when unset, and configured tests run under plain `cargo test`. An unconfigured run never counts as live proof diff --git a/litellm-rust/crates/redis-connect/Cargo.toml b/litellm-rust/crates/redis-connect/Cargo.toml new file mode 100644 index 00000000000..29b3293b856 --- /dev/null +++ b/litellm-rust/crates/redis-connect/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "litellm-redis-connect" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-auth-aws.workspace = true +litellm-auth-azure.workspace = true +litellm-auth-gcp.workspace = true +litellm-auth-types.workspace = true +litellm-error.workspace = true +litellm-secret-value.workspace = true +strum.workspace = true +litellm-circuit-breaker.workspace = true +litellm-redis-types.workspace = true +litellm-core-utils.workspace = true +litellm-http.workspace = true +litellm-redis-client.workspace = true +futures-util.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +veil.workspace = true +time.workspace = true +url.workspace = true +tokio = { workspace = true, features = ["sync", "time", "macros"] } +tracing.workspace = true + +[dev-dependencies] +litellm-redis-resolve.workspace = true +litellm-http = { workspace = true, features = ["test-support"] } +litellm-redis-testing.workspace = true +rustls.workspace = true +rstest.workspace = true +serde_json.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] } diff --git a/litellm-rust/crates/redis-connect/src/connect.rs b/litellm-rust/crates/redis-connect/src/connect.rs new file mode 100644 index 00000000000..d11f6ba47ff --- /dev/null +++ b/litellm-rust/crates/redis-connect/src/connect.rs @@ -0,0 +1,152 @@ +use std::{sync::Arc, time::Duration}; + +use litellm_circuit_breaker::{Breaker, Config as BreakerConfig}; +use litellm_core_utils::settings::Lookup; +use litellm_redis_client::{ + Credentials, CredentialsProvider, DEFAULT_TIMEOUT, Namespace, RedisClient, RedisConfig, + RedisEvents, Target, Timeouts, +}; +use litellm_redis_types::{Auth, HealthCheck, Resolution, StaticAuth}; + +use crate::{ + CredentialSource, Error, RefreshingCredentials, + sources::{AzureAd, ElastiCacheIam, GcpIam}, +}; + +pub type SharedEnvironment = Arc; + +pub struct HostServices { + pub events: Option>, + pub http: litellm_http::Client, + pub environment: SharedEnvironment, + pub credential_provider: Option>, +} + +// litellm/constants.py REDIS_CLUSTER_HEALTH_CHECK_INTERVAL, the default _redis.py sets for cluster clients only +const CLUSTER_HEALTH_CHECK_INTERVAL: Duration = Duration::from_secs(25); + +/// A breaker whose transitions reach `events`, for every client that talks to one Redis. +pub(crate) fn breaker(config: BreakerConfig, events: Option>) -> Breaker { + Breaker::new( + config, + Box::new(move |transition| { + if let Some(events) = &events { + events.breaker(transition); + } + }), + ) +} + +pub fn connect( + resolved: Resolution, + host: HostServices, + breaker: Option, +) -> Result { + tracing::info!(target = ?resolved.target, namespace = ?resolved.namespace, "redis client configured"); + let credentials = credentials(resolved.auth, &host)?; + let events = host.events; + let health_check = health_check(&resolved.target, resolved.health_check); + Ok(RedisClient::new(RedisConfig { + target: resolved.target, + credentials, + namespace: resolved.namespace.map(Namespace::new).unwrap_or_default(), + timeouts: Timeouts::new( + resolved.timeout.unwrap_or(DEFAULT_TIMEOUT), + resolved.connect_timeout, + ), + breaker, + events, + health_check, + keepalive: resolved.keepalive, + })?) +} + +fn health_check(target: &Target, configured: HealthCheck) -> Option { + match (target, configured) { + (_, HealthCheck::Every(interval)) => Some(interval), + (_, HealthCheck::Disabled) => None, + (Target::Cluster(_), HealthCheck::Unset) => Some(CLUSTER_HEALTH_CHECK_INTERVAL), + (_, HealthCheck::Unset) => None, + } +} + +fn credentials(auth: Auth, host: &HostServices) -> Result { + let refreshing = |provider: RefreshingCredentials| { + Credentials::Provider(CredentialsProvider(Arc::new(provider))) + }; + Ok(match auth { + Auth::Static(StaticAuth { username, password }) => { + Credentials::Static { username, password } + } + Auth::Provider(_) => match &host.credential_provider { + Some(source) => refreshing(RefreshingCredentials::new(Arc::clone(source))), + None => return Err(Error::NoHostCredentialProvider), + }, + Auth::AzureAd(auth) => refreshing(RefreshingCredentials::new(AzureAd::new( + auth, + host.environment.clone(), + ))), + Auth::AwsIam(auth) => refreshing(RefreshingCredentials::new(ElastiCacheIam::new( + auth, + &*host.environment, + )?)), + Auth::GcpIam { service_account } => refreshing(RefreshingCredentials::new(GcpIam::new( + host.http.clone(), + service_account, + host.environment.clone(), + ))), + }) +} + +#[cfg(test)] +mod tests { + use litellm_redis_client::{Cluster, Node, Standalone}; + use rstest::rstest; + + use super::*; + + fn cluster() -> Target { + Target::Cluster(Cluster { + nodes: vec![Node { + host: "node".to_owned(), + port: 7000, + }], + tls: None, + }) + } + + fn standalone() -> Target { + Target::Standalone(Standalone { + host: "redis".to_owned(), + port: 6379, + tls: None, + db: 0, + }) + } + + #[rstest] + #[case::an_unset_cluster_takes_the_python_default( + cluster(), + HealthCheck::Unset, + Some(CLUSTER_HEALTH_CHECK_INTERVAL) + )] + #[case::zero_keeps_a_cluster_off(cluster(), HealthCheck::Disabled, None)] + #[case::an_unset_standalone_stays_off(standalone(), HealthCheck::Unset, None)] + #[case::a_configured_interval_wins( + cluster(), + HealthCheck::Every(Duration::from_secs(5)), + Some(Duration::from_secs(5)) + )] + #[case::a_negative_interval_checks_every_command( + standalone(), + HealthCheck::Every(Duration::ZERO), + Some(Duration::ZERO) + )] + fn health_checks_follow_redis_py( + #[case] target: Target, + #[case] configured: HealthCheck, + #[case] expected: Option, + ) { + assert_eq!(health_check(&target, configured), expected); + } +} diff --git a/litellm-rust/crates/redis-connect/src/error.rs b/litellm-rust/crates/redis-connect/src/error.rs new file mode 100644 index 00000000000..98343a03fb8 --- /dev/null +++ b/litellm-rust/crates/redis-connect/src/error.rs @@ -0,0 +1,102 @@ +#[derive(veil::Redact, thiserror::Error)] +pub enum Error { + #[error(transparent)] + Connect(#[from] litellm_redis_client::ConnectError), + #[error( + "Redis setting credential_provider names a Python object, which this host cannot supply" + )] + NoHostCredentialProvider, + #[error( + "Redis credential setting {setting} must be set in the process environment, not only environment_variables" + )] + ProcessEnvironmentRequired { setting: &'static str }, + #[error("{stage} failed{}", http_status_suffix(*http_status))] + Credential { + stage: CredentialStage, + http_status: Option, + #[source] + #[redact(with = "[REDACTED]")] + cause: litellm_error::Source, + }, + #[error("credential acquisition exceeded {timeout:?}")] + CredentialTimeout { + timeout: std::time::Duration, + #[source] + cause: tokio::time::error::Elapsed, + }, + #[error("Redis credential has expired")] + ExpiredCredential, + #[error("no {provider} credential is available for Redis")] + MissingCredential { provider: &'static str }, +} + +#[derive(Clone, Copy, Debug, strum::Display, strum::VariantArray)] +pub enum CredentialStage { + #[strum(serialize = "caller identity")] + Identity, + #[strum(serialize = "Redis token signing")] + Signing, + #[strum(serialize = "Redis token exchange")] + Exchange, + #[strum(serialize = "Redis token response")] + Response, + #[strum(serialize = "Redis credential_provider")] + Provider, +} + +impl Error { + pub fn credential( + stage: CredentialStage, + ) -> impl FnOnce(E) -> Self { + move |cause| Self::Credential { + stage, + http_status: None, + cause: litellm_error::Source::new(cause), + } + } + + pub(crate) fn credential_response( + stage: CredentialStage, + status: u16, + ) -> impl FnOnce(litellm_http::Error) -> Self { + move |cause| Self::Credential { + stage, + http_status: Some(status), + cause: litellm_error::Source::new(cause), + } + } + + pub(crate) fn upstream_status(&self) -> Option { + match self { + Self::Credential { http_status, .. } => *http_status, + Self::Connect(_) + | Self::NoHostCredentialProvider + | Self::ProcessEnvironmentRequired { .. } + | Self::CredentialTimeout { .. } + | Self::ExpiredCredential + | Self::MissingCredential { .. } => None, + } + } + + pub(crate) fn kind(&self) -> &'static str { + match self { + Self::Connect(_) => "connection", + Self::NoHostCredentialProvider => "host-credential-provider", + Self::ProcessEnvironmentRequired { .. } => "process-environment-required", + Self::Credential { stage, .. } => match stage { + CredentialStage::Identity => "caller-identity", + CredentialStage::Signing => "token-signing", + CredentialStage::Exchange => "token-exchange", + CredentialStage::Response => "token-response", + CredentialStage::Provider => "credential-provider", + }, + Self::CredentialTimeout { .. } => "acquisition-timeout", + Self::ExpiredCredential => "expired-credential", + Self::MissingCredential { .. } => "missing-credential", + } + } +} + +fn http_status_suffix(status: Option) -> String { + status.map_or_else(String::new, |status| format!(" (HTTP {status})")) +} diff --git a/litellm-rust/crates/redis-connect/src/events.rs b/litellm-rust/crates/redis-connect/src/events.rs new file mode 100644 index 00000000000..a556b7d370b --- /dev/null +++ b/litellm-rust/crates/redis-connect/src/events.rs @@ -0,0 +1,129 @@ +use std::{ + sync::Mutex, + time::{Duration, Instant}, +}; + +use litellm_circuit_breaker::{State, Transition}; +use litellm_core_utils::settings::Lookup; +use litellm_redis_client::{CallEvent, FailureKind, Outcome, RedisEvents}; + +const DEFAULT_TIMEOUT_LOG_INTERVAL: Duration = Duration::from_secs(5); + +pub struct TracingEvents { + throttle: TimeoutLogThrottle, +} + +impl TracingEvents { + pub fn new(lookup: &dyn Lookup) -> Self { + let interval = lookup + .get("REDIS_TIMEOUT_LOG_INTERVAL") + .and_then(|seconds| seconds.parse::().ok()) + .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok()) + .unwrap_or(DEFAULT_TIMEOUT_LOG_INTERVAL); + Self { + throttle: TimeoutLogThrottle::new(interval), + } + } +} + +impl RedisEvents for TracingEvents { + fn call(&self, event: &CallEvent) { + let Outcome::Failed(kind) = event.outcome else { + return; + }; + match kind { + FailureKind::BreakerOpen => { + tracing::debug!(operation = ?event.operation, "redis call rejected by the open circuit breaker") + } + FailureKind::Timeout => match self.throttle.admit(Instant::now()) { + None => { + tracing::debug!(operation = ?event.operation, duration = ?event.duration, "redis call timed out") + } + Some(0) => { + tracing::error!(operation = ?event.operation, duration = ?event.duration, "redis call timed out") + } + Some(suppressed) => tracing::error!( + operation = ?event.operation, + duration = ?event.duration, + suppressed, + "redis call timed out, more redis timeouts since the previous line were logged at debug" + ), + }, + FailureKind::Transport | FailureKind::Server | FailureKind::Decode => { + tracing::error!(operation = ?event.operation, failure = ?kind, "redis call failed") + } + } + } + + fn breaker(&self, transition: &Transition) { + match transition.to { + State::Open { .. } => tracing::warn!( + generation = transition.generation, + "redis circuit breaker opened, fast-failing redis calls" + ), + State::HalfOpen => tracing::info!( + generation = transition.generation, + "redis circuit breaker half open, probing redis" + ), + State::Closed { .. } => tracing::info!( + generation = transition.generation, + "redis circuit breaker closed, redis recovered" + ), + } + } +} + +struct TimeoutLogThrottle { + interval: Duration, + state: Mutex<(Option, u64)>, +} + +impl TimeoutLogThrottle { + fn new(interval: Duration) -> Self { + Self { + interval, + state: Mutex::new((None, 0)), + } + } + + fn admit(&self, now: Instant) -> Option { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let (last_logged_at, suppressed) = &mut *state; + if last_logged_at.is_some_and(|last| now.duration_since(last) < self.interval) { + *suppressed += 1; + return None; + } + *last_logged_at = Some(now); + Some(std::mem::take(suppressed)) + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use rstest::rstest; + + use super::TimeoutLogThrottle; + + #[rstest] + fn admits_one_line_per_interval_and_counts_the_rest() { + let throttle = TimeoutLogThrottle::new(Duration::from_secs(5)); + let start = Instant::now(); + let at = |seconds: u64| start + Duration::from_secs(seconds); + + let admitted = [ + throttle.admit(at(0)), + throttle.admit(at(1)), + throttle.admit(at(4)), + throttle.admit(at(5)), + throttle.admit(at(6)), + throttle.admit(at(20)), + ]; + + assert_eq!(admitted, [Some(0), None, None, Some(2), None, Some(1)]); + } +} diff --git a/litellm-rust/crates/redis-connect/src/hosts.rs b/litellm-rust/crates/redis-connect/src/hosts.rs new file mode 100644 index 00000000000..11e69d45aff --- /dev/null +++ b/litellm-rust/crates/redis-connect/src/hosts.rs @@ -0,0 +1,241 @@ +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use litellm_circuit_breaker::{Breaker, Config as BreakerConfig, WeakBreaker}; +use litellm_redis_client::{RedisClient, RedisEvents, WeakRedisClient}; +use litellm_redis_types::{Endpoint, Resolution}; + +use crate::{Error, HostServices, breaker, connect}; + +/// The Redis servers one host talks to. Every caller on a server shares its breaker, and callers with the same +/// settings share one client, which admits through its server's breaker. Both are held weakly, so each is freed +/// with its last user and the next caller builds a fresh one. +#[derive(Default)] +pub struct RedisHosts { + breakers: Mutex, WeakBreaker)>>, + clients: Mutex>, +} + +#[derive(PartialEq)] +struct ClientKey { + resolved: Resolution, + guarded: bool, +} + +impl RedisHosts { + /// The breaker of the server `endpoint` names, built from `config` by its first caller. Without an endpoint + /// the breaker is this caller's own, which `breakers` still lists. + pub fn breaker( + &self, + endpoint: Option, + config: BreakerConfig, + events: Option>, + ) -> Breaker { + let mut breakers = live(&self.breakers, WeakBreaker::upgrade); + if let Some(shared) = breakers + .iter() + .filter(|(key, _)| key.is_some() && *key == endpoint) + .find_map(|(_, weak)| weak.upgrade()) + { + return shared; + } + let built = breaker(config, events); + breakers.push((endpoint, built.downgrade())); + built + } + + /// The client for `resolved`, connected by its first caller and guarded by its server's breaker unless `breaker` + /// is none. The lock is held while it connects, so concurrent callers with the same settings connect once. + pub fn client( + &self, + resolved: Resolution, + host: HostServices, + breaker: Option, + ) -> Result { + let breaker = breaker + .map(|config| self.breaker(resolved.target.endpoint(), config, host.events.clone())); + let key = ClientKey { + resolved, + guarded: breaker.is_some(), + }; + let mut clients = live(&self.clients, WeakRedisClient::upgrade); + if let Some(shared) = clients + .iter() + .filter(|(entry, _)| *entry == key) + .find_map(|(_, weak)| weak.upgrade()) + { + return Ok(shared); + } + let client = connect(key.resolved.clone(), host, breaker)?; + clients.push((key, client.downgrade())); + Ok(client) + } + + /// Every breaker a caller still holds, for a state gauge. + pub fn breakers(&self) -> Vec { + live(&self.breakers, WeakBreaker::upgrade) + .iter() + .filter_map(|(_, weak)| weak.upgrade()) + .collect() + } +} + +fn live( + entries: &Mutex>, + upgrade: fn(&W) -> Option, +) -> MutexGuard<'_, Vec<(K, W)>> { + let mut entries = entries.lock().unwrap_or_else(PoisonError::into_inner); + entries.retain(|(_, weak)| upgrade(weak).is_some()); + entries +} + +#[cfg(test)] +mod tests { + use std::{ + sync::Arc, + time::{Duration, Instant}, + }; + + use futures_util::FutureExt; + use litellm_circuit_breaker::{Config, Failure, State}; + use litellm_redis_types::{ + Auth, Endpoint, HealthCheck, Resolution, Standalone, StaticAuth, Target, + }; + use rstest::rstest; + + use super::RedisHosts; + use crate::HostServices; + + const CONFIG: Config = Config { + failure_threshold: 1, + recovery_timeout: Duration::from_secs(60), + timeout_min_duration: Duration::ZERO, + }; + + fn target(host: &str, db: i64) -> Target { + Target::Standalone(Standalone { + host: host.to_owned(), + port: 6379, + tls: None, + db, + }) + } + + fn endpoint(host: &str, db: i64) -> Option { + target(host, db).endpoint() + } + + fn resolved(host: &str, namespace: Option<&str>) -> Resolution { + Resolution { + target: target(host, 0), + auth: Auth::Static(StaticAuth::default()), + namespace: namespace.map(str::to_owned), + timeout: None, + connect_timeout: None, + health_check: HealthCheck::Unset, + keepalive: true, + ignored: Vec::new(), + } + } + + fn services() -> HostServices { + HostServices { + events: None, + http: litellm_http::Client::for_test(), + environment: Arc::new(|_: &str| None), + credential_provider: None, + } + } + + fn open(breaker: &litellm_circuit_breaker::Breaker) { + breaker.failed(Failure::Hard, Instant::now()); + } + + #[rstest] + fn callers_on_one_server_share_its_breaker_whatever_their_db() { + let hosts = RedisHosts::default(); + let first = hosts.breaker(endpoint("r1", 0), CONFIG, None); + let second = hosts.breaker(endpoint("r1", 1), CONFIG, None); + let other = hosts.breaker(endpoint("r2", 0), CONFIG, None); + + open(&first); + + assert!(matches!(second.state(), State::Open { .. })); + assert!(matches!(other.state(), State::Closed { .. })); + } + + #[rstest] + fn a_breaker_without_an_endpoint_is_listed_but_never_shared() { + let hosts = RedisHosts::default(); + let own = hosts.breaker(None, CONFIG, None); + let shared = hosts.breaker(endpoint("r1", 0), CONFIG, None); + + open(&own); + + assert!(matches!(shared.state(), State::Closed { .. })); + assert_eq!(hosts.breakers().len(), 2); + } + + #[rstest] + fn a_server_whose_callers_are_all_gone_gets_a_fresh_breaker() { + let hosts = RedisHosts::default(); + let first = hosts.breaker(endpoint("r1", 0), CONFIG, None); + open(&first); + drop(first); + + let again = hosts.breaker(endpoint("r1", 0), CONFIG, None); + + assert!(matches!(again.state(), State::Closed { .. })); + assert_eq!(hosts.breakers().len(), 1); + } + + #[rstest] + #[tokio::test] + async fn callers_with_the_same_settings_share_one_client() { + let hosts = RedisHosts::default(); + let first = hosts + .client(resolved("r1", Some("a")), services(), Some(CONFIG)) + .unwrap(); + + let same = hosts + .client(resolved("r1", Some("a")), services(), Some(CONFIG)) + .unwrap(); + let other_namespace = hosts + .client(resolved("r1", Some("b")), services(), Some(CONFIG)) + .unwrap(); + + assert!(first.same_client(&same)); + assert!(!first.same_client(&other_namespace)); + } + + #[rstest] + #[tokio::test] + async fn a_client_admits_through_its_servers_breaker() { + let hosts = RedisHosts::default(); + let breaker = hosts.breaker(endpoint("r1", 1), CONFIG, None); + let client = hosts + .client(resolved("r1", None), services(), Some(CONFIG)) + .unwrap(); + + open(&breaker); + + assert!(matches!( + client.ping().now_or_never(), + Some(Err(litellm_redis_client::Error::BreakerOpen)) + )); + } + + #[rstest] + #[tokio::test] + async fn a_client_without_a_breaker_is_never_shared_with_a_guarded_one() { + let hosts = RedisHosts::default(); + let guarded = hosts + .client(resolved("r1", None), services(), Some(CONFIG)) + .unwrap(); + + let unguarded = hosts + .client(resolved("r1", None), services(), None) + .unwrap(); + + assert!(!guarded.same_client(&unguarded)); + } +} diff --git a/litellm-rust/crates/redis-connect/src/lib.rs b/litellm-rust/crates/redis-connect/src/lib.rs new file mode 100644 index 00000000000..7e37f721679 --- /dev/null +++ b/litellm-rust/crates/redis-connect/src/lib.rs @@ -0,0 +1,14 @@ +mod connect; +mod error; +mod events; +mod hosts; +mod refresh; +mod sources; + +pub(crate) use connect::breaker; +pub use connect::{HostServices, SharedEnvironment, connect}; +pub use error::{CredentialStage, Error}; +pub use events::TracingEvents; +pub use hosts::RedisHosts; +pub(crate) use refresh::RefreshingCredentials; +pub use refresh::{CredentialSource, Expiring, SourceFuture}; diff --git a/litellm-rust/crates/redis-connect/src/refresh.rs b/litellm-rust/crates/redis-connect/src/refresh.rs new file mode 100644 index 00000000000..1e05d3d6289 --- /dev/null +++ b/litellm-rust/crates/redis-connect/src/refresh.rs @@ -0,0 +1,783 @@ +use std::{ + future::{Future, pending}, + pin::Pin, + sync::{Arc, Mutex}, + time::{Duration, SystemTime}, +}; + +use futures_util::{Stream, stream}; +use litellm_redis_client::{BasicAuth, RedisError, StreamingCredentialsProvider}; +use litellm_secret_value::SecretValue; +use tokio::{sync::watch, time::Instant}; + +use crate::Error; + +const TOKEN_ONLY_USERNAME: &str = "default"; +const UNKNOWN_LIFETIME: Duration = Duration::from_secs(300); +const MIN_REFRESH: Duration = Duration::from_secs(1); +const ACQUISITION_TIMEOUT: Duration = Duration::from_secs(30); +const FIRST_RETRY: Duration = Duration::from_secs(1); +const MAX_RETRY: Duration = Duration::from_secs(60); + +#[derive(Clone, PartialEq, Eq)] +pub struct Expiring { + pub username: Option, + pub password: SecretValue, + pub expires_at: Option, +} + +pub type SourceFuture<'a> = Pin> + Send + 'a>>; + +pub trait CredentialSource: Send + Sync + 'static { + fn fetch(&self) -> SourceFuture<'_>; +} + +impl CredentialSource for Arc { + fn fetch(&self) -> SourceFuture<'_> { + (**self).fetch() + } +} + +#[derive(Clone)] +enum Latest { + Pending, + Ready { + username: String, + password: SecretValue, + expires_at: Option, + }, + Unavailable(Arc), +} + +impl Latest { + fn is_same_token(&self, other: &Self) -> bool { + matches!( + (self, other), + ( + Self::Ready { username, password, .. }, + Self::Ready { username: other_username, password: other_password, .. }, + ) if username == other_username && password == other_password + ) + } + + fn expires_at(&self) -> Option { + match self { + Self::Ready { expires_at, .. } => *expires_at, + Self::Pending | Self::Unavailable(_) => None, + } + } +} + +type Start = Box; + +#[derive(Clone)] +pub struct RefreshingCredentials { + latest: watch::Receiver, + start: Arc>>, +} + +impl RefreshingCredentials { + pub fn new(source: impl CredentialSource) -> Self { + let (sender, latest) = watch::channel(Latest::Pending); + let start: Start = Box::new(move || drop(tokio::spawn(refresh(source, sender)))); + Self { + latest, + start: Arc::new(Mutex::new(Some(start))), + } + } + + fn start_once(&self) { + let start = self + .start + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .take(); + if let Some(start) = start { + start(); + } + } +} + +impl StreamingCredentialsProvider for RefreshingCredentials { + fn subscribe( + &self, + ) -> Pin> + Send + 'static>> { + self.start_once(); + Box::pin(stream::unfold( + (self.latest.clone(), true), + |(mut receiver, first)| async move { + if !first { + receiver.changed().await.ok()?; + } + let latest = receiver + .wait_for(|latest| !matches!(latest, Latest::Pending)) + .await + .ok()? + .clone(); + let item = match latest { + Latest::Ready { + username, password, .. + } => Ok(BasicAuth::new(username, password.expose().to_owned())), + Latest::Unavailable(error) => Err(public_error(&error)), + Latest::Pending => unreachable!("wait_for skips Pending"), + }; + Some((item, (receiver, false))) + }, + )) + } +} + +fn public_error(error: &Error) -> RedisError { + RedisError::from(( + litellm_redis_client::RedisErrorKind::AuthenticationFailed, + "fetching Redis credentials failed", + error.to_string(), + )) +} + +struct Fresh { + latest: Latest, + refresh_in: Duration, +} + +fn fresh(credentials: Expiring, now: SystemTime) -> Result { + let remaining = credentials + .expires_at + .map(|expires_at| expires_at.duration_since(now).unwrap_or_default()); + if remaining == Some(Duration::ZERO) { + return Err(Error::ExpiredCredential); + } + Ok(Fresh { + latest: Latest::Ready { + username: credentials + .username + .unwrap_or_else(|| TOKEN_ONLY_USERNAME.to_owned()), + password: credentials.password, + expires_at: remaining.map(|remaining| Instant::now() + remaining), + }, + refresh_in: refresh_after(credentials.expires_at, now), + }) +} + +async fn until_expiry( + future: impl Future, + sender: &watch::Sender, +) -> Option { + tokio::pin!(future); + loop { + let expires_at = sender.borrow().expires_at(); + let expiration = async { + match expires_at { + Some(deadline) => tokio::time::sleep_until(deadline).await, + None => pending().await, + } + }; + tokio::select! { + biased; + () = sender.closed() => return None, + () = expiration => { + sender.send_replace(Latest::Unavailable(Arc::new(Error::ExpiredCredential))); + } + result = &mut future => return Some(result), + } + } +} + +async fn refresh(source: impl CredentialSource, sender: watch::Sender) { + let mut retry = FIRST_RETRY; + loop { + let acquisition = async { + tokio::time::timeout(ACQUISITION_TIMEOUT, source.fetch()) + .await + .map_err(|cause| Error::CredentialTimeout { + timeout: ACQUISITION_TIMEOUT, + cause, + })? + .and_then(|credentials| fresh(credentials, SystemTime::now())) + }; + let Some(result) = until_expiry(acquisition, &sender).await else { + return; + }; + let wait = match result { + Ok(Fresh { latest, refresh_in }) => { + retry = FIRST_RETRY; + sender.send_if_modified(|current| { + let changed = !current.is_same_token(&latest); + *current = latest; + changed + }); + refresh_in + } + Err(error) => { + tracing::warn!(failure = error.kind(), upstream_status = error.upstream_status(), retry_in = ?retry, "fetching Redis credentials failed"); + if !matches!(*sender.borrow(), Latest::Ready { .. }) { + sender.send_replace(Latest::Unavailable(Arc::new(error))); + } + let wait = retry; + retry = (retry * 2).min(MAX_RETRY); + wait + } + }; + if until_expiry(tokio::time::sleep(wait), &sender) + .await + .is_none() + { + return; + } + } +} + +fn refresh_after(expires_at: Option, now: SystemTime) -> Duration { + let remaining = expires_at + .map(|expires_at| expires_at.duration_since(now).unwrap_or_default()) + .unwrap_or(UNKNOWN_LIFETIME); + (remaining * 3 / 4).max(MIN_REFRESH) +} + +#[cfg(test)] +mod tests { + use litellm_redis_client::commands; + use std::{ + collections::BTreeSet, + sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, + }, + time::{Duration, SystemTime}, + }; + + use futures_util::StreamExt; + use litellm_redis_client::StreamingCredentialsProvider; + use litellm_secret_value::SecretValue; + use rstest::rstest; + + use super::{CredentialSource, Expiring, RefreshingCredentials, SourceFuture, refresh_after}; + use litellm_redis_client::{ + Cluster, Credentials, CredentialsProvider, Namespace, Node, RedisClient, RedisConfig, + RedisUrl, Sentinel, Target, Timeouts, + }; + use litellm_redis_testing::{ + ClientConnection, RedisDeployment, RedisSentinel, SERVICE_NAME, Topology, eventually, + }; + + #[rstest] + #[case::three_quarters_of_the_lifetime( + Some(Duration::from_secs(900)), + Duration::from_secs(675) + )] + #[case::already_expired(Some(Duration::ZERO), Duration::from_secs(1))] + #[case::unknown_expiry(None, Duration::from_secs(225))] + fn refreshes_well_before_expiry( + #[case] lifetime: Option, + #[case] expected: Duration, + ) { + let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000); + + assert_eq!( + refresh_after(lifetime.map(|lifetime| now + lifetime), now), + expected + ); + } + + struct Scripted { + replies: Mutex>>, + calls: Arc, + } + + impl CredentialSource for Scripted { + fn fetch(&self) -> SourceFuture<'_> { + self.calls.fetch_add(1, Ordering::SeqCst); + let reply = self.replies.lock().unwrap().remove(0); + Box::pin(async move { + reply.map_err(|reason| { + crate::Error::credential(crate::error::CredentialStage::Identity)( + std::io::Error::other(reason), + ) + }) + }) + } + } + + fn token(password: &str, lifetime: Duration) -> Result { + Ok(Expiring { + username: None, + password: SecretValue::new(password), + expires_at: Some(SystemTime::now() + lifetime), + }) + } + + #[rstest] + #[tokio::test(start_paused = true)] + async fn pushes_each_refresh_and_keeps_the_last_token_through_failures() { + let calls = Arc::new(AtomicUsize::new(0)); + let provider = RefreshingCredentials::new(Scripted { + replies: Mutex::new(vec![ + token("first", Duration::from_secs(8)), + Err("token endpoint down"), + token("second", Duration::from_secs(3600)), + ]), + calls: calls.clone(), + }); + let mut stream = provider.subscribe(); + + let first = stream.next().await.unwrap().unwrap(); + let second = stream.next().await.unwrap().unwrap(); + let late = provider.subscribe().next().await.unwrap().unwrap(); + + assert_eq!((first.username(), first.password()), ("default", "first")); + assert_eq!(second.password(), "second"); + assert_eq!(late.password(), "second"); + assert_eq!(calls.load(Ordering::SeqCst), 3); + } + + #[rstest] + fn building_outside_a_runtime_fetches_nothing_until_a_connection_subscribes() { + let calls = Arc::new(AtomicUsize::new(0)); + let provider = RefreshingCredentials::new(Scripted { + replies: Mutex::new(vec![token("first", Duration::from_secs(3600))]), + calls: calls.clone(), + }); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let before = calls.load(Ordering::SeqCst); + let first = runtime.block_on(async { provider.subscribe().next().await.unwrap().unwrap() }); + + assert_eq!(before, 0); + assert_eq!(first.password(), "first"); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + #[rstest] + #[tokio::test(start_paused = true)] + async fn a_failure_before_any_token_fails_the_connection_instead_of_hanging() { + let provider = RefreshingCredentials::new(Scripted { + replies: Mutex::new(vec![ + Err("no identity"), + token("late", Duration::from_secs(3600)), + ]), + calls: Arc::new(AtomicUsize::new(0)), + }); + let mut stream = provider.subscribe(); + + assert!(stream.next().await.unwrap().is_err()); + assert_eq!(stream.next().await.unwrap().unwrap().password(), "late"); + } + + struct Blocking { + first: Mutex>, + started: Arc, + cancelled: Arc, + } + + struct Cancellation(Arc); + + impl Drop for Cancellation { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + impl CredentialSource for Blocking { + fn fetch(&self) -> SourceFuture<'_> { + let first = self.first.lock().unwrap().take(); + Box::pin(async move { + if let Some(first) = first { + return Ok(first); + } + let _guard = Cancellation(self.cancelled.clone()); + self.started.notify_one(); + std::future::pending().await + }) + } + } + + #[rstest] + #[tokio::test(start_paused = true)] + async fn a_refresh_that_returns_the_same_token_is_not_pushed_again() { + let calls = Arc::new(AtomicUsize::new(0)); + let provider = RefreshingCredentials::new(Scripted { + replies: Mutex::new(vec![ + token("same", Duration::from_secs(8)), + token("same", Duration::from_secs(3600)), + ]), + calls: calls.clone(), + }); + let mut stream = provider.subscribe(); + assert_eq!(stream.next().await.unwrap().unwrap().password(), "same"); + + let pushed = tokio::time::timeout(Duration::from_secs(60), stream.next()).await; + + assert_eq!(calls.load(Ordering::SeqCst), 2); + assert!(pushed.is_err(), "an unchanged token was pushed again"); + } + + #[rstest] + #[tokio::test(start_paused = true)] + async fn expiration_is_published_while_refresh_is_stalled() { + let started = Arc::new(tokio::sync::Notify::new()); + let provider = RefreshingCredentials::new(Blocking { + first: Mutex::new(Some(token("first", Duration::from_secs(4)).unwrap())), + started: started.clone(), + cancelled: Arc::new(AtomicUsize::new(0)), + }); + let mut stream = provider.subscribe(); + assert_eq!(stream.next().await.unwrap().unwrap().password(), "first"); + let start = tokio::time::Instant::now(); + started.notified().await; + let expired = stream.next().await.unwrap().unwrap_err(); + assert!(start.elapsed() <= Duration::from_secs(5)); + assert!(expired.to_string().contains("expired")); + let fresh = provider.subscribe().next().await.unwrap().unwrap_err(); + assert!(fresh.to_string().contains("expired")); + } + + #[rstest] + #[tokio::test(start_paused = true)] + async fn expiration_is_published_once_and_later_failures_keep_their_cause() { + let replies = std::iter::once(token("first", Duration::from_secs(8))) + .chain(std::iter::repeat_n(Err("token endpoint down"), 16)) + .collect(); + let provider = RefreshingCredentials::new(Scripted { + replies: Mutex::new(replies), + calls: Arc::new(AtomicUsize::new(0)), + }); + let mut stream = provider.subscribe(); + assert_eq!(stream.next().await.unwrap().unwrap().password(), "first"); + + let failures: Vec = stream + .take(5) + .map(|item| item.unwrap_err().to_string()) + .collect() + .await; + + assert!(failures[0].contains("expired"), "{failures:?}"); + assert!( + failures[1..] + .iter() + .all(|failure| failure.contains("caller identity") && !failure.contains("expired")), + "{failures:?}" + ); + } + + #[rstest] + #[tokio::test(start_paused = true)] + async fn initial_acquisition_has_a_bounded_wait() { + let provider = RefreshingCredentials::new(Blocking { + first: Mutex::new(None), + started: Arc::new(tokio::sync::Notify::new()), + cancelled: Arc::new(AtomicUsize::new(0)), + }); + let error = provider.subscribe().next().await.unwrap().unwrap_err(); + assert!(error.to_string().contains("exceeded")); + } + + #[rstest] + #[tokio::test(start_paused = true)] + async fn dropping_all_subscribers_cancels_inflight_acquisition() { + let started = Arc::new(tokio::sync::Notify::new()); + let cancelled = Arc::new(AtomicUsize::new(0)); + let provider = RefreshingCredentials::new(Blocking { + first: Mutex::new(None), + started: started.clone(), + cancelled: cancelled.clone(), + }); + let stream = provider.subscribe(); + started.notified().await; + drop(stream); + drop(provider); + tokio::task::yield_now().await; + assert_eq!(cancelled.load(Ordering::SeqCst), 1); + } + + #[rstest] + #[tokio::test(start_paused = true)] + async fn an_expired_fetch_is_rejected_and_a_later_valid_token_recovers() { + let provider = RefreshingCredentials::new(Scripted { + replies: Mutex::new(vec![ + Ok(Expiring { + username: None, + password: SecretValue::new("expired"), + expires_at: Some(SystemTime::UNIX_EPOCH), + }), + token("recovered", Duration::from_secs(3600)), + ]), + calls: Arc::new(AtomicUsize::new(0)), + }); + let mut stream = provider.subscribe(); + assert!( + stream + .next() + .await + .unwrap() + .unwrap_err() + .to_string() + .contains("expired") + ); + assert_eq!( + stream.next().await.unwrap().unwrap().password(), + "recovered" + ); + } + + #[rstest] + fn credential_failure_keeps_its_typed_cause_but_hides_it_at_the_redis_boundary() { + let error = crate::Error::credential(crate::error::CredentialStage::Identity)( + std::io::Error::new(std::io::ErrorKind::PermissionDenied, "credential-value"), + ); + assert_eq!( + litellm_error::find::(&error) + .unwrap() + .kind(), + std::io::ErrorKind::PermissionDenied + ); + let debug = format!("{error:?}"); + assert!(!debug.contains("credential-value")); + assert!(debug.contains("[REDACTED]")); + let public = super::public_error(&error).to_string(); + assert!(public.contains("caller identity")); + assert!(!public.contains("credential-value")); + } + + #[rstest] + fn credential_http_failures_expose_status_without_exposing_the_response_body() { + let error = crate::Error::credential_response(crate::CredentialStage::Exchange, 503)( + litellm_http::Error::upstream(503, "credential-value".into()), + ); + let public = super::public_error(&error).to_string(); + assert!(public.contains("HTTP 503")); + assert!(!public.contains("credential-value")); + assert_eq!( + litellm_error::find::(&error) + .unwrap() + .upstream_response() + .unwrap() + .0, + 503 + ); + } + + fn endpoint(deployment: &RedisDeployment) -> Target { + match deployment { + RedisDeployment::Standalone(server) => Target::Url(RedisUrl { + url: SecretValue::new(server.url()), + tls: None, + db: None, + username: None, + password: None, + }), + RedisDeployment::Cluster(cluster) => Target::Cluster(Cluster { + nodes: cluster + .nodes() + .map(|(host, port)| Node { + host: host.to_owned(), + port, + }) + .collect(), + tls: None, + }), + } + } + + fn user(username: &str, password: &str, lifetime: Duration) -> Result { + Ok(Expiring { + username: Some(username.to_owned()), + password: SecretValue::new(password), + expires_at: Some(SystemTime::now() + lifetime), + }) + } + + fn connection_users(connections: &[ClientConnection]) -> BTreeSet<&str> { + connections + .iter() + .map(|connection| connection.user.as_str()) + .collect() + } + + fn connection_ids(connections: &[ClientConnection]) -> BTreeSet<(u16, u64)> { + connections + .iter() + .map(|connection| (connection.node, connection.id)) + .collect() + } + + #[rstest] + #[tokio::test] + async fn a_refreshed_token_reauthenticates_live_connections_before_the_old_one_expires( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = RedisDeployment::start(topology).await.unwrap(); + server + .cli_on_every_node(&[ + "ACL", "SETUSER", "alice", "on", ">first", "~*", "&*", "+@all", + ]) + .await + .unwrap(); + server + .cli_on_every_node(&[ + "ACL", "SETUSER", "bob", "on", ">second", "~*", "&*", "+@all", + ]) + .await + .unwrap(); + let provider = RefreshingCredentials::new(Scripted { + replies: Mutex::new(vec![ + user("alice", "first", Duration::from_secs(2)), + user("bob", "second", Duration::from_secs(3600)), + ]), + calls: Arc::new(AtomicUsize::new(0)), + }); + let client = RedisClient::new(RedisConfig { + target: endpoint(&server), + credentials: Credentials::Provider(CredentialsProvider(Arc::new(provider))), + namespace: Namespace::default(), + timeouts: Timeouts::default(), + breaker: None, + events: None, + health_check: None, + keepalive: true, + }) + .unwrap(); + client + .send(commands::set("k", "1", Duration::from_secs(60))) + .await + .unwrap(); + let before = server.clients().await.unwrap(); + + let after = eventually(async || { + let connections = server.clients().await.ok()?; + (!connections.is_empty() && connection_users(&connections) == BTreeSet::from(["bob"])) + .then_some(connections) + }) + .await; + server + .cli_on_every_node(&["ACL", "DELUSER", "alice"]) + .await + .unwrap(); + + assert!(!before.is_empty()); + assert_eq!(connection_users(&before), BTreeSet::from(["alice"])); + assert_eq!(connection_ids(&after), connection_ids(&before)); + assert_eq!( + client + .send(commands::get::("k")) + .await + .unwrap() + .as_deref(), + Some("1") + ); + server + .cli_on_every_node(&["CLIENT", "KILL", "USER", "bob"]) + .await + .unwrap(); + let value = eventually(async || { + client + .send(commands::get::("k")) + .await + .ok() + .flatten() + }) + .await; + let reconnected = server.clients().await.unwrap(); + + assert_eq!(value, "1"); + assert!(!reconnected.is_empty()); + assert_eq!(connection_users(&reconnected), BTreeSet::from(["bob"])); + assert!(connection_ids(&reconnected).is_disjoint(&connection_ids(&after))); + } + + /// Cloud credentials rotate while Sentinel fails over: the refreshed user must be what the + /// client presents to the master Sentinel promotes, not the one it connected with first. + #[rstest] + #[tokio::test] + async fn a_refreshed_user_reaches_the_master_sentinel_promotes() { + let sentinel = RedisSentinel::start().await.unwrap(); + sentinel + .cli_on_data_nodes(&[ + "ACL", "SETUSER", "alice", "on", ">first", "~*", "&*", "+@all", + ]) + .await + .unwrap(); + sentinel + .cli_on_data_nodes(&[ + "ACL", "SETUSER", "bob", "on", ">second", "~*", "&*", "+@all", + ]) + .await + .unwrap(); + let provider = RefreshingCredentials::new(Scripted { + replies: Mutex::new(vec![ + user("alice", "first", Duration::from_secs(2)), + user("bob", "second", Duration::from_secs(3600)), + ]), + calls: Arc::new(AtomicUsize::new(0)), + }); + let (host, port) = sentinel.sentinel(); + let client = RedisClient::new(RedisConfig { + target: Target::Sentinel(Sentinel { + monitors: vec![Node { + host: host.to_owned(), + port, + }], + service_name: SERVICE_NAME.to_owned(), + monitor_auth: None, + tls: None, + db: 0, + }), + credentials: Credentials::Provider(CredentialsProvider(Arc::new(provider))), + namespace: Namespace::default(), + timeouts: Timeouts::new(Duration::from_secs(1), None), + breaker: None, + events: None, + health_check: None, + keepalive: true, + }) + .unwrap(); + client + .send(commands::set("before", "1", Duration::from_secs(60))) + .await + .unwrap(); + let master = sentinel.master_port().await.unwrap(); + let app_users = |connections: &[ClientConnection]| -> BTreeSet { + connections + .iter() + .filter(|connection| connection.user != "default") + .map(|connection| connection.user.clone()) + .collect() + }; + eventually(async || { + let connections = sentinel.clients_on(master).await.ok()?; + (app_users(&connections) == BTreeSet::from(["bob".to_owned()])).then_some(()) + }) + .await; + sentinel + .cli_on_data_nodes(&["ACL", "DELUSER", "alice"]) + .await + .unwrap(); + + let promoted = sentinel.failover().await.unwrap(); + eventually(async || { + client + .send(commands::set("after", "2", Duration::from_secs(60))) + .await + .ok() + }) + .await; + let on_promoted = sentinel.clients_on(promoted).await.unwrap(); + + assert_eq!(promoted, sentinel.replica_port()); + assert_eq!( + app_users(&on_promoted), + BTreeSet::from(["bob".to_owned()]), + "{on_promoted:?}" + ); + assert_eq!( + client + .send(commands::get::("after")) + .await + .unwrap() + .as_deref(), + Some("2") + ); + } +} diff --git a/litellm-rust/crates/redis-connect/src/sources/aws.rs b/litellm-rust/crates/redis-connect/src/sources/aws.rs new file mode 100644 index 00000000000..1e87bdd1b12 --- /dev/null +++ b/litellm-rust/crates/redis-connect/src/sources/aws.rs @@ -0,0 +1,188 @@ +use std::time::{Duration, SystemTime}; + +use litellm_auth_aws::{AwsAuthService, SigningScope, presign_url}; +use litellm_core_utils::settings::Lookup; +use litellm_redis_types::AwsIamAuth; + +use crate::Error; +use crate::error::CredentialStage; +use litellm_secret_value::SecretValue; + +use crate::{CredentialSource, Expiring, SourceFuture}; + +const SERVICE: &str = "elasticache"; +const TOKEN_LIFETIME: Duration = Duration::from_secs(900); + +const SDK_IDENTITY_ENVIRONMENT: &[&str] = &[ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_PROFILE", + "AWS_DEFAULT_PROFILE", + "AWS_SHARED_CREDENTIALS_FILE", + "AWS_CONFIG_FILE", + "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_ROLE_ARN", + "AWS_ROLE_SESSION_NAME", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", + "AWS_CONTAINER_AUTHORIZATION_TOKEN", + "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", + "AWS_EC2_METADATA_DISABLED", + "AWS_EC2_METADATA_SERVICE_ENDPOINT", + "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE", + "AWS_ENDPOINT_URL", + "AWS_ENDPOINT_URL_STS", +]; + +pub(crate) struct ElastiCacheIam { + service: AwsAuthService, + auth: AwsIamAuth, +} + +impl ElastiCacheIam { + pub(crate) fn new(auth: AwsIamAuth, environment: &dyn Lookup) -> Result { + validate_environment(environment, &|name: &str| std::env::var(name).ok())?; + Ok(Self { + service: AwsAuthService::default(), + auth, + }) + } +} + +fn validate_environment(environment: &dyn Lookup, process: &dyn Lookup) -> Result<(), Error> { + match SDK_IDENTITY_ENVIRONMENT.iter().find(|name| { + environment.get(name).is_some_and(|value| { + Some(SecretValue::new(value)) != process.get(name).map(SecretValue::new) + }) + }) { + Some(setting) => Err(Error::ProcessEnvironmentRequired { setting }), + None => Ok(()), + } +} + +pub(crate) fn connect_url(auth: &AwsIamAuth) -> String { + let query = [ + Some(("Action", "connect")), + Some(("User", auth.user_name.as_str())), + auth.serverless + .then_some(("ResourceType", "ServerlessCache")), + ]; + let mut url = String::from("https://"); + url.push_str(&auth.cache_name.to_lowercase()); + url.push_str("/?"); + url.push_str( + &url::form_urlencoded::Serializer::new(String::new()) + .extend_pairs(query.into_iter().flatten()) + .finish(), + ); + url +} + +impl CredentialSource for ElastiCacheIam { + fn fetch(&self) -> SourceFuture<'_> { + Box::pin(async move { + let credentials = self + .service + .default_chain_credentials(&self.auth.region) + .await + .map_err(Error::credential(CredentialStage::Identity))?; + let now = SystemTime::now(); + let signed = presign_url( + &connect_url(&self.auth), + TOKEN_LIFETIME, + SigningScope { + region: &self.auth.region, + service: SERVICE, + credentials: &credentials, + time: now, + }, + ) + .map_err(Error::credential(CredentialStage::Signing))?; + let token_expiry = now + TOKEN_LIFETIME; + Ok(Expiring { + username: Some(self.auth.user_name.clone()), + password: SecretValue::new(signed.trim_start_matches("https://")), + expires_at: Some( + credentials + .expiry() + .map_or(token_expiry, |expiry| expiry.min(token_expiry)), + ), + }) + }) + } +} + +#[cfg(test)] +mod tests { + use litellm_redis_types::AwsIamAuth; + use rstest::rstest; + + use super::{connect_url, validate_environment}; + + #[rstest] + #[case::profile("AWS_PROFILE")] + #[case::role("AWS_ROLE_ARN")] + #[case::web_identity("AWS_WEB_IDENTITY_TOKEN_FILE")] + #[case::source_key("AWS_ACCESS_KEY_ID")] + #[case::source_secret("AWS_SECRET_ACCESS_KEY")] + #[case::session("AWS_SESSION_TOKEN")] + #[case::profile_path("AWS_SHARED_CREDENTIALS_FILE")] + #[case::container("AWS_CONTAINER_CREDENTIALS_FULL_URI")] + fn configured_identity_overrides_cannot_silently_use_an_ambient_identity( + #[case] setting: &'static str, + ) { + let configured = |name: &str| (name == setting).then(|| "credential-value".to_owned()); + let process = |name: &str| (name == setting).then(|| "ambient".to_owned()); + let error = validate_environment(&configured, &process).unwrap_err(); + + assert!(matches!( + error, + crate::Error::ProcessEnvironmentRequired { setting: rejected } if rejected == setting + )); + assert!(!error.to_string().contains("credential-value")); + assert!(!format!("{error:?}").contains("credential-value")); + } + + #[rstest] + #[case::process_environment("AWS_PROFILE", Some("profile"), Some("profile"))] + #[case::ambient_only("AWS_PROFILE", None, Some("profile"))] + #[case::unconfigured("AWS_PROFILE", None, None)] + #[case::unrelated_bedrock_setting("AWS_ROLE_NAME", Some("bedrock"), None)] + fn unchanged_or_unrelated_settings_preserve_the_sdk_chain( + #[case] setting: &str, + #[case] configured: Option<&str>, + #[case] ambient: Option<&str>, + ) { + let configured = |name: &str| { + (name == setting) + .then_some(configured) + .flatten() + .map(str::to_owned) + }; + let ambient = |name: &str| { + (name == setting) + .then_some(ambient) + .flatten() + .map(str::to_owned) + }; + assert!(validate_environment(&configured, &ambient).is_ok()); + } + + #[rstest] + #[case::provisioned(false, "https://cache-1/?Action=connect&User=iam+user%2F1")] + #[case::serverless( + true, + "https://cache-1/?Action=connect&User=iam+user%2F1&ResourceType=ServerlessCache" + )] + fn the_connect_url_matches_python_urlencode(#[case] serverless: bool, #[case] expected: &str) { + let auth = AwsIamAuth { + user_name: "iam user/1".to_owned(), + cache_name: "Cache-1".to_owned(), + region: "us-east-1".to_owned(), + serverless, + }; + + assert_eq!(connect_url(&auth), expected); + } +} diff --git a/litellm-rust/crates/redis-connect/src/sources/azure.rs b/litellm-rust/crates/redis-connect/src/sources/azure.rs new file mode 100644 index 00000000000..6dc72e6ce72 --- /dev/null +++ b/litellm-rust/crates/redis-connect/src/sources/azure.rs @@ -0,0 +1,64 @@ +use litellm_auth_azure::{AzureAuthService, AzureIdentity}; +use litellm_auth_types::ResolvedCredential; +use litellm_redis_types::AzureAdAuth; + +use crate::Error; +use crate::error::CredentialStage; + +use crate::{CredentialSource, Expiring, SharedEnvironment, SourceFuture}; + +const AZURE_REDIS_SCOPE: &str = "https://redis.azure.com/.default"; + +pub(crate) struct AzureAd { + service: AzureAuthService, + identity: AzureIdentity, + username: Option, + lookup: SharedEnvironment, +} + +impl AzureAd { + pub(crate) fn new(auth: AzureAdAuth, lookup: SharedEnvironment) -> Self { + // _build_azure_credential in litellm/_redis.py picks the credential class the same way + let identity = match (auth.client_id, auth.tenant_id, auth.client_secret) { + (Some(client_id), Some(tenant_id), Some(client_secret)) => { + AzureIdentity::ClientSecret { + tenant_id, + client_id, + client_secret, + } + } + (Some(client_id), _, _) => AzureIdentity::ManagedIdentity { client_id }, + (None, _, _) => AzureIdentity::Default, + }; + Self { + service: AzureAuthService::default(), + identity, + username: auth.username, + lookup, + } + } +} + +impl CredentialSource for AzureAd { + fn fetch(&self) -> SourceFuture<'_> { + Box::pin(async move { + let credential = self + .service + .identity_token(&self.identity, AZURE_REDIS_SCOPE, &|name| { + self.lookup.get(name) + }) + .await + .map_err(Error::credential(CredentialStage::Identity))? + .ok_or(Error::MissingCredential { provider: "Azure" })?; + let (token, expires_at) = match credential { + ResolvedCredential::AccessToken { token, expires_on } => (token, expires_on), + ResolvedCredential::Static(token) => (token, None), + }; + Ok(Expiring { + username: self.username.clone(), + password: token, + expires_at, + }) + }) + } +} diff --git a/litellm-rust/crates/redis-connect/src/sources/gcp.rs b/litellm-rust/crates/redis-connect/src/sources/gcp.rs new file mode 100644 index 00000000000..9fb238709a1 --- /dev/null +++ b/litellm-rust/crates/redis-connect/src/sources/gcp.rs @@ -0,0 +1,152 @@ +use std::time::SystemTime; + +use litellm_auth_gcp::{CredentialSource as GcpCredentialSource, GcpAuth}; +use serde::Deserialize; + +use crate::Error; +use crate::error::CredentialStage; +use litellm_secret_value::SecretValue; + +use crate::{CredentialSource, Expiring, SharedEnvironment, SourceFuture}; + +const IAM_CREDENTIALS: &str = "https://iamcredentials.googleapis.com/v1"; +const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform"; + +pub(crate) struct GcpIam { + http: litellm_http::Client, + identity: GcpAuth, + service_account: String, + lookup: SharedEnvironment, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct AccessToken { + access_token: String, + expire_time: String, +} + +impl GcpIam { + pub(crate) fn new( + http: litellm_http::Client, + service_account: String, + lookup: SharedEnvironment, + ) -> Self { + Self { + http, + identity: GcpAuth::default(), + service_account, + lookup, + } + } +} + +fn token_url(service_account: &str) -> String { + match service_account.starts_with("projects/") { + true => format!("{IAM_CREDENTIALS}/{service_account}:generateAccessToken"), + false => format!( + "{IAM_CREDENTIALS}/projects/-/serviceAccounts/{service_account}:generateAccessToken" + ), + } +} + +impl CredentialSource for GcpIam { + fn fetch(&self) -> SourceFuture<'_> { + Box::pin(async move { + let lookup = |name: &str| self.lookup.get(name); + let caller = self + .identity + .access_token(GcpCredentialSource::application_default(&lookup)) + .await + .map_err(Error::credential(CredentialStage::Identity))?; + let response = self + .http + .post(token_url(&self.service_account)) + .bearer_auth(caller.expose()) + .json(&serde_json::json!({ "scope": [CLOUD_PLATFORM_SCOPE] })) + .send() + .await + .map_err(Error::credential(CredentialStage::Exchange))?; + let status = response.status().as_u16(); + let reply: AccessToken = response + .error_for_status() + .map_err(litellm_http::Error::from) + .map_err(Error::credential_response( + CredentialStage::Exchange, + status, + ))? + .json() + .await + .map_err(litellm_http::Error::from) + .map_err(Error::credential(CredentialStage::Response))?; + let expires_at = time::OffsetDateTime::parse( + &reply.expire_time, + &time::format_description::well_known::Rfc3339, + ) + .map(SystemTime::from) + .map_err(Error::credential(CredentialStage::Response))?; + Ok(Expiring { + username: None, + password: SecretValue::new(reply.access_token), + expires_at: Some(expires_at), + }) + }) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::{GcpIam, token_url}; + use crate::CredentialSource; + + #[rstest] + #[case::bare_email( + "redis@project.iam.gserviceaccount.com", + "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/redis@project.iam.gserviceaccount.com:generateAccessToken" + )] + #[case::resource_name( + "projects/-/serviceAccounts/redis@project.iam.gserviceaccount.com", + "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/redis@project.iam.gserviceaccount.com:generateAccessToken" + )] + #[case::project_scoped_resource_name( + "projects/project/serviceAccounts/redis@project.iam.gserviceaccount.com", + "https://iamcredentials.googleapis.com/v1/projects/project/serviceAccounts/redis@project.iam.gserviceaccount.com:generateAccessToken" + )] + fn a_bare_service_account_email_is_addressed_as_a_resource_name( + #[case] service_account: &str, + #[case] expected: &str, + ) { + assert_eq!(token_url(service_account), expected); + } + + #[rstest] + #[tokio::test] + async fn the_caller_identity_comes_from_application_default_credentials_not_vertex_settings() { + let environment = |name: &str| { + match name { + "VERTEXAI_CREDENTIALS" => { + Some(r#"{"type":"service_account","project_id":"vertex"}"#) + } + "GOOGLE_APPLICATION_CREDENTIALS" => Some("/nonexistent/redis-adc.json"), + _ => None, + } + .map(str::to_owned) + }; + let source = GcpIam::new( + litellm_http::Client::for_test(), + "projects/-/serviceAccounts/redis@project.iam.gserviceaccount.com".to_owned(), + std::sync::Arc::new(environment), + ); + + let Err(error) = source.fetch().await else { + panic!("a missing credentials file must fail"); + }; + + assert_eq!( + litellm_error::find::(&error).map(std::io::Error::kind), + Some(std::io::ErrorKind::NotFound) + ); + } +} diff --git a/litellm-rust/crates/redis-connect/src/sources/mod.rs b/litellm-rust/crates/redis-connect/src/sources/mod.rs new file mode 100644 index 00000000000..988ad3e1d23 --- /dev/null +++ b/litellm-rust/crates/redis-connect/src/sources/mod.rs @@ -0,0 +1,7 @@ +mod aws; +mod azure; +mod gcp; + +pub(crate) use aws::ElastiCacheIam; +pub(crate) use azure::AzureAd; +pub(crate) use gcp::GcpIam; diff --git a/litellm-rust/crates/redis-connect/tests/connect.rs b/litellm-rust/crates/redis-connect/tests/connect.rs new file mode 100644 index 00000000000..b1d541f4a5e --- /dev/null +++ b/litellm-rust/crates/redis-connect/tests/connect.rs @@ -0,0 +1,230 @@ +use litellm_redis_client::commands; +use std::{collections::BTreeMap, sync::Arc, time::Duration}; + +use litellm_redis_connect::{ + CredentialSource, Error, Expiring, HostServices, SourceFuture, connect, +}; +use litellm_redis_testing::{ + RedisDeployment, RedisSentinel, RedisServer, RedisTlsServer, SERVICE_NAME, Topology, eventually, +}; +use litellm_redis_types::{ProviderHandle, Resolution}; +use litellm_secret_value::SecretValue; +use rstest::rstest; + +// a host hands its credential_provider object over as a handle, so these tests do the same +fn resolved(kwargs: serde_json::Value) -> Resolution { + let provider = kwargs + .get("credential_provider") + .map(|_| ProviderHandle::new(1)); + let kwargs = serde_json::from_value::>(kwargs).unwrap(); + litellm_redis_resolve::interpret(&litellm_redis_resolve::kwargs(kwargs), provider, None) + .unwrap() +} + +fn services() -> HostServices { + HostServices { + events: None, + http: litellm_http::Client::for_test(), + environment: Arc::new(|_: &str| None), + credential_provider: None, + } +} + +struct HostPassword(&'static str); + +impl CredentialSource for HostPassword { + fn fetch(&self) -> SourceFuture<'_> { + Box::pin(async move { + Ok(Expiring { + username: None, + password: SecretValue::new(self.0), + expires_at: None, + }) + }) + } +} + +fn port(url: &str) -> u16 { + url.rsplit(':').next().unwrap().parse().unwrap() +} + +#[rstest] +#[case::trusted_ca(true, true)] +#[case::system_roots_only(false, false)] +#[tokio::test] +async fn python_tls_kwargs_reach_the_server(#[case] with_ca: bool, #[case] connects: bool) { + if rustls::crypto::CryptoProvider::get_default().is_none() { + let _ = rustls::crypto::ring::default_provider().install_default(); + } + let server = RedisTlsServer::start(false).await.unwrap(); + let kwargs = serde_json::json!({ + "host": server.host(), + "port": server.port(), + "ssl": true, + "ssl_ca_certs": with_ca.then(|| server.files().ca.display().to_string()), + "socket_timeout": 2, + }); + + let pinged = connect(resolved(kwargs), services(), None) + .unwrap() + .ping() + .await; + + assert_eq!(pinged.is_ok(), connects, "{pinged:?}"); +} + +#[rstest] +#[case::right_password("right", true)] +#[case::wrong_password("wrong", false)] +#[tokio::test] +async fn static_credentials_authenticate( + #[case] password: &str, + #[case] connects: bool, + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, +) { + let server = RedisDeployment::start(topology).await.unwrap(); + server + .cli_on_every_node(&["ACL", "SETUSER", "default", "on", ">right"]) + .await + .unwrap(); + let kwargs = match &server { + RedisDeployment::Standalone(server) => serde_json::json!({ + "host": "127.0.0.1", + "port": port(server.url()), + "password": password, + "socket_timeout": 2, + }), + RedisDeployment::Cluster(cluster) => serde_json::json!({ + "startup_nodes": cluster.nodes().map(|(host, port)| serde_json::json!({ + "host": host, + "port": port, + })).collect::>(), + "password": password, + "socket_timeout": 2, + }), + }; + + let pinged = connect(resolved(kwargs), services(), None) + .unwrap() + .ping() + .await; + + assert_eq!(pinged.is_ok(), connects, "{pinged:?}"); +} + +#[rstest] +#[tokio::test] +async fn python_sentinel_kwargs_follow_the_master_through_a_failover() { + let sentinel = RedisSentinel::start().await.unwrap(); + let (host, sentinel_port) = sentinel.sentinel(); + let kwargs = serde_json::json!({ + "sentinel_nodes": [[host, sentinel_port]], + "service_name": SERVICE_NAME, + "socket_timeout": 1, + }); + let client = connect(resolved(kwargs), services(), None).unwrap(); + client + .send(commands::set("before", "1", Duration::from_secs(600))) + .await + .unwrap(); + + let promoted = sentinel.failover().await.unwrap(); + eventually(async || { + client + .send(commands::set("after", "2", Duration::from_secs(600))) + .await + .ok() + }) + .await; + + assert_eq!(promoted, sentinel.replica_port()); + assert_eq!(sentinel.master_port().await.unwrap(), promoted); + assert_eq!( + client + .send(commands::get::("after")) + .await + .unwrap() + .as_deref(), + Some("2") + ); +} + +#[rstest] +fn a_python_credential_provider_without_a_host_source_fails_at_boot() { + let error = connect( + resolved(serde_json::json!({"host": "redis", "credential_provider": true})), + services(), + None, + ) + .err() + .unwrap(); + + assert!( + matches!(error, Error::NoHostCredentialProvider), + "{error:?}" + ); +} + +#[rstest] +#[case::profile("AWS_PROFILE")] +#[case::role("AWS_ROLE_ARN")] +#[case::web_identity("AWS_WEB_IDENTITY_TOKEN_FILE")] +fn config_only_aws_identities_fail_before_connecting(#[case] setting: &'static str) { + let configured = format!("{}-override", std::env::var(setting).unwrap_or_default()); + let host = HostServices { + environment: Arc::new(move |name: &str| (name == setting).then(|| configured.clone())), + ..services() + }; + let error = connect( + resolved(serde_json::json!({ + "host": "redis", "ssl": true, "aws_iam_auth": true, + "aws_iam_user_name": "user", "aws_iam_cache_name": "cache", + "aws_iam_region": "us-east-1", + })), + host, + None, + ) + .err() + .expect("must not select ambient credentials"); + + assert!(matches!( + error, + Error::ProcessEnvironmentRequired { setting: rejected } if rejected == setting + )); +} + +#[rstest] +#[case::right_password("right", true)] +#[case::wrong_password("wrong", false)] +#[tokio::test] +async fn a_python_credential_provider_authenticates_through_the_host_source( + #[case] password: &'static str, + #[case] connects: bool, +) { + let server = RedisServer::start().await.unwrap(); + server + .cli(&["ACL", "SETUSER", "default", "on", ">right"]) + .await + .unwrap(); + let kwargs = serde_json::json!({ + "host": "127.0.0.1", + "port": port(server.url()), + "password": "ignored", + "credential_provider": true, + "socket_timeout": 2, + }); + + let pinged = connect( + resolved(kwargs), + HostServices { + credential_provider: Some(Arc::new(HostPassword(password))), + ..services() + }, + None, + ) + .unwrap() + .ping() + .await; + + assert_eq!(pinged.is_ok(), connects, "{pinged:?}"); +} diff --git a/litellm-rust/crates/redis-connect/tests/live/aws.rs b/litellm-rust/crates/redis-connect/tests/live/aws.rs new file mode 100644 index 00000000000..da5ec323392 --- /dev/null +++ b/litellm-rust/crates/redis-connect/tests/live/aws.rs @@ -0,0 +1,17 @@ +use rstest::{fixture, rstest}; + +use super::{LiveConfig, config, roundtrip}; + +#[fixture] +fn live_config() -> Option { + config("LITELLM_REDIS_AWS_LIVE_CONFIG") +} + +#[rstest] +#[tokio::test] +async fn credentials_authenticate_existing_and_fresh_connections(live_config: Option) { + let Some(config) = live_config else { + return; + }; + roundtrip(config).await; +} diff --git a/litellm-rust/crates/redis-connect/tests/live/azure.rs b/litellm-rust/crates/redis-connect/tests/live/azure.rs new file mode 100644 index 00000000000..a7f678edef4 --- /dev/null +++ b/litellm-rust/crates/redis-connect/tests/live/azure.rs @@ -0,0 +1,17 @@ +use rstest::{fixture, rstest}; + +use super::{LiveConfig, config, roundtrip}; + +#[fixture] +fn live_config() -> Option { + config("LITELLM_REDIS_AZURE_LIVE_CONFIG") +} + +#[rstest] +#[tokio::test] +async fn credentials_authenticate_existing_and_fresh_connections(live_config: Option) { + let Some(config) = live_config else { + return; + }; + roundtrip(config).await; +} diff --git a/litellm-rust/crates/redis-connect/tests/live/gcp.rs b/litellm-rust/crates/redis-connect/tests/live/gcp.rs new file mode 100644 index 00000000000..0965a7a9e9a --- /dev/null +++ b/litellm-rust/crates/redis-connect/tests/live/gcp.rs @@ -0,0 +1,17 @@ +use rstest::{fixture, rstest}; + +use super::{LiveConfig, config, roundtrip}; + +#[fixture] +fn live_config() -> Option { + config("LITELLM_REDIS_GCP_LIVE_CONFIG") +} + +#[rstest] +#[tokio::test] +async fn credentials_authenticate_existing_and_fresh_connections(live_config: Option) { + let Some(config) = live_config else { + return; + }; + roundtrip(config).await; +} diff --git a/litellm-rust/crates/redis-connect/tests/live/main.rs b/litellm-rust/crates/redis-connect/tests/live/main.rs new file mode 100644 index 00000000000..90e7637530e --- /dev/null +++ b/litellm-rust/crates/redis-connect/tests/live/main.rs @@ -0,0 +1,111 @@ +mod aws; +mod azure; +mod gcp; + +use std::{collections::BTreeMap, path::PathBuf, sync::Arc, time::Duration}; + +use litellm_http::{ + ClientVariant, HttpClientPool, HttpSettings, HttpSettingsLayer, Resolution, + media::PublicDnsResolver, +}; +use litellm_redis_client::commands; +use litellm_redis_connect::{HostServices, connect}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct LiveConfig { + redis: BTreeMap, + hold_seconds: u64, +} + +fn config(variable: &str) -> Option { + let path = PathBuf::from(std::env::var_os(variable)?); + Some( + serde_json::from_slice(&std::fs::read(path).expect("read live Redis config")) + .expect("parse live Redis config"), + ) +} + +async fn roundtrip(config: LiveConfig) { + if rustls::crypto::CryptoProvider::get_default().is_none() { + let _ = rustls::crypto::ring::default_provider().install_default(); + } + let lookup = |name: &str| std::env::var(name).ok(); + let resolved = litellm_redis_resolve::interpret( + &litellm_redis_resolve::kwargs(config.redis), + None, + lookup("REDIS_USERNAME").as_deref(), + ) + .expect("resolve live Redis config"); + let http_settings = HttpSettings::from_layers([HttpSettingsLayer::from_environment(&lookup)]); + let pool = HttpClientPool::new(Arc::new(PublicDnsResolver)); + let http = pool + .client( + &Resolution::from(&http_settings).config, + ClientVariant::Provider, + ) + .expect("build production HTTP client"); + let services = || HostServices { + events: None, + http: http.clone(), + environment: Arc::new(|name: &str| std::env::var(name).ok()), + credential_provider: None, + }; + let client = connect(resolved.clone(), services(), None).expect("build live Redis client"); + let marker = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let key = format!("litellm:redis-connect:live:{}:{marker}", std::process::id()); + let ttl = Duration::from_secs(config.hold_seconds + 120); + + client.ping().await.expect("authenticate and ping"); + client + .send(commands::set(&key, "before-refresh", ttl)) + .await + .expect("set before refresh"); + assert_eq!( + client + .send(commands::get::(&key)) + .await + .unwrap() + .as_deref(), + Some("before-refresh") + ); + + tokio::time::sleep(Duration::from_secs(config.hold_seconds)).await; + + assert_eq!( + client + .send(commands::get::(&key)) + .await + .unwrap() + .as_deref(), + Some("before-refresh") + ); + let fresh = connect(resolved, services(), None).expect("build a fresh live Redis client"); + assert_eq!( + fresh + .send(commands::get::(&key)) + .await + .unwrap() + .as_deref(), + Some("before-refresh") + ); + fresh + .send(commands::set(&key, "after-refresh", ttl)) + .await + .expect("set after refresh"); + assert_eq!( + client + .send(commands::get::(&key)) + .await + .unwrap() + .as_deref(), + Some("after-refresh") + ); + client + .send(commands::del(&[&key])) + .await + .expect("delete live test key"); +} diff --git a/litellm-rust/crates/redis-resolve/AGENTS.md b/litellm-rust/crates/redis-resolve/AGENTS.md new file mode 100644 index 00000000000..b7ce8278975 --- /dev/null +++ b/litellm-rust/crates/redis-resolve/AGENTS.md @@ -0,0 +1,31 @@ +## Interpreting merged kwargs + +- The boundary is merged redis-py kwargs in, `litellm_redis_types::Resolution` out. `interpret` takes the dict a host already merged the way `_merge_redis_kwargs` in `litellm/_redis.py` does, with its `REDIS_*` values and `get_secret` fallbacks filled, and stands in for redis-py's constructors and `from_url`. It never reads the environment or a secret, so both hosts share it. Every `Entry` keeps the source a warning or error names, which `kwargs` sets to the kwarg name and the gateway merge sets to `REDIS_` or the fallback name +- The `Resolution` holds the target, the auth and the timeouts Rust models, and every setting the Rust client ignores as an `Ignored` with its source and consequence, which the host logs. A host passes its `credential_provider` object as a `ProviderHandle`, and a `credential_provider` key without one is `Error::ProviderWithoutHandle`. Python's Azure AD auth reads `REDIS_USERNAME` from `os.environ` instead of the merged kwargs, so it is a required argument of `interpret` that each host reads from its own process environment, the gateway through its `Lookup` and python-bridge through Python's `os.environ`. An empty one is unset, like `or None` +- `options.rs` reads the kwargs into the typed `Options` and collects the ignored ones. `endpoint.rs`, `auth.rs` and `tls.rs` pick the target, the credentials and TLS from `Options` alone +- Rust never introspects redis-py. Every Redis kwarg redis-py or `litellm/_redis.py` accepts is one row in `SETTINGS` in `src/keys.rs`, and that row is the whole answer for the kwarg: its `Handling` (`Read`, `Warn` with the consequence the warning names, or `Reject`), whether a url query carries it (`FromUrl`), whether `_get_redis_env_kwarg_mapping` reads it as `REDIS_`, and the names `_merge_redis_kwargs` falls back to. The last two columns are only read by the gateway merge, and they stay on the row so there is one table. A `Read` row is a field of `Options`, which an inline test pins. A kwarg with no row is not a Redis setting, so it is dropped without a warning +- `redis-py-kwargs.json` holds the Python facts each row answers: which redis-py versions name the kwarg in which signature, and whether Python reads its `REDIS_*` name under the locked redis-py. Never edit it by hand. `redis_py_kwargs.py` next to it regenerates it, and its docstring says how to run it. A kwarg litellm reads that no signature or `REDIS_*` name covers goes in its `LITELLM_ONLY`, and a new redis-py version in its `VERSIONS` and the `redis-compat` CI matrix together. `test_redis_py_kwargs_catalog_matches_the_installed_redis_py` in `tests/unit/test_redis.py` checks it against each version the `redis-compat` CI matrix installs, and `tests/catalog.rs` fails until `SETTINGS` has exactly one row per kwarg in it with the same `environment`. A redis-py bump that adds a kwarg therefore cannot land without a row deciding how Rust handles it +- The contract is `resolves_like_python_redis` in `tests/interpret.rs` and its mirror `test_redis_settings_resolve_like_the_rust_resolver_expects` in `tests/unit/test_redis.py`. Each case holds the merged dict, the sources the merge gave it, the `REDIS_USERNAME` the host passes, and the expected outcome written next to it. `Resolves` is the view both sides build. `Refuses` is a config Rust refuses and Python connects with, a disagreement on purpose. `Fails` is a config redis-py 5.3.1 already raises on, building the client or its first connection, so Rust failing changes no deployment that works today. Its Rust side names the class `Error::raised_building_the_python_client` answers, `None` when Python only raises once a command connects, and its Python mirror `_Fails` asserts the class and message redis-py raises. The Python side asserts that `_merge_redis_kwargs` returns that merged dict and reads the effective redis-py 5.3.1 connection objects without connecting, and `merges_like_python_redis` in `gateway/src/redis/surface_tests.rs` pins the same merged dicts from the Rust merge. A new rule is a new case in every file, never a generated fixture. Python and Rust pods share one Redis during the switch, so a change here is a change to a wire contract +- A url carries its own TLS. redis-py's url allowlist is the plain `Connection` signature, so every `ssl_*` setting outside the url is dropped, and a credential provider strips the url's userinfo. A `rediss://` url's `ssl_cert_reqs`, `ssl_ca_certs`, `ssl_ca_data`, `ssl_ca_path`, `ssl_certfile`, `ssl_keyfile`, `ssl_password` and `ssl_check_hostname` query parameters are read like `parse_url` reads them +- TLS options and `db` are only read for the endpoint that uses them, so a bad `ssl_cert_reqs` never fails a url target. A cluster with a non-zero `db`, and Sentinel with a `host` or `port`, are errors because redis-py raises building those clients +- An empty value (`""`, `[]`, `{}`) is the same as an unset one, like Python's `x or ...`. A null is unset too, except `ssl_cert_reqs`, where redis-py reads `None` as `CERT_NONE`, so a null skips certificate verification +- `ssl_cert_reqs` takes `none`, `optional` and `required` case-sensitively like redis-py's lookup, so `None` (what `REDIS_SSL_CERT_REQS=None` gives) is `Error::Invalid`, and an `ssl.VerifyMode` int, where 0 skips verification and 1 or 2 verifies. `optional` verifies, because rustls has no optional mode +- Precedence matches Python. The target is cluster, then URL, then Sentinel (`sentinel_nodes` plus `service_name`), then host. Auth is a `credential_provider` handle, then GCP, then Azure, then AWS, and the first one wins with a warning +- Numbers are read with `litellm_python_compat::number`, so whitespace and digit underscores parse like Python's `int()` and `float()`. A value they cannot parse (`port: abc`, `socket_timeout: slow`) is dropped with a warning and the default is used, like `_coerce_redis_kwargs_types`. A number that parses but is out of range (negative seconds, a port above 65535) fails with `Error::Invalid` +- Flags (`ssl`, `ssl_check_hostname`, `aws_iam_auth`, `aws_iam_serverless`) are read one way on every surface. `true`, `1` and `yes` are true and `false`, `0`, `no`, `off`, `f` and `n` are false, the text redis-py's `_str_to_bool` and the proxy's pydantic models agree on. Any other text is `Error::Invalid`, so a TLS setting is never guessed +- The resolver fails on a setting it cannot honor only when that setting is a security check. A `Warn` row is dropped and returned as an `Ignored` naming its source (`ssl_ciphers` or `REDIS_SSL_CIPHERS`) and the consequence on its row, because a Python deployment that worked must keep working when its stores move to Rust. Only a value that is malformed (`Error::Invalid` or `Error::UrlQuery`), a target that is missing or a `Reject` row that asks for its check fails. Promoting a `Warn` row to `Read` is then a pure improvement with no behavior cliff, while moving one to `Reject` can fail a config that connects today + - A `Warn` row with a `met` check is dropped silently when its value asks for nothing more than Rust does, so `ssl_min_version` of TLS 1.2 or lower never warns + - A `Reject` row is a security check rustls cannot perform that Python's locked async client already refuses with a `TypeError`, so failing closed breaks no proxy that runs today. `tests/catalog.rs` fails if the locked redis-py's async, cluster or connection signature accepts a `Reject` row, so a redis-py bump forces the decision again. A row's `Asked` says which values ask for the check, from the kwargs, `REDIS_*` or a url query, and those are `Error::Unenforceable` + - `ssl_validate_ocsp` and `ssl_validate_ocsp_stapled` ask when true, because rustls has no client-side OCSP. False asks for nothing and is dropped without a warning + - `ssl_include_verify_flags` and `ssl_exclude_verify_flags` ask when set, because rustls takes no X.509 verify flags such as CRL checking. An empty list asks for nothing + - `ssl_ca_data` and `ssl_ca_path` are read into `TlsOptions` and trusted next to `ssl_ca_certs`. `ssl_password` is read into `TlsOptions::key_password`, which `redis-client` uses to decrypt a PKCS#8 key like the sync client's `load_cert_chain(password=...)`. redis-py 5.3.1's async client refuses it, so this is a superset of the path the proxy uses, like `ssl_ca_path` + - `unix_socket_path` resolves to `Target::Unix`, which like redis-py's `UnixDomainSocketConnection` replaces host, port and TLS. Python still requires `host` next to it, so Rust does too + - A url's `?db=`, `?username=`, `?password=`, `?socket_timeout=`, `?socket_connect_timeout=` and `?health_check_interval=` are read like `parse_url` reads them, so `?db=` wins over the path, the userinfo wins over `?username=` and `?password=`, and the timeouts and health check win over the kwargs like `from_url`. A repeated parameter takes its first value like `parse_qs`. A number `int()` or `float()` cannot read is `Error::UrlQuery`, as `parse_url` raises `ValueError`. Any other query parameter, including one with no row, is ignored naming `url ?`, because redis-py turns each one into a client kwarg. Which ones a url carries is the `FromUrl` column + - `socket_keepalive` is a flag that defaults to true on every target, like `_get_redis_cluster_kwargs` defaults it for a cluster. Python leaves it off for a standalone or Sentinel client, so an unset value there is a difference on purpose, because without keepalive a load balancer drops an idle connection silently. False turns it off. `socket_keepalive_options` is warned, and `redis-client` probes after its own idle time instead + - `health_check_interval` is read as an `int` of seconds like `_coerce_redis_kwargs_types` into `HealthCheck`. Zero is `Disabled` and stays off on every target, a negative interval checks before every command like redis-py, and only `Unset` takes a default. `redis-connect` gives an unset cluster Python's 25s default and `redis-client` pings every node before the next command once the client sat idle that long, outside the breaker and the call events +- `Error::raised_building_the_python_client` is how python-bridge treats a config Rust cannot use. Where Python raises building its client, the bridge raises the same class from `NativeRedisClient.connect`. Where Python builds the client and only fails once a command connects, which `RedisCache` catches and logs on every call, the stores stay on memory and the bridge logs an error +- Cases where Rust and Python disagree on purpose are cases in both contract tests with a different expected outcome on each side + - `ssl_ca_path`: redis-py 5.3.1's async `Redis` has no such parameter and raises `TypeError` building the client, while Rust trusts the directory like `load_verify_locations(capath=...)` + - `on`, `t` and `y` for a flag are `Error::Invalid` on every surface. pydantic reads them as true for `coordination_redis` and `_str_to_bool` reads them as false everywhere else, so Rust refuses to pick one. Other unrecognized text is false in Python and an error here + - `ssl_cert_reqs` as an `ssl.VerifyMode` int: redis-py 5.3.1's sync client takes it, while its async `RedisSSLContext` leaves `cert_reqs` unset and raises `AttributeError` once a command connects. Rust reads it like the sync client + - Empty `sentinel_nodes` with no `REDIS_SENTINEL_NODES` is no target at all, so it fails as "either host or url must be specified". redis-py keeps the empty list and fails building the Sentinel client diff --git a/litellm-rust/crates/redis-resolve/Cargo.toml b/litellm-rust/crates/redis-resolve/Cargo.toml new file mode 100644 index 00000000000..b3d3fc01e44 --- /dev/null +++ b/litellm-rust/crates/redis-resolve/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "litellm-redis-resolve" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-python-compat.workspace = true +litellm-redis-types.workspace = true +litellm-secret-value.workspace = true +serde_json.workspace = true +strum.workspace = true +thiserror.workspace = true +tracing.workspace = true +url.workspace = true + +[dev-dependencies] +rstest.workspace = true +serde.workspace = true diff --git a/litellm-rust/crates/redis-resolve/redis-py-kwargs.json b/litellm-rust/crates/redis-resolve/redis-py-kwargs.json new file mode 100644 index 00000000000..22c4495818b --- /dev/null +++ b/litellm-rust/crates/redis-resolve/redis-py-kwargs.json @@ -0,0 +1,118 @@ +{ + "meta": { + "description": "What redis-py and litellm/_redis.py accept for every Redis kwarg the Rust resolver can be handed. SETTINGS in src/keys.rs has exactly one row per arg here", + "redis_py_versions": [ + "5.3.1", + "6.4.0", + "7.4.1", + "8.0.1" + ], + "python": "3.12", + "method": "python redis_py_kwargs.py next to this file, see its docstring. It reads inspect.getfullargspec(inspect.unwrap(cls.__init__)) args and kwonlyargs, minus self", + "signatures": { + "sync": "redis.Redis.__init__. _get_redis_kwargs uses it as the config allowlist and the REDIS_ environment mapping", + "async": "redis.asyncio.Redis.__init__, which the proxy constructs. A kwarg in sync but not async raises TypeError when the client is built", + "connection": "redis.Connection or redis.asyncio.Connection MRO, the url allowlist from _get_redis_url_kwargs", + "cluster": "redis.asyncio.RedisCluster.__init__, the allowlist from _get_redis_cluster_kwargs" + }, + "environment": "true when _get_redis_env_kwarg_mapping reads the arg as REDIS_ under the locked redis-py", + "redis_py": "each redis-py version whose signatures name the arg, with which ones. An empty object means only litellm reads it", + "checked_by": "tests/unit/test_redis.py checks the installed redis-py's column and the REDIS_* names against this file, and tests/catalog.rs checks SETTINGS against it", + "redis_py_locked": "5.3.1" + }, + "args": { + "address_remap": {"environment": false, "redis_py": {"5.3.1": ["cluster"], "6.4.0": ["cluster"], "7.4.1": ["cluster"], "8.0.1": ["cluster"]}}, + "auto_close_connection_pool": {"environment": false, "redis_py": {"5.3.1": ["async"], "6.4.0": ["async"], "7.4.1": ["async"], "8.0.1": ["async"]}}, + "aws_iam_auth": {"environment": true, "redis_py": {}}, + "aws_iam_cache_name": {"environment": true, "redis_py": {}}, + "aws_iam_region": {"environment": true, "redis_py": {}}, + "aws_iam_serverless": {"environment": true, "redis_py": {}}, + "aws_iam_user_name": {"environment": true, "redis_py": {}}, + "azure_client_id": {"environment": true, "redis_py": {}}, + "azure_client_secret": {"environment": true, "redis_py": {}}, + "azure_redis_ad_token": {"environment": true, "redis_py": {}}, + "azure_tenant_id": {"environment": true, "redis_py": {}}, + "cache": {"environment": true, "redis_py": {"5.3.1": ["sync"], "6.4.0": ["sync"], "7.4.1": ["sync"], "8.0.1": ["sync"]}}, + "cache_config": {"environment": true, "redis_py": {"5.3.1": ["sync"], "6.4.0": ["sync"], "7.4.1": ["sync"], "8.0.1": ["sync"]}}, + "charset": {"environment": true, "redis_py": {"5.3.1": ["sync"]}}, + "client_name": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "cluster_error_retry_attempts": {"environment": false, "redis_py": {"5.3.1": ["cluster"], "6.4.0": ["cluster"], "7.4.1": ["cluster"], "8.0.1": ["cluster"]}}, + "command_packer": {"environment": false, "redis_py": {"5.3.1": ["connection"], "6.4.0": ["connection"], "7.4.1": ["connection"], "8.0.1": ["connection"]}}, + "connection_class": {"environment": false, "redis_py": {}}, + "connection_error_retry_attempts": {"environment": false, "redis_py": {"5.3.1": ["cluster"]}}, + "connection_pool": {"environment": false, "redis_py": {"5.3.1": ["sync", "async"], "6.4.0": ["sync", "async"], "7.4.1": ["sync", "async"], "8.0.1": ["sync", "async"]}}, + "credential_provider": {"environment": false, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "db": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "decode_responses": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "driver_info": {"environment": false, "redis_py": {"7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "dynamic_startup_nodes": {"environment": false, "redis_py": {"6.4.0": ["cluster"], "7.4.1": ["cluster"], "8.0.1": ["cluster"]}}, + "encoder_class": {"environment": false, "redis_py": {"5.3.1": ["connection"], "6.4.0": ["connection"], "7.4.1": ["connection"], "8.0.1": ["connection"]}}, + "encoding": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "encoding_errors": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "errors": {"environment": true, "redis_py": {"5.3.1": ["sync"]}}, + "event_dispatcher": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "gcp_service_account": {"environment": true, "redis_py": {}}, + "gcp_ssl_ca_certs": {"environment": true, "redis_py": {}}, + "health_check_interval": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "host": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "legacy_responses": {"environment": false, "redis_py": {"8.0.1": ["sync", "async", "connection", "cluster"]}}, + "lib_name": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "lib_version": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "load_balancing_strategy": {"environment": false, "redis_py": {"5.3.1": ["cluster"], "6.4.0": ["cluster"], "7.4.1": ["cluster"], "8.0.1": ["cluster"]}}, + "maint_notifications_config": {"environment": false, "redis_py": {"7.4.1": ["sync", "connection"], "8.0.1": ["sync", "connection"]}}, + "maint_notifications_pool_handler": {"environment": false, "redis_py": {"7.4.1": ["connection"], "8.0.1": ["connection"]}}, + "maintenance_notification_hash": {"environment": false, "redis_py": {"7.4.1": ["connection"], "8.0.1": ["connection"]}}, + "maintenance_state": {"environment": false, "redis_py": {"7.4.1": ["connection"], "8.0.1": ["connection"]}}, + "max_connections": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "cluster"], "6.4.0": ["sync", "async", "cluster"], "7.4.1": ["sync", "async", "cluster"], "8.0.1": ["sync", "async", "cluster"]}}, + "namespace": {"environment": false, "redis_py": {}}, + "orig_host_address": {"environment": false, "redis_py": {"7.4.1": ["connection"], "8.0.1": ["connection"]}}, + "orig_socket_connect_timeout": {"environment": false, "redis_py": {"7.4.1": ["connection"], "8.0.1": ["connection"]}}, + "orig_socket_timeout": {"environment": false, "redis_py": {"7.4.1": ["connection"], "8.0.1": ["connection"]}}, + "oss_cluster_maint_notifications_handler": {"environment": false, "redis_py": {"7.4.1": ["sync", "connection"], "8.0.1": ["sync", "connection"]}}, + "parser": {"environment": false, "redis_py": {"7.4.1": ["connection"], "8.0.1": ["connection"]}}, + "parser_class": {"environment": false, "redis_py": {"5.3.1": ["connection"], "6.4.0": ["connection"], "7.4.1": ["connection"], "8.0.1": ["connection"]}}, + "password": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "path": {"environment": false, "redis_py": {"5.3.1": ["cluster"], "6.4.0": ["cluster"], "7.4.1": ["cluster"], "8.0.1": ["cluster"]}}, + "policy_resolver": {"environment": false, "redis_py": {"7.4.1": ["cluster"], "8.0.1": ["cluster"]}}, + "port": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "protocol": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "read_from_replicas": {"environment": false, "redis_py": {"5.3.1": ["cluster"], "6.4.0": ["cluster"], "7.4.1": ["cluster"], "8.0.1": ["cluster"]}}, + "redis_connect_func": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection"], "6.4.0": ["sync", "async", "connection"], "7.4.1": ["sync", "async", "connection"], "8.0.1": ["sync", "async", "connection"]}}, + "reinitialize_steps": {"environment": false, "redis_py": {"5.3.1": ["cluster"], "6.4.0": ["cluster"], "7.4.1": ["cluster"], "8.0.1": ["cluster"]}}, + "require_full_coverage": {"environment": false, "redis_py": {"5.3.1": ["cluster"], "6.4.0": ["cluster"], "7.4.1": ["cluster"], "8.0.1": ["cluster"]}}, + "retry": {"environment": false, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "retry_on_error": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "retry_on_timeout": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection"], "6.4.0": ["sync", "async", "connection"], "7.4.1": ["sync", "async", "connection"], "8.0.1": ["sync", "async", "connection"]}}, + "sentinel_nodes": {"environment": false, "redis_py": {}}, + "sentinel_password": {"environment": false, "redis_py": {}}, + "service_name": {"environment": false, "redis_py": {}}, + "single_connection_client": {"environment": true, "redis_py": {"5.3.1": ["sync", "async"], "6.4.0": ["sync", "async"], "7.4.1": ["sync", "async"], "8.0.1": ["sync", "async"]}}, + "socket_connect_timeout": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "socket_keepalive": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "socket_keepalive_options": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "socket_read_size": {"environment": false, "redis_py": {"5.3.1": ["connection"], "6.4.0": ["connection"], "7.4.1": ["connection"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "socket_timeout": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}}, + "socket_type": {"environment": false, "redis_py": {"5.3.1": ["connection"], "6.4.0": ["connection"], "7.4.1": ["connection"], "8.0.1": ["connection"]}}, + "ssl": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "cluster"], "6.4.0": ["sync", "async", "cluster"], "7.4.1": ["sync", "async", "cluster"], "8.0.1": ["sync", "async", "cluster"]}}, + "ssl_ca_certs": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "cluster"], "6.4.0": ["sync", "async", "cluster"], "7.4.1": ["sync", "async", "cluster"], "8.0.1": ["sync", "async", "cluster"]}}, + "ssl_ca_data": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "cluster"], "6.4.0": ["sync", "async", "cluster"], "7.4.1": ["sync", "async", "cluster"], "8.0.1": ["sync", "async", "cluster"]}}, + "ssl_ca_path": {"environment": true, "redis_py": {"5.3.1": ["sync"], "6.4.0": ["sync"], "7.4.1": ["sync", "async"], "8.0.1": ["sync", "async"]}}, + "ssl_cert_reqs": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "cluster"], "6.4.0": ["sync", "async", "cluster"], "7.4.1": ["sync", "async", "cluster"], "8.0.1": ["sync", "async", "cluster"]}}, + "ssl_certfile": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "cluster"], "6.4.0": ["sync", "async", "cluster"], "7.4.1": ["sync", "async", "cluster"], "8.0.1": ["sync", "async", "cluster"]}}, + "ssl_check_hostname": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "cluster"], "6.4.0": ["sync", "async", "cluster"], "7.4.1": ["sync", "async", "cluster"], "8.0.1": ["sync", "async", "cluster"]}}, + "ssl_ciphers": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "cluster"], "6.4.0": ["sync", "async", "cluster"], "7.4.1": ["sync", "async", "cluster"], "8.0.1": ["sync", "async", "cluster"]}}, + "ssl_exclude_verify_flags": {"environment": false, "redis_py": {"7.4.1": ["sync", "async", "cluster"], "8.0.1": ["sync", "async", "cluster"]}}, + "ssl_include_verify_flags": {"environment": false, "redis_py": {"7.4.1": ["sync", "async", "cluster"], "8.0.1": ["sync", "async", "cluster"]}}, + "ssl_keyfile": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "cluster"], "6.4.0": ["sync", "async", "cluster"], "7.4.1": ["sync", "async", "cluster"], "8.0.1": ["sync", "async", "cluster"]}}, + "ssl_min_version": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "cluster"], "6.4.0": ["sync", "async", "cluster"], "7.4.1": ["sync", "async", "cluster"], "8.0.1": ["sync", "async", "cluster"]}}, + "ssl_ocsp_context": {"environment": true, "redis_py": {"5.3.1": ["sync"], "6.4.0": ["sync"], "7.4.1": ["sync"], "8.0.1": ["sync"]}}, + "ssl_ocsp_expected_cert": {"environment": true, "redis_py": {"5.3.1": ["sync"], "6.4.0": ["sync"], "7.4.1": ["sync"], "8.0.1": ["sync"]}}, + "ssl_password": {"environment": true, "redis_py": {"5.3.1": ["sync"], "6.4.0": ["sync"], "7.4.1": ["sync", "async"], "8.0.1": ["sync", "async"]}}, + "ssl_validate_ocsp": {"environment": true, "redis_py": {"5.3.1": ["sync"], "6.4.0": ["sync"], "7.4.1": ["sync"], "8.0.1": ["sync"]}}, + "ssl_validate_ocsp_stapled": {"environment": true, "redis_py": {"5.3.1": ["sync"], "6.4.0": ["sync"], "7.4.1": ["sync"], "8.0.1": ["sync"]}}, + "startup_nodes": {"environment": false, "redis_py": {"5.3.1": ["cluster"], "6.4.0": ["cluster"], "7.4.1": ["cluster"], "8.0.1": ["cluster"]}}, + "unix_socket_path": {"environment": true, "redis_py": {"5.3.1": ["sync", "async"], "6.4.0": ["sync", "async"], "7.4.1": ["sync", "async"], "8.0.1": ["sync", "async"]}}, + "url": {"environment": true, "redis_py": {}}, + "username": {"environment": true, "redis_py": {"5.3.1": ["sync", "async", "connection", "cluster"], "6.4.0": ["sync", "async", "connection", "cluster"], "7.4.1": ["sync", "async", "connection", "cluster"], "8.0.1": ["sync", "async", "connection", "cluster"]}} + } +} diff --git a/litellm-rust/crates/redis-resolve/redis_py_kwargs.py b/litellm-rust/crates/redis-resolve/redis_py_kwargs.py new file mode 100644 index 00000000000..9459cfd3761 --- /dev/null +++ b/litellm-rust/crates/redis-resolve/redis_py_kwargs.py @@ -0,0 +1,110 @@ +"""Regenerates redis-py-kwargs.json, the Python facts SETTINGS in src/keys.rs answers. + +Run it from the repository root in the project environment, which has the locked redis-py and litellm: + + uv run python litellm-rust/crates/redis-resolve/redis_py_kwargs.py + +It reads every other redis-py version's signatures in a throwaway environment through +`uv run --no-project --with redis==`, so nothing else is installed. Then update SETTINGS until +tests/catalog.rs passes. +""" + +import inspect +import json +import subprocess +import sys +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +VERSIONS: Final = ("5.3.1", "6.4.0", "7.4.1", "8.0.1") +PYTHON: Final = "3.12" +CATALOG: Final = Path(__file__).with_name("redis-py-kwargs.json") +# Keys litellm/_redis.py or its callers read that neither a redis-py signature nor the REDIS_* mapping names +LITELLM_ONLY: Final = frozenset( + {"connection_class", "namespace", "sentinel_nodes", "sentinel_password", "service_name"} +) + +META: Final = { + "description": "What redis-py and litellm/_redis.py accept for every Redis kwarg the Rust resolver can be handed. SETTINGS in src/keys.rs has exactly one row per arg here", + "redis_py_versions": list(VERSIONS), + "python": PYTHON, + "method": "python redis_py_kwargs.py next to this file, see its docstring. It reads inspect.getfullargspec(inspect.unwrap(cls.__init__)) args and kwonlyargs, minus self", + "signatures": { + "sync": "redis.Redis.__init__. _get_redis_kwargs uses it as the config allowlist and the REDIS_ environment mapping", + "async": "redis.asyncio.Redis.__init__, which the proxy constructs. A kwarg in sync but not async raises TypeError when the client is built", + "connection": "redis.Connection or redis.asyncio.Connection MRO, the url allowlist from _get_redis_url_kwargs", + "cluster": "redis.asyncio.RedisCluster.__init__, the allowlist from _get_redis_cluster_kwargs", + }, + "environment": "true when _get_redis_env_kwarg_mapping reads the arg as REDIS_ under the locked redis-py", + "redis_py": "each redis-py version whose signatures name the arg, with which ones. An empty object means only litellm reads it", + "checked_by": "tests/unit/test_redis.py checks the installed redis-py's column and the REDIS_* names against this file, and tests/catalog.rs checks SETTINGS against it", +} + + +def _named_args(*classes: type) -> frozenset[str]: + return frozenset( + name + for cls in classes + for spec in (inspect.getfullargspec(inspect.unwrap(cls.__init__)),) + for name in spec.args + spec.kwonlyargs + if name != "self" + ) + + +def arg_signatures() -> dict[str, list[str]]: + """Each kwarg the installed redis-py names, with the signatures that name it.""" + import redis + import redis.asyncio as async_redis + + connection_classes: Final = (*redis.Connection.__mro__, *async_redis.Connection.__mro__) + signatures: Final = { + "sync": _named_args(redis.Redis), + "async": _named_args(async_redis.Redis), + "connection": _named_args(*(cls for cls in connection_classes if cls is not object)), + "cluster": _named_args(async_redis.RedisCluster), + } + names: Final = sorted(frozenset().union(*signatures.values())) + return {name: [kind for kind, named in signatures.items() if name in named] for name in names} + + +def _arg_signatures_of(version: str) -> Mapping[str, list[str]]: + command: Final = ("uv", "run", "--no-project", "--python", PYTHON, "--with", f"redis=={version}") + script: Final = ("python", "-I", __file__, "--arg-signatures") + return json.loads(subprocess.run((*command, *script), check=True, capture_output=True, text=True).stdout) + + +def catalog() -> dict[str, object]: + import redis + + from litellm._redis import _get_redis_env_kwarg_mapping + + environment: Final = frozenset(_get_redis_env_kwarg_mapping().values()) + by_version: Final = {version: _arg_signatures_of(version) for version in VERSIONS} + names: Final = sorted(LITELLM_ONLY | environment | frozenset().union(*by_version.values())) + return { + "meta": {**META, "redis_py_locked": redis.__version__}, + "args": { + name: { + "environment": name in environment, + "redis_py": {version: args[name] for version, args in by_version.items() if name in args}, + } + for name in names + }, + } + + +# one line per arg, so a version bump reads as a diff of the args it touched +def render(catalog: Mapping[str, object]) -> str: + meta: Final = json.dumps(catalog["meta"], indent=2).replace("\n", "\n ") + args: Final = catalog["args"] + assert isinstance(args, dict) + rows: Final = ",\n".join(f" {json.dumps(name)}: {json.dumps(arg)}" for name, arg in args.items()) + return f'{{\n "meta": {meta},\n "args": {{\n{rows}\n }}\n}}\n' + + +if __name__ == "__main__": + if sys.argv[1:] == ["--arg-signatures"]: + sys.stdout.write(json.dumps(arg_signatures())) + else: + CATALOG.write_text(render(catalog())) diff --git a/litellm-rust/crates/redis-resolve/src/auth.rs b/litellm-rust/crates/redis-resolve/src/auth.rs new file mode 100644 index 00000000000..620f1a2e11b --- /dev/null +++ b/litellm-rust/crates/redis-resolve/src/auth.rs @@ -0,0 +1,95 @@ +use litellm_redis_types::{Auth, AwsIamAuth, AzureAdAuth, ProviderHandle, StaticAuth}; + +use crate::{ + Error, + options::{self, AwsOptions, Options, flag, secret, text}, +}; + +pub(super) fn redis_auth( + options: &Options, + provider: Option, + redis_username: Option<&str>, +) -> Result { + match (provider, &options.credential_provider) { + (Some(handle), _) => return Ok(Auth::Provider(handle)), + (None, Some(_)) => return Err(Error::ProviderWithoutHandle), + (None, None) => {} + } + let gcp = text(options.gcp.service_account.as_ref()); + let azure = text(options.azure.ad_token.as_ref()) + .is_some_and(|enabled| enabled.eq_ignore_ascii_case("true")); + let aws = flag(options.aws.auth.as_ref())?; + match (gcp, azure, aws) { + (Some(service_account), azure, aws) => { + if azure { + tracing::warn!( + "Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. Using GCP IAM. Remove one to avoid misconfiguration." + ); + } + if aws { + tracing::warn!( + "Both GCP IAM (gcp_service_account) and AWS ElastiCache IAM (aws_iam_auth) are configured for Redis. Using GCP IAM. Remove one to avoid misconfiguration." + ); + } + Ok(Auth::GcpIam { service_account }) + } + (None, true, aws) => { + if aws { + tracing::warn!( + "Both Azure AD (azure_redis_ad_token) and AWS ElastiCache IAM (aws_iam_auth) are configured for Redis. Using Azure AD. Remove one to avoid misconfiguration." + ); + } + Ok(Auth::AzureAd(AzureAdAuth { + client_id: text(options.azure.client_id.as_ref()), + tenant_id: text(options.azure.tenant_id.as_ref()), + client_secret: secret(options.azure.client_secret.as_ref()), + username: redis_username + .filter(|username| !username.is_empty()) + .map(str::to_owned), + })) + } + (None, false, true) => aws_iam(&options.aws, uses_tls(options)?).map(Auth::AwsIam), + (None, false, false) => Ok(Auth::Static(StaticAuth { + username: secret(options.username.as_ref()), + password: secret(options.password.as_ref()), + })), + } +} + +// _uses_tls: a url's scheme decides unless cluster nodes are set, otherwise the ssl kwarg does +fn uses_tls(options: &Options) -> Result { + match (&options.startup_nodes, text(options.url.as_ref())) { + (None, Some(url)) => Ok(url + .split_once("://") + .is_some_and(|(scheme, _)| scheme.eq_ignore_ascii_case("rediss"))), + _ => flag(options.ssl.as_ref()), + } +} + +fn aws_iam(aws: &AwsOptions, uses_tls: bool) -> Result { + if !uses_tls { + return Err(Error::AwsIamWithoutTls); + } + match ( + text(aws.user_name.as_ref()), + text(aws.cache_name.as_ref()), + text(aws.region.as_ref()), + ) { + (Some(user_name), Some(cache_name), Some(region)) => Ok(AwsIamAuth { + user_name, + cache_name, + region, + serverless: options::flag(aws.serverless.as_ref())?, + }), + (user_name, cache_name, region) => Err(Error::AwsIamIncomplete { + missing: [ + ("aws_iam_user_name", user_name.is_none()), + ("aws_iam_cache_name", cache_name.is_none()), + ("aws_iam_region", region.is_none()), + ] + .into_iter() + .filter_map(|(name, missing)| missing.then_some(name)) + .collect(), + }), + } +} diff --git a/litellm-rust/crates/redis-resolve/src/endpoint.rs b/litellm-rust/crates/redis-resolve/src/endpoint.rs new file mode 100644 index 00000000000..e3722b1b7cc --- /dev/null +++ b/litellm-rust/crates/redis-resolve/src/endpoint.rs @@ -0,0 +1,180 @@ +use litellm_redis_types::{ + Auth, Cluster, Node, RedisUrl, Sentinel, Standalone, StaticAuth, Target, TlsOptions, UnixSocket, +}; +use litellm_secret_value::SecretValue; + +use serde_json::Value; + +use crate::{ + Entry, Error, Expected, + options::{Options, as_text, flag, integer, invalid, secret, text}, + tls::{self, QuerySettings}, +}; + +/// The target, first match wins like `get_redis_client`: cluster, url, Sentinel, then host. A +/// url also returns the settings its query carries. +pub(super) fn target(options: &Options, auth: &Auth) -> Result<(Target, QuerySettings), Error> { + if let Some(nodes) = cluster_nodes(options.startup_nodes.as_ref())? { + reject_cluster_db(options.db.as_ref())?; + let cluster = Cluster { + nodes, + tls: tls_for(options, auth)?, + }; + return Ok((Target::Cluster(cluster), QuerySettings::default())); + } + if let Some(url) = text(options.url.as_ref()) { + let query = tls::url_query(&url)?; + let url = RedisUrl { + url: SecretValue::new(match auth { + Auth::Static(_) => url, + _ => without_userinfo(&url), + }), + tls: query.tls, + db: query.db, + username: query.username, + password: query.password, + }; + return Ok((Target::Url(url), query.settings)); + } + if let Some(sentinels) = options.sentinel.nodes.as_ref() { + let sentinel = sentinel(options, auth, sentinels)?; + return Ok((Target::Sentinel(sentinel), QuerySettings::default())); + } + let host = text(options.host.as_ref()).ok_or(Error::NoTarget)?; + let target = match text(options.unix_socket_path.as_ref()) { + Some(path) => Target::Unix(UnixSocket { + path: path.into(), + db: db(options)?, + }), + None => Target::Standalone(Standalone { + host, + port: integer(options.port.as_ref())?.unwrap_or(6379), + tls: tls_for(options, auth)?, + db: db(options)?, + }), + }; + Ok((target, QuerySettings::default())) +} + +fn tls_for(options: &Options, auth: &Auth) -> Result, Error> { + if !flag(options.ssl.as_ref())? { + return Ok(None); + } + let gcp_ca_certs = match auth { + Auth::GcpIam { .. } => options.gcp.ssl_ca_certs.as_ref(), + _ => None, + }; + tls::tls_options(&options.tls, gcp_ca_certs).map(Some) +} + +fn db(options: &Options) -> Result { + integer(options.db.as_ref()).map(|db| db.unwrap_or(0)) +} + +fn sentinel(options: &Options, auth: &Auth, sentinels: &Entry) -> Result { + let monitors = sentinel_nodes(&sentinels.value) + .ok_or_else(|| invalid(&sentinels.source, Expected::SentinelNodes))?; + let service_name = + text(options.sentinel.service_name.as_ref()).ok_or(Error::NoSentinelService)?; + // redis.asyncio.Sentinel passes the connection kwargs to each monitor's Redis(host=..., port=...), so a + // host or port setting raises TypeError when litellm builds the client + if let Some(entry) = options.host.as_ref().or(options.port.as_ref()) { + return Err(Error::SentinelWithHost { + setting: entry.source.clone(), + }); + } + Ok(Sentinel { + monitors, + service_name, + monitor_auth: monitor_auth(options, auth), + tls: tls_for(options, auth)?, + db: db(options)?, + }) +} + +// _sentinel_auth_kwargs gives the monitors the data node's connection kwargs with sentinel_password as the +// password, so a static username reaches them too +fn monitor_auth(options: &Options, auth: &Auth) -> Option { + let password = secret(options.sentinel.password.as_ref())?; + let username = match auth { + Auth::Static(StaticAuth { username, .. }) => username.clone(), + _ => None, + }; + Some(StaticAuth { + username, + password: Some(password), + }) +} + +fn cluster_nodes(entry: Option<&Entry>) -> Result>, Error> { + let Some(entry) = entry else { + return Ok(None); + }; + let nodes = nodes(&entry.value, |node| { + node_address( + node.get("host").and_then(as_text), + node.get("port").and_then(as_text), + ) + }) + .ok_or_else(|| invalid(&entry.source, Expected::StartupNodes))?; + Ok((!nodes.is_empty()).then_some(nodes)) +} + +fn sentinel_nodes(value: &Value) -> Option> { + let nodes = nodes(value, |pair| match pair { + Value::Array(pair) if pair.len() == 2 => node_address(as_text(&pair[0]), as_text(&pair[1])), + _ => None, + })?; + (!nodes.is_empty()).then_some(nodes) +} + +fn nodes(value: &Value, parse: impl Fn(&Value) -> Option) -> Option> { + match value { + Value::String(json) => serde_json::from_str::>(json) + .ok()? + .iter() + .map(parse) + .collect(), + Value::Array(nodes) => nodes.iter().map(parse).collect(), + _ => None, + } +} + +fn node_address(host: Option, port: Option) -> Option { + Some(Node { + host: host?, + port: port?.parse().ok()?, + }) +} + +// redis-py's RedisCluster raises unless db is 0 or None +fn reject_cluster_db(db: Option<&Entry>) -> Result<(), Error> { + let Some(entry) = db else { + return Ok(()); + }; + let is_zero = match &entry.value { + Value::String(_) => matches!(integer::(Some(entry)), Ok(None | Some(0))), + Value::Bool(flag) => !flag, + Value::Number(number) => number.as_f64() == Some(0.0), + _ => false, + }; + match is_zero { + true => Ok(()), + false => Err(Error::DbInCluster { + setting: entry.source.clone(), + }), + } +} + +// _get_redis_url_kwargs keeps only plain Connection kwargs, so redis-py drops every ssl_* setting for a url +// and a credential provider replaces the url's userinfo, which _url_without_userinfo strips +fn without_userinfo(url: &str) -> String { + let Some((scheme, rest)) = url.split_once("://") else { + return url.to_owned(); + }; + let (authority, tail) = rest.split_at(rest.find(['/', '?', '#']).unwrap_or(rest.len())); + let host = authority + .rsplit_once('@') + .map_or(authority, |(_, host)| host); + format!("{scheme}://{host}{tail}") +} diff --git a/litellm-rust/crates/redis-resolve/src/error.rs b/litellm-rust/crates/redis-resolve/src/error.rs new file mode 100644 index 00000000000..b13b65edfb5 --- /dev/null +++ b/litellm-rust/crates/redis-resolve/src/error.rs @@ -0,0 +1,80 @@ +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("invalid Redis setting {setting}: expected {expected}")] + Invalid { setting: String, expected: Expected }, + #[error("invalid Redis setting {setting}: redis-py's parse_url cannot read it as a number")] + UrlQuery { setting: String }, + #[error("either host or url must be specified for redis")] + NoTarget, + #[error("service_name is required with sentinel_nodes")] + NoSentinelService, + #[error("{setting} cannot be combined with sentinel_nodes, which redis-py rejects at startup")] + SentinelWithHost { setting: String }, + #[error("{setting} must be 0 in cluster mode")] + DbInCluster { setting: String }, + #[error("credential_provider is a host object, and the host passed no handle for it")] + ProviderWithoutHandle, + #[error( + "{setting} asks for {check}, which rustls cannot perform, so Redis is not connected without them" + )] + Unenforceable { + setting: String, + check: &'static str, + }, + #[error("AWS ElastiCache IAM Redis authentication requires TLS")] + AwsIamWithoutTls, + #[error("AWS ElastiCache IAM Redis authentication requires: {}", missing.join(", "))] + AwsIamIncomplete { missing: Vec<&'static str> }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)] +pub enum Expected { + #[strum(serialize = "an integer")] + Integer, + #[strum(serialize = "a boolean")] + Boolean, + #[strum(serialize = "a non-negative number of seconds")] + Seconds, + #[strum(serialize = "none, optional, required, null or an ssl.VerifyMode value")] + CertRequirement, + #[strum(serialize = "a list of objects with host and port")] + StartupNodes, + #[strum(serialize = "a non-empty list of [host, port] pairs")] + SentinelNodes, +} + +/// The exception class redis-py 5.3.1 raises building its client. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PythonException { + ValueError, + TypeError, + RedisClusterException, +} + +impl Error { + /// What `_get_redis_client_logic` and redis-py's async client constructors raise on the same kwargs, or `None` + /// when Python builds the client and fails only once a command opens a connection, or never fails. + pub fn raised_building_the_python_client(&self) -> Option { + match self { + Self::Invalid { expected, .. } => match expected { + Expected::StartupNodes | Expected::SentinelNodes => { + Some(PythonException::ValueError) + } + Expected::Integer + | Expected::Boolean + | Expected::Seconds + | Expected::CertRequirement => None, + }, + Self::UrlQuery { .. } + | Self::NoTarget + | Self::NoSentinelService + | Self::AwsIamWithoutTls + | Self::AwsIamIncomplete { .. } => Some(PythonException::ValueError), + Self::SentinelWithHost { .. } | Self::Unenforceable { .. } => { + Some(PythonException::TypeError) + } + Self::DbInCluster { .. } => Some(PythonException::RedisClusterException), + Self::ProviderWithoutHandle => None, + } + } +} diff --git a/litellm-rust/crates/redis-resolve/src/keys.rs b/litellm-rust/crates/redis-resolve/src/keys.rs new file mode 100644 index 00000000000..c4897701d6f --- /dev/null +++ b/litellm-rust/crates/redis-resolve/src/keys.rs @@ -0,0 +1,304 @@ +use litellm_redis_types::Ignored; +use serde_json::Value; + +/// How the Rust client honors a redis-py kwarg. +#[derive(Clone, Copy, Debug)] +pub enum Handling { + /// Modeled by the resolver into a field of `Options`. + Read, + /// Dropped and returned as an `Ignored` naming `consequence`, unless `met` says the value asks for nothing the + /// Rust client does not already do. + Warn { + consequence: &'static str, + met: Option bool>, + }, + /// A security check the Rust client cannot perform and Python's async client already refuses, so asking for it + /// fails instead of connecting without it. + Reject { check: &'static str, asked: Asked }, +} + +/// Which values of a `Reject` row ask for its check. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Asked { + /// A flag read as true. False asks for nothing. + WhenTrue, + /// Any present value, like a list of flags. An empty one asks for nothing. + WhenSet, +} + +/// Whether redis-py's `parse_url` reads the setting from a url's query and the Rust resolver does too. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FromUrl { + No, + Any, + Rediss, +} + +#[derive(Clone, Copy, Debug)] +pub struct Setting { + pub kwarg: &'static str, + pub handling: Handling, + pub from_url: FromUrl, + /// `_get_redis_env_kwarg_mapping` reads it as `REDIS_`. + pub environment: bool, + /// The names `_merge_redis_kwargs` reads with `get_secret` when the merged kwargs lack the setting. + pub fallback: &'static [&'static str], +} + +impl Setting { + const fn new(kwarg: &'static str, handling: Handling) -> Self { + Self { + kwarg, + handling, + from_url: FromUrl::No, + environment: false, + fallback: &[], + } + } + + const fn env(self) -> Self { + Self { + environment: true, + ..self + } + } + + const fn in_url(self, from_url: FromUrl) -> Self { + Self { from_url, ..self } + } + + const fn or_secrets(self, fallback: &'static [&'static str]) -> Self { + Self { fallback, ..self } + } + + pub fn environment_name(&self) -> Option { + self.environment + .then(|| format!("REDIS_{}", self.kwarg.to_ascii_uppercase())) + } + + pub fn is_read(&self) -> bool { + matches!(self.handling, Handling::Read) + } + + /// The warning a set value earns, or `None` when the setting is read, refused or already met. + pub(crate) fn warning(&self, value: &Value) -> Option<&'static str> { + match self.handling { + Handling::Warn { consequence, met } if !met.is_some_and(|met| met(value)) => { + Some(consequence) + } + _ => None, + } + } +} + +const fn read(kwarg: &'static str) -> Setting { + Setting::new(kwarg, Handling::Read) +} + +const fn warn(kwarg: &'static str, consequence: &'static str) -> Setting { + Setting::new( + kwarg, + Handling::Warn { + consequence, + met: None, + }, + ) +} + +const fn warn_unless( + kwarg: &'static str, + consequence: &'static str, + met: fn(&Value) -> bool, +) -> Setting { + Setting::new( + kwarg, + Handling::Warn { + consequence, + met: Some(met), + }, + ) +} + +const fn reject(kwarg: &'static str, check: &'static str, asked: Asked) -> Setting { + Setting::new(kwarg, Handling::Reject { check, asked }) +} + +const NO_EQUIVALENT: &str = "the Rust client has no equivalent"; +const OWN_STACK: &str = "the Rust client uses its own pool, retries and events"; +const OWN_CONNECTION: &str = "it tunes redis-py's own connection, and the Rust client has its own"; +const OWN_CLUSTER: &str = "the Rust cluster client discovers, routes and retries on its own"; +const MAINTENANCE: &str = "the Rust client does not follow server maintenance notifications"; +const RUSTLS: &str = "rustls negotiates TLS 1.2+ with its own ECDHE AEAD suites"; +const REVOCATION: &str = "OCSP revocation checks"; +const OCSP: &str = "it only configures ssl_validate_ocsp, which the Rust client refuses when set"; +const VERIFY_FLAGS: &str = "X.509 verify flags such as CRL checking"; + +/// Every Redis kwarg redis-py or `litellm/_redis.py` accepts, one row each. `redis-py-kwargs.json` records which +/// redis-py versions and `REDIS_*` names accept them, and `tests/catalog.rs` keeps the two in step. +pub const SETTINGS: &[Setting] = &[ + warn("address_remap", OWN_CLUSTER), + warn("auto_close_connection_pool", OWN_STACK), + read("aws_iam_auth").env(), + read("aws_iam_cache_name").env(), + read("aws_iam_region") + .env() + .or_secrets(&["AWS_REGION", "AWS_DEFAULT_REGION"]), + read("aws_iam_serverless").env(), + read("aws_iam_user_name").env(), + read("azure_client_id") + .env() + .or_secrets(&["AZURE_CLIENT_ID"]), + read("azure_client_secret") + .env() + .or_secrets(&["AZURE_CLIENT_SECRET"]), + read("azure_redis_ad_token") + .env() + .or_secrets(&["REDIS_AZURE_AD_TOKEN"]), + read("azure_tenant_id") + .env() + .or_secrets(&["AZURE_TENANT_ID"]), + warn("cache", NO_EQUIVALENT).env(), + warn("cache_config", NO_EQUIVALENT).env(), + warn("charset", NO_EQUIVALENT).env(), + warn("client_name", NO_EQUIVALENT).env(), + warn("cluster_error_retry_attempts", OWN_CLUSTER), + warn("command_packer", OWN_CONNECTION), + warn("connection_class", OWN_STACK), + warn("connection_error_retry_attempts", OWN_CLUSTER), + warn("connection_pool", OWN_STACK), + read("credential_provider"), + read("db").env().in_url(FromUrl::Any), + warn("decode_responses", NO_EQUIVALENT).env(), + warn("driver_info", NO_EQUIVALENT), + warn("dynamic_startup_nodes", OWN_CLUSTER), + warn("encoder_class", OWN_CONNECTION), + warn("encoding", NO_EQUIVALENT).env(), + warn("encoding_errors", NO_EQUIVALENT).env(), + warn("errors", NO_EQUIVALENT).env(), + warn("event_dispatcher", OWN_STACK).env(), + read("gcp_service_account") + .env() + .or_secrets(&["REDIS_GCP_SERVICE_ACCOUNT"]), + read("gcp_ssl_ca_certs") + .env() + .or_secrets(&["REDIS_GCP_SSL_CA_CERTS"]), + read("health_check_interval").env().in_url(FromUrl::Any), + read("host").env(), + warn("legacy_responses", NO_EQUIVALENT), + warn("lib_name", NO_EQUIVALENT).env(), + warn("lib_version", NO_EQUIVALENT).env(), + warn("load_balancing_strategy", OWN_CLUSTER), + warn("maint_notifications_config", MAINTENANCE), + warn("maint_notifications_pool_handler", MAINTENANCE), + warn("maintenance_notification_hash", MAINTENANCE), + warn("maintenance_state", MAINTENANCE), + warn("max_connections", "the Rust client has no connection cap").env(), + read("namespace"), + warn("orig_host_address", MAINTENANCE), + warn("orig_socket_connect_timeout", MAINTENANCE), + warn("orig_socket_timeout", MAINTENANCE), + warn("oss_cluster_maint_notifications_handler", MAINTENANCE), + warn("parser", OWN_CONNECTION), + warn("parser_class", OWN_CONNECTION), + read("password").env().in_url(FromUrl::Any), + warn("path", OWN_CLUSTER), + warn("policy_resolver", OWN_CLUSTER), + read("port").env(), + warn("protocol", NO_EQUIVALENT).env(), + warn("read_from_replicas", OWN_CLUSTER), + warn( + "redis_connect_func", + "the Python object is not callable from Rust, so static username and password are used", + ) + .env(), + warn("reinitialize_steps", OWN_CLUSTER), + warn("require_full_coverage", OWN_CLUSTER), + warn("retry", OWN_STACK), + warn("retry_on_error", NO_EQUIVALENT).env(), + warn("retry_on_timeout", NO_EQUIVALENT).env(), + read("sentinel_nodes").or_secrets(&["REDIS_SENTINEL_NODES"]), + read("sentinel_password").or_secrets(&["REDIS_SENTINEL_PASSWORD"]), + read("service_name").or_secrets(&["REDIS_SERVICE_NAME"]), + warn("single_connection_client", NO_EQUIVALENT).env(), + read("socket_connect_timeout").env().in_url(FromUrl::Any), + read("socket_keepalive").env(), + warn("socket_keepalive_options", NO_EQUIVALENT).env(), + warn("socket_read_size", OWN_CONNECTION), + read("socket_timeout").env().in_url(FromUrl::Any), + warn("socket_type", OWN_CONNECTION), + read("ssl").env(), + read("ssl_ca_certs").env().in_url(FromUrl::Rediss), + read("ssl_ca_data").env().in_url(FromUrl::Rediss), + read("ssl_ca_path").env().in_url(FromUrl::Rediss), + read("ssl_cert_reqs").env().in_url(FromUrl::Rediss), + read("ssl_certfile").env().in_url(FromUrl::Rediss), + read("ssl_check_hostname").env().in_url(FromUrl::Rediss), + warn("ssl_ciphers", RUSTLS).env(), + reject("ssl_exclude_verify_flags", VERIFY_FLAGS, Asked::WhenSet), + reject("ssl_include_verify_flags", VERIFY_FLAGS, Asked::WhenSet), + read("ssl_keyfile").env().in_url(FromUrl::Rediss), + warn_unless("ssl_min_version", RUSTLS, is_met_by_rustls).env(), + warn("ssl_ocsp_context", OCSP).env(), + warn("ssl_ocsp_expected_cert", OCSP).env(), + read("ssl_password").env().in_url(FromUrl::Rediss), + reject("ssl_validate_ocsp", REVOCATION, Asked::WhenTrue).env(), + reject("ssl_validate_ocsp_stapled", REVOCATION, Asked::WhenTrue).env(), + read("startup_nodes").or_secrets(&["REDIS_CLUSTER_NODES"]), + read("unix_socket_path").env(), + read("url").env(), + read("username").env().in_url(FromUrl::Any), +]; + +// rustls only speaks TLS 1.2 and 1.3, so a minimum of 1.2 or lower is already met. redis-py takes an +// ssl.TLSVersion, which reaches Rust as its name or its value +fn is_met_by_rustls(value: &Value) -> bool { + match value { + Value::Number(number) => matches!(number.as_i64(), Some(-2 | 769 | 770 | 771)), + Value::String(text) => matches!( + text.strip_prefix("TLSVersion.") + .unwrap_or(text) + .to_ascii_uppercase() + .as_str(), + "MINIMUM_SUPPORTED" | "TLSV1" | "TLSV1_1" | "TLSV1_2" + ), + _ => false, + } +} + +pub fn setting(kwarg: &str) -> Option<&'static Setting> { + SETTINGS.iter().find(|setting| setting.kwarg == kwarg) +} + +/// The check a `Reject` row stands for and which values ask for it. +pub(crate) fn refusal(kwarg: &str) -> Option<(&'static str, Asked)> { + match setting(kwarg)?.handling { + Handling::Reject { check, asked } => Some((check, asked)), + Handling::Read | Handling::Warn { .. } => None, + } +} + +/// A warned setting the kwargs set, with what dropping it means. A kwarg with no row is not a Redis setting, so it +/// is dropped without a warning. +pub(crate) fn ignored(source: &str, kwarg: &str, value: &Value) -> Option { + let consequence = setting(kwarg)?.warning(value)?; + Some(Ignored { + setting: source.to_owned(), + consequence, + }) +} + +/// A url query parameter the resolver does not read. redis-py turns every one into a client kwarg, so even one with +/// no row is reported. +pub(crate) fn ignored_query(source: &str, name: &str, value: &Value) -> Option { + let consequence = match setting(name) { + None => NO_EQUIVALENT, + Some(setting) if setting.is_read() => { + "the Rust client does not read it from this url's query" + } + Some(setting) => setting.warning(value)?, + }; + Some(Ignored { + setting: source.to_owned(), + consequence, + }) +} diff --git a/litellm-rust/crates/redis-resolve/src/lib.rs b/litellm-rust/crates/redis-resolve/src/lib.rs new file mode 100644 index 00000000000..b7aa5d18f64 --- /dev/null +++ b/litellm-rust/crates/redis-resolve/src/lib.rs @@ -0,0 +1,95 @@ +use std::{collections::BTreeMap, time::Duration}; + +use litellm_redis_types::{Auth, HealthCheck, ProviderHandle, Resolution, StaticAuth, Target}; +use serde_json::Value; + +mod auth; +mod endpoint; +mod error; +mod keys; +mod options; +mod tls; + +pub use error::{Error, Expected, PythonException}; +pub use keys::{Asked, FromUrl, Handling, SETTINGS, Setting, setting}; +pub use options::is_present; + +use options::Options; + +/// One merged redis-py kwarg and the name a warning or error reports it under. +#[derive(Clone, Debug, PartialEq)] +pub struct Entry { + pub value: Value, + pub source: String, +} + +pub type Kwargs = BTreeMap; + +/// Kwargs whose every entry is reported under its own kwarg name. +pub fn kwargs(fields: impl IntoIterator) -> Kwargs { + fields + .into_iter() + .map(|(kwarg, value)| { + let entry = Entry { + value, + source: kwarg.clone(), + }; + (kwarg, entry) + }) + .collect() +} + +/// What redis-py's constructors make of kwargs a host already merged the way `_get_redis_client_logic` does. A +/// host passes its `credential_provider` object as `provider`, and `REDIS_USERNAME` as its process environment +/// holds it, which Python's Azure AD auth reads from `os.environ` instead of the merged kwargs. +pub fn interpret( + kwargs: &Kwargs, + provider: Option, + redis_username: Option<&str>, +) -> Result { + let (options, ignored) = Options::read(kwargs)?; + let auth = auth::redis_auth(&options, provider, redis_username)?; + let (target, query) = endpoint::target(&options, &auth)?; + let auth = match (&target, auth) { + (Target::Url(_), Auth::Static(StaticAuth { username, .. })) => Auth::Static(StaticAuth { + username, + password: None, + }), + (_, auth) => auth, + }; + let timeout = query.timeout.map_or_else( + || options::seconds(options.socket_timeout.as_ref()), + |timeout| Ok(Some(timeout)), + )?; + let connect_timeout = query.connect_timeout.map_or_else( + || options::seconds(options.socket_connect_timeout.as_ref()), + |timeout| Ok(Some(timeout)), + )?; + let health_check_interval = query.health_check_interval.map_or_else( + || options::integer::(options.health_check_interval.as_ref()), + |interval| Ok(Some(interval)), + )?; + Ok(Resolution { + target, + auth, + namespace: options::text(options.namespace.as_ref()), + timeout, + connect_timeout, + health_check: health_check(health_check_interval), + keepalive: options + .socket_keepalive + .as_ref() + .map_or(Ok(true), |entry| options::flag(Some(entry)))?, + ignored: ignored.into_iter().chain(query.ignored).collect(), + }) +} + +// redis-py checks health only when the interval is truthy, so zero turns it off and a negative interval checks +// before every command +fn health_check(seconds: Option) -> HealthCheck { + match seconds { + None => HealthCheck::Unset, + Some(0) => HealthCheck::Disabled, + Some(seconds) => HealthCheck::Every(Duration::from_secs(seconds.try_into().unwrap_or(0))), + } +} diff --git a/litellm-rust/crates/redis-resolve/src/options.rs b/litellm-rust/crates/redis-resolve/src/options.rs new file mode 100644 index 00000000000..f2539e46ca5 --- /dev/null +++ b/litellm-rust/crates/redis-resolve/src/options.rs @@ -0,0 +1,281 @@ +use std::time::Duration; + +use litellm_python_compat::{BigInt, number}; +use litellm_redis_types::Ignored; +use litellm_secret_value::SecretValue; +use serde_json::Value; + +use crate::{ + Entry, Error, Expected, Kwargs, + keys::{self, Asked}, +}; + +/// Every setting the resolver reads. A setting is `None` when it is unset, null or empty, except that a null +/// `tls.cert_reqs` is kept because redis-py reads `ssl_cert_reqs=None` as `CERT_NONE`. +pub(super) struct Options { + pub(super) url: Option, + pub(super) host: Option, + pub(super) port: Option, + pub(super) db: Option, + pub(super) unix_socket_path: Option, + pub(super) startup_nodes: Option, + pub(super) sentinel: SentinelOptions, + pub(super) username: Option, + pub(super) password: Option, + pub(super) credential_provider: Option, + pub(super) gcp: GcpOptions, + pub(super) azure: AzureOptions, + pub(super) aws: AwsOptions, + pub(super) ssl: Option, + pub(super) tls: TlsInput, + pub(super) namespace: Option, + pub(super) socket_timeout: Option, + pub(super) socket_connect_timeout: Option, + pub(super) health_check_interval: Option, + pub(super) socket_keepalive: Option, +} + +pub(super) struct SentinelOptions { + pub(super) nodes: Option, + pub(super) service_name: Option, + pub(super) password: Option, +} + +pub(super) struct GcpOptions { + pub(super) service_account: Option, + pub(super) ssl_ca_certs: Option, +} + +pub(super) struct AzureOptions { + pub(super) ad_token: Option, + pub(super) client_id: Option, + pub(super) tenant_id: Option, + pub(super) client_secret: Option, +} + +pub(super) struct AwsOptions { + pub(super) auth: Option, + pub(super) user_name: Option, + pub(super) cache_name: Option, + pub(super) region: Option, + pub(super) serverless: Option, +} + +/// The TLS settings, read from the kwargs or from a `rediss://` url's query. +pub(super) struct TlsInput { + pub(super) cert_reqs: Option, + pub(super) ca_certs: Option, + pub(super) ca_data: Option, + pub(super) ca_path: Option, + pub(super) certfile: Option, + pub(super) keyfile: Option, + pub(super) key_password: Option, + pub(super) check_hostname: Option, +} + +/// The settings the kwargs set that the Rust client drops, once none of them asks for a check it cannot perform. +fn ignored(kwargs: &Kwargs) -> Result, Error> { + refuse_unenforceable(kwargs.iter())?; + Ok(kwargs + .iter() + .filter_map(|(key, entry)| keys::ignored(&entry.source, key, &entry.value)) + .collect()) +} + +impl Options { + /// The typed settings and the ones the Rust client drops. + pub(super) fn read(kwargs: &Kwargs) -> Result<(Self, Vec), Error> { + let ignored = ignored(kwargs)?; + let options = Self::from_lookup(|key| kwargs.get(key).cloned()); + Ok((options, ignored)) + } + + fn from_lookup(lookup: impl Fn(&'static str) -> Option) -> Self { + let get = |key| lookup(key).filter(|entry| is_present(&entry.value)); + Self { + url: get("url"), + host: get("host"), + port: get("port"), + db: get("db"), + unix_socket_path: get("unix_socket_path"), + startup_nodes: get("startup_nodes"), + sentinel: SentinelOptions { + nodes: get("sentinel_nodes"), + service_name: get("service_name"), + password: get("sentinel_password"), + }, + username: get("username"), + password: get("password"), + credential_provider: get("credential_provider"), + gcp: GcpOptions { + service_account: get("gcp_service_account"), + ssl_ca_certs: get("gcp_ssl_ca_certs"), + }, + azure: AzureOptions { + ad_token: get("azure_redis_ad_token"), + client_id: get("azure_client_id"), + tenant_id: get("azure_tenant_id"), + client_secret: get("azure_client_secret"), + }, + aws: AwsOptions { + auth: get("aws_iam_auth"), + user_name: get("aws_iam_user_name"), + cache_name: get("aws_iam_cache_name"), + region: get("aws_iam_region"), + serverless: get("aws_iam_serverless"), + }, + ssl: get("ssl"), + tls: TlsInput { + cert_reqs: lookup("ssl_cert_reqs") + .filter(|entry| entry.value.is_null() || is_present(&entry.value)), + ca_certs: get("ssl_ca_certs"), + ca_data: get("ssl_ca_data"), + ca_path: get("ssl_ca_path"), + certfile: get("ssl_certfile"), + keyfile: get("ssl_keyfile"), + key_password: get("ssl_password"), + check_hostname: get("ssl_check_hostname"), + }, + namespace: get("namespace"), + socket_timeout: get("socket_timeout"), + socket_connect_timeout: get("socket_connect_timeout"), + health_check_interval: get("health_check_interval"), + socket_keepalive: get("socket_keepalive"), + } + } +} + +/// A `Reject` row whose value asks for its check fails, because Rust cannot perform it. +pub(super) fn refuse_unenforceable<'a>( + entries: impl IntoIterator, +) -> Result<(), Error> { + entries + .into_iter() + .filter_map(|(key, entry)| Some((keys::refusal(key)?, entry))) + .try_for_each(|((check, asked), entry)| { + let asks = match asked { + Asked::WhenTrue => flag(Some(entry))?, + Asked::WhenSet => is_present(&entry.value), + }; + match asks { + true => Err(Error::Unenforceable { + setting: entry.source.clone(), + check, + }), + false => Ok(()), + } + }) +} + +// Python reads most of these with `x or fallback`, so an empty value counts as unset everywhere +pub fn is_present(value: &Value) -> bool { + match value { + Value::Null => false, + Value::String(text) => !text.is_empty(), + Value::Array(items) => !items.is_empty(), + Value::Object(entries) => !entries.is_empty(), + Value::Bool(_) | Value::Number(_) => true, + } +} + +pub(super) fn invalid(setting: &str, expected: Expected) -> Error { + Error::Invalid { + setting: setting.to_owned(), + expected, + } +} + +pub(super) fn as_text(value: &Value) -> Option { + match value { + Value::String(text) => Some(text.clone()), + Value::Number(number) => Some(number.to_string()), + Value::Bool(flag) => Some(flag.to_string()), + _ => None, + } +} + +pub(super) fn text(entry: Option<&Entry>) -> Option { + entry.and_then(|entry| as_text(&entry.value)) +} + +pub(super) fn secret(entry: Option<&Entry>) -> Option { + text(entry).map(SecretValue::new) +} + +/// A flag every Python surface reads the same way. `_str_to_bool` takes only `true`, `1` and `yes` +/// as true while the proxy's pydantic models also take `on`, `t` and `y`, so text they disagree on, +/// or text neither accepts, is an error instead of a guess. +pub(super) fn flag(entry: Option<&Entry>) -> Result { + let Some(entry) = entry else { + return Ok(false); + }; + match &entry.value { + Value::Bool(flag) => Ok(*flag), + Value::Number(number) => Ok(number.as_f64().is_some_and(|number| number != 0.0)), + Value::String(text) => match text.to_ascii_lowercase().as_str() { + "true" | "1" | "yes" => Ok(true), + "false" | "0" | "no" | "off" | "f" | "n" => Ok(false), + _ => Err(invalid(&entry.source, Expected::Boolean)), + }, + _ => Err(invalid(&entry.source, Expected::Boolean)), + } +} + +pub(super) fn integer>(entry: Option<&Entry>) -> Result, Error> { + coerced(entry, Expected::Integer, number::int, |number| { + T::try_from(number).ok() + }) +} + +pub(super) fn seconds(entry: Option<&Entry>) -> Result, Error> { + coerced(entry, Expected::Seconds, number::float, |seconds| { + Duration::try_from_secs_f64(seconds).ok() + }) +} + +fn coerced( + entry: Option<&Entry>, + expected: Expected, + parse: impl Fn(&str) -> Option, + convert: impl Fn(N) -> Option, +) -> Result, Error> { + let Some((entry, text)) = entry.and_then(|entry| Some((entry, as_text(&entry.value)?))) else { + return Ok(None); + }; + match parse(&text) { + Some(number) => convert(number) + .map(Some) + .ok_or_else(|| invalid(&entry.source, expected)), + None => { + tracing::warn!(setting = %entry.source, "Redis setting is not a number, so the default is used like redis-py"); + Ok(None) + } + } +} + +#[cfg(test)] +mod tests { + use std::{cell::RefCell, collections::BTreeSet}; + + use rstest::rstest; + + use super::Options; + use crate::SETTINGS; + + // a Read row that Options never asks for would be neither modeled nor passed through nor warned about + #[rstest] + fn options_ask_for_exactly_the_read_rows() { + let asked = RefCell::new(BTreeSet::new()); + let _ = Options::from_lookup(|kwarg| { + asked.borrow_mut().insert(kwarg); + None + }); + + let read: BTreeSet<&str> = SETTINGS + .iter() + .filter(|setting| setting.is_read()) + .map(|setting| setting.kwarg) + .collect(); + assert_eq!(asked.into_inner(), read); + } +} diff --git a/litellm-rust/crates/redis-resolve/src/tls.rs b/litellm-rust/crates/redis-resolve/src/tls.rs new file mode 100644 index 00000000000..a46a6ccc650 --- /dev/null +++ b/litellm-rust/crates/redis-resolve/src/tls.rs @@ -0,0 +1,189 @@ +use std::time::Duration; + +use litellm_python_compat::number; +use litellm_redis_types::{Ignored, TlsOptions}; +use litellm_secret_value::SecretValue; + +use serde_json::Value; + +use crate::{ + Entry, Error, Expected, + keys::{self, FromUrl}, + options::{self, TlsInput, flag, invalid, secret, text}, +}; + +/// `gcp_ca_certs` replaces `ssl_ca_certs` when GCP IAM is on, as `_get_redis_client_logic` does. +pub(super) fn tls_options( + input: &TlsInput, + gcp_ca_certs: Option<&Entry>, +) -> Result { + Ok(TlsOptions { + ca_certs: text(gcp_ca_certs.or(input.ca_certs.as_ref())).map(Into::into), + ca_data: text(input.ca_data.as_ref()), + ca_path: text(input.ca_path.as_ref()).map(Into::into), + certfile: text(input.certfile.as_ref()).map(Into::into), + keyfile: text(input.keyfile.as_ref()).map(Into::into), + key_password: secret(input.key_password.as_ref()), + verify_certificate: verify_certificate(input.cert_reqs.as_ref())?, + check_hostname: flag(input.check_hostname.as_ref())?, + }) +} + +// RedisSSLContext looks text up case-sensitively and takes an ssl.VerifyMode as is +fn verify_certificate(cert_reqs: Option<&Entry>) -> Result { + let Some(entry) = cert_reqs else { + return Ok(true); + }; + match &entry.value { + // Explicit None intentionally means CERT_NONE, matching redis-py rather than the unset default. + Value::Null => Ok(false), + Value::String(text) if text == "none" => Ok(false), + Value::String(text) if text == "optional" || text == "required" => Ok(true), + Value::Number(mode) => match mode.as_i64() { + Some(0) => Ok(false), + Some(1 | 2) => Ok(true), + _ => Err(invalid(&entry.source, Expected::CertRequirement)), + }, + _ => Err(invalid(&entry.source, Expected::CertRequirement)), + } +} + +/// What redis-py's `parse_url` reads from a url's query on top of the url itself. +pub(super) struct UrlQuery { + pub(super) tls: Option, + pub(super) db: Option, + pub(super) username: Option, + pub(super) password: Option, + pub(super) settings: QuerySettings, +} + +/// The connection settings a url's query sets, which win over the kwargs because `from_url` applies them last. +#[derive(Default)] +pub(super) struct QuerySettings { + pub(super) timeout: Option, + pub(super) connect_timeout: Option, + pub(super) health_check_interval: Option, + pub(super) ignored: Vec, +} + +pub(super) fn url_query(url: &str) -> Result { + let Ok(url) = url::Url::parse(url) else { + return Ok(UrlQuery { + tls: None, + db: None, + username: None, + password: None, + settings: QuerySettings::default(), + }); + }; + let tls_scheme = url.scheme().eq_ignore_ascii_case("rediss"); + let pairs: Vec<(String, Entry)> = url + .query_pairs() + .filter(|(_, value)| !value.is_empty()) + .map(|(name, value)| { + ( + name.to_string(), + Entry { + value: Value::String(value.into_owned()), + source: format!("url ?{name}"), + }, + ) + }) + .collect(); + // parse_qs keeps every value of a repeated parameter and parse_url reads the first + let query: Vec<(String, Entry)> = pairs + .iter() + .enumerate() + .filter(|(index, (name, _))| !pairs[..*index].iter().any(|(earlier, _)| earlier == name)) + .map(|(_, pair)| pair.clone()) + .collect(); + options::refuse_unenforceable(query.iter().map(|(name, entry)| (name, entry)))?; + let (query, ignored): (Vec<_>, Vec<_>) = query.into_iter().partition(|(name, _)| { + keys::setting(name).is_some_and(|setting| match setting.from_url { + FromUrl::Any => true, + FromUrl::Rediss => tls_scheme, + FromUrl::No => false, + }) + }); + let get = |key: &str| { + query + .iter() + .find(|(name, _)| name == key) + .map(|(_, entry)| entry.clone()) + }; + let seconds = |key: &str| { + query_number( + get(key).as_ref(), + number::float, + Expected::Seconds, + |seconds| Duration::try_from_secs_f64(seconds).ok(), + ) + }; + let integer = |key: &str| { + query_number( + get(key).as_ref(), + number::int, + Expected::Integer, + |number| i64::try_from(number).ok(), + ) + }; + let db = integer("db")?; + let settings = QuerySettings { + timeout: seconds("socket_timeout")?, + connect_timeout: seconds("socket_connect_timeout")?, + health_check_interval: integer("health_check_interval")?, + ignored: ignored + .iter() + .filter_map(|(name, entry)| keys::ignored_query(&entry.source, name, &entry.value)) + .collect(), + }; + let tls = match tls_scheme { + true => Some(TlsOptions { + ca_certs: text(get("ssl_ca_certs").as_ref()).map(Into::into), + ca_data: text(get("ssl_ca_data").as_ref()), + ca_path: text(get("ssl_ca_path").as_ref()).map(Into::into), + certfile: text(get("ssl_certfile").as_ref()).map(Into::into), + keyfile: text(get("ssl_keyfile").as_ref()).map(Into::into), + key_password: secret(get("ssl_password").as_ref()), + verify_certificate: verify_certificate(get("ssl_cert_reqs").as_ref())?, + check_hostname: text(get("ssl_check_hostname").as_ref()) + .is_some_and(|text| url_bool(&text)), + }), + false => None, + }; + Ok(UrlQuery { + tls, + db, + username: secret(get("username").as_ref()), + password: secret(get("password").as_ref()), + settings, + }) +} + +// parse_url reads these with int() or float() and raises ValueError building the client on anything else +fn query_number( + entry: Option<&Entry>, + parse: impl Fn(&str) -> Option, + expected: Expected, + convert: impl Fn(N) -> Option, +) -> Result, Error> { + let Some(entry) = entry else { + return Ok(None); + }; + let number = text(Some(entry)) + .and_then(|text| parse(&text)) + .ok_or_else(|| Error::UrlQuery { + setting: entry.source.clone(), + })?; + convert(number) + .map(Some) + .ok_or_else(|| invalid(&entry.source, expected)) +} + +// redis-py's to_bool: only 0, F, FALSE, N and NO (any case) are false +fn url_bool(text: &str) -> bool { + !matches!( + text.to_ascii_uppercase().as_str(), + "" | "0" | "F" | "FALSE" | "N" | "NO" + ) +} diff --git a/litellm-rust/crates/redis-resolve/tests/catalog.rs b/litellm-rust/crates/redis-resolve/tests/catalog.rs new file mode 100644 index 00000000000..b26ad0eebfb --- /dev/null +++ b/litellm-rust/crates/redis-resolve/tests/catalog.rs @@ -0,0 +1,129 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use litellm_redis_resolve::{Asked, Error, Handling, SETTINGS, Setting, interpret, kwargs}; +use rstest::rstest; +use serde::Deserialize; +use serde_json::json; + +#[derive(Deserialize)] +struct Catalog { + meta: Meta, + args: BTreeMap, +} + +#[derive(Deserialize)] +struct Meta { + redis_py_locked: String, +} + +#[derive(Deserialize)] +struct Arg { + environment: bool, + redis_py: BTreeMap>, +} + +// the signatures of the clients the proxy builds, where an unknown kwarg raises TypeError +const PROXY_SIGNATURES: &[&str] = &["async", "cluster", "connection"]; + +fn rejected() -> impl Iterator { + SETTINGS + .iter() + .filter_map(|setting| match setting.handling { + Handling::Reject { asked, .. } => Some((setting, asked)), + Handling::Read | Handling::Warn { .. } => None, + }) +} + +fn catalog() -> Catalog { + serde_json::from_str(include_str!("../redis-py-kwargs.json")).unwrap() +} + +fn host_with(kwarg: &str, value: serde_json::Value) -> litellm_redis_resolve::Kwargs { + kwargs([ + ("host".to_owned(), json!("redis")), + (kwarg.to_owned(), value), + ]) +} + +#[rstest] +fn every_catalogued_kwarg_has_exactly_one_settings_row() { + let rows: Vec<&str> = SETTINGS.iter().map(|setting| setting.kwarg).collect(); + let unique: BTreeSet<&str> = rows.iter().copied().collect(); + assert_eq!(rows.len(), unique.len(), "a kwarg has two SETTINGS rows"); + + let catalog = catalog(); + let catalogued: BTreeSet<&str> = catalog.args.keys().map(String::as_str).collect(); + assert_eq!(unique, catalogued); +} + +#[rstest] +fn settings_read_redis_environment_names_exactly_where_python_does() { + let catalog = catalog(); + + let differing: Vec<&str> = SETTINGS + .iter() + .filter(|setting| { + catalog + .args + .get(setting.kwarg) + .is_some_and(|arg| arg.environment != setting.environment) + }) + .map(|setting| setting.kwarg) + .collect(); + assert!(differing.is_empty(), "{differing:?}"); +} + +#[rstest] +fn every_warned_setting_is_reported() { + let warned = SETTINGS + .iter() + .filter(|setting| matches!(setting.handling, Handling::Warn { met: None, .. })); + + for setting in warned { + let resolved = interpret(&host_with(setting.kwarg, json!("set")), None, None).unwrap(); + + let reported: Vec<&str> = resolved + .ignored + .iter() + .map(|ignored| ignored.setting.as_str()) + .collect(); + assert_eq!(reported, vec![setting.kwarg]); + } +} + +#[rstest] +fn every_rejected_setting_fails_when_asked_for_and_not_otherwise() { + for (setting, asked) in rejected() { + let (asking, not_asking) = match asked { + Asked::WhenTrue => (json!(true), json!(false)), + Asked::WhenSet => (json!(["VERIFY_CRL_CHECK_LEAF"]), json!([])), + }; + + let error = interpret(&host_with(setting.kwarg, asking), None, None).unwrap_err(); + assert!( + matches!(&error, Error::Unenforceable { setting: source, .. } if source == setting.kwarg), + "{error:?}" + ); + assert!(interpret(&host_with(setting.kwarg, not_asking), None, None).is_ok()); + } +} + +#[rstest] +fn python_already_refuses_every_rejected_setting() { + let catalog = catalog(); + + let accepted: Vec<&str> = rejected() + .map(|(setting, _)| setting.kwarg) + .filter(|kwarg| { + catalog.args[*kwarg] + .redis_py + .get(&catalog.meta.redis_py_locked) + .is_some_and(|signatures| { + signatures + .iter() + .any(|signature| PROXY_SIGNATURES.contains(&signature.as_str())) + }) + }) + .collect(); + assert!(accepted.is_empty(), "{accepted:?}"); +} diff --git a/litellm-rust/crates/redis-resolve/tests/interpret.rs b/litellm-rust/crates/redis-resolve/tests/interpret.rs new file mode 100644 index 00000000000..fd59ec2acb1 --- /dev/null +++ b/litellm-rust/crates/redis-resolve/tests/interpret.rs @@ -0,0 +1,790 @@ +use std::collections::BTreeMap; + +use litellm_redis_resolve::{Entry, Error, Expected, Kwargs, PythonException, interpret, kwargs}; +use litellm_redis_types::{ + Auth, HealthCheck, Node, ProviderHandle, Resolution, StaticAuth, Target, TlsOptions, +}; +use rstest::rstest; +use serde_json::json; + +use Outcome::{Fails, Refuses, Resolves}; +use PythonException::{RedisClusterException, TypeError, ValueError}; + +const PROVIDER: ProviderHandle = ProviderHandle::new(7); + +fn merged(fields: serde_json::Value, sources: &[(&str, &str)]) -> Kwargs { + let fields = serde_json::from_value::>(fields).unwrap(); + kwargs(fields) + .into_iter() + .map(|(kwarg, entry)| { + let source = sources + .iter() + .find(|(named, _)| *named == kwarg) + .map_or(entry.source, |(_, source)| (*source).to_owned()); + (kwarg, Entry { source, ..entry }) + }) + .collect() +} + +// a host hands its credential_provider object over as a handle, so the cases do the same +fn interpret_merged( + fields: serde_json::Value, + sources: &[(&str, &str)], + redis_username: Option<&str>, +) -> Result { + let has_provider = fields + .get("credential_provider") + .is_some_and(|provider| !provider.is_null()); + interpret( + &merged(fields, sources), + has_provider.then_some(PROVIDER), + redis_username, + ) +} + +fn interpret_kwargs(fields: serde_json::Value) -> Result { + interpret_merged(fields, &[], None) +} + +/// What a contract case expects of the resolver, next to what its Python mirror expects of redis-py. +enum Outcome { + Resolves(serde_json::Value), + /// Rust refuses a config Python connects with, a disagreement on purpose. + Refuses(&'static str), + /// redis-py 5.3.1 already raises on the config, building the client or its first connection, so failing + /// here breaks no deployment that works today. The class is what it raises building the client, and `None` + /// when it only raises once a command connects. + Fails(&'static str, Option), +} + +#[rstest] +fn a_provider_handle_beats_every_cloud_auth_and_static_credentials() { + let resolved = interpret_kwargs(json!({ + "host": "redis", "password": "pass", "credential_provider": "host-object", + "gcp_service_account": "account", "aws_iam_auth": true, "azure_redis_ad_token": "true", + })) + .unwrap(); + + assert_eq!(resolved.auth, Auth::Provider(PROVIDER)); +} + +#[rstest] +fn a_rediss_url_reads_ssl_password_from_its_query() { + let resolved = + interpret_kwargs(json!({"url": "rediss://cache:6380/0?ssl_password=query-key-secret"})) + .unwrap(); + + let Target::Url(parsed) = resolved.target else { + panic!("expected a url target, got {:?}", resolved.target); + }; + let key_password = parsed.tls.and_then(|tls| tls.key_password); + assert_eq!(expose(key_password.as_ref()), Some("query-key-secret")); + assert!(resolved.ignored.is_empty(), "{:?}", resolved.ignored); +} + +#[rstest] +#[case::read_on_a_plain_url( + "redis://cache:6379/1?db=3&username=query-user&password=query-secret", + Some(3), + Some("query-user"), + Some("query-secret") +)] +#[case::absent("rediss://cache:6380/1", None, None, None)] +fn a_url_query_carries_db_and_credentials_like_parse_url( + #[case] url: &str, + #[case] db: Option, + #[case] username: Option<&str>, + #[case] password: Option<&str>, +) { + let resolved = interpret_kwargs(json!({"url": url})).unwrap(); + + let Target::Url(parsed) = resolved.target else { + panic!("expected a url target, got {:?}", resolved.target); + }; + assert_eq!(parsed.db, db); + assert_eq!(expose(parsed.username.as_ref()), username); + assert_eq!(expose(parsed.password.as_ref()), password); + assert!(resolved.ignored.is_empty(), "{:?}", resolved.ignored); +} + +#[rstest] +fn a_url_query_db_that_is_not_an_integer_fails_like_parse_url() { + let error = interpret_kwargs(json!({"url": "redis://cache:6379?db=first"})).unwrap_err(); + + assert!(matches!(error, Error::UrlQuery { setting } if setting == "url ?db")); +} + +#[rstest] +#[case::tls_1_2_is_already_met(json!("TLSv1_2"), false)] +#[case::the_enum_name(json!("TLSVersion.TLSv1_2"), false)] +#[case::the_enum_value(json!(771), false)] +#[case::tls_1_3_is_not_enforced(json!("TLSv1_3"), true)] +fn ssl_min_version_warns_only_when_rustls_cannot_meet_it( + #[case] version: serde_json::Value, + #[case] warns: bool, +) { + let resolved = + interpret_kwargs(json!({"host": "redis", "ssl": true, "ssl_min_version": version})) + .unwrap(); + + assert_eq!( + !resolved.ignored.is_empty(), + warns, + "{:?}", + resolved.ignored + ); +} + +#[rstest] +#[case::unset(json!({"host": "redis"}), true)] +#[case::empty_like_unset(json!({"host": "redis", "socket_keepalive": ""}), true)] +#[case::enabled(json!({"host": "redis", "socket_keepalive": "yes"}), true)] +#[case::disabled(json!({"host": "redis", "socket_keepalive": false}), false)] +#[case::disabled_as_text(json!({"host": "redis", "socket_keepalive": "0"}), false)] +#[case::disabled_on_a_cluster(json!({"startup_nodes": [{"host": "node-a", "port": 7000}], "socket_keepalive": "false"}), false)] +fn keepalive_stays_on_unless_socket_keepalive_turns_it_off( + #[case] kwargs: serde_json::Value, + #[case] keepalive: bool, +) { + let resolved = interpret_kwargs(kwargs).unwrap(); + + assert_eq!(resolved.keepalive, keepalive); + assert!(resolved.ignored.is_empty(), "{:?}", resolved.ignored); +} + +#[rstest] +fn a_socket_keepalive_no_python_surface_agrees_on_is_invalid() { + let error = interpret_kwargs(json!({"host": "redis", "socket_keepalive": "on"})).unwrap_err(); + + assert!( + matches!(&error, Error::Invalid { setting, expected: Expected::Boolean } if setting == "socket_keepalive"), + "{error:?}" + ); +} + +fn tls_view(tls: Option<&TlsOptions>) -> serde_json::Value { + tls.map_or(serde_json::Value::Null, |tls| { + serde_json::json!({ + "ca_certs": tls.ca_certs, + "ca_data": tls.ca_data, + "ca_path": tls.ca_path, + "certfile": tls.certfile, + "keyfile": tls.keyfile, + "verify_certificate": tls.verify_certificate, + "check_hostname": tls.check_hostname, + }) + }) +} + +fn nodes_view(nodes: &[Node]) -> Vec { + nodes + .iter() + .map(|node| format!("{}:{}", node.host, node.port)) + .collect() +} + +fn expose(value: Option<&litellm_secret_value::SecretValue>) -> Option<&str> { + value.map(litellm_secret_value::SecretValue::expose) +} + +fn view(resolved: &Resolution) -> serde_json::Value { + let target = match &resolved.target { + Target::Unix(socket) => serde_json::json!({ + "mode": "unix", "path": socket.path, "tls": null, + }), + Target::Standalone(host) => serde_json::json!({ + "mode": "host", + "host": host.host, + "port": host.port, + "db": host.db, + "tls": tls_view(host.tls.as_ref()), + }), + Target::Url(url) => serde_json::json!({ + "mode": "url", + "url": url.url.expose(), + "tls": tls_view(url.tls.as_ref()), + }), + Target::Cluster(cluster) => serde_json::json!({ + "mode": "cluster", + "nodes": nodes_view(&cluster.nodes), + "tls": tls_view(cluster.tls.as_ref()), + }), + Target::Sentinel(sentinel) => serde_json::json!({ + "mode": "sentinel", + "nodes": nodes_view(&sentinel.monitors), + "service_name": sentinel.service_name, + "sentinel_password": expose(sentinel.monitor_auth.as_ref().and_then(|auth| auth.password.as_ref())), + "db": sentinel.db, + "tls": tls_view(sentinel.tls.as_ref()), + }), + }; + let auth = match &resolved.auth { + Auth::Static(StaticAuth { username, password }) => serde_json::json!({ + "kind": "static", + "username": expose(username.as_ref()), + "password": expose(password.as_ref()), + }), + Auth::Provider(_) => serde_json::json!({"kind": "host"}), + Auth::GcpIam { service_account } => { + serde_json::json!({"kind": "gcp_iam", "service_account": service_account}) + } + Auth::AzureAd(azure) => { + serde_json::json!({"kind": "azure_ad", "username": azure.username}) + } + Auth::AwsIam(aws) => serde_json::json!({ + "kind": "aws_iam", + "user_name": aws.user_name, + "cache_name": aws.cache_name, + "region": aws.region, + "serverless": aws.serverless, + }), + }; + let serde_json::Value::Object(target) = target else { + unreachable!("every target view is an object") + }; + let rest = serde_json::json!({ + "auth": auth, + "socket_timeout": resolved.timeout.map_or(5.0, |timeout| timeout.as_secs_f64()), + "socket_connect_timeout": resolved.connect_timeout.or(resolved.timeout).map_or(5.0, |timeout| timeout.as_secs_f64()), + "health_check_interval": match (&resolved.target, resolved.health_check) { + (_, HealthCheck::Every(interval)) => Some(interval.as_secs_f64()), + (Target::Cluster(_), HealthCheck::Unset) => Some(25.0), + (_, HealthCheck::Unset | HealthCheck::Disabled) => None, + }, + }); + let serde_json::Value::Object(rest) = rest else { + unreachable!("the view is an object") + }; + serde_json::Value::Object(target.into_iter().chain(rest).collect()) +} + +fn expected_view(fields: serde_json::Value) -> serde_json::Value { + let serde_json::Value::Object(fields) = fields else { + unreachable!("expected views are objects") + }; + let defaults = json!({ + "auth": {"kind": "static", "username": null, "password": null}, + "tls": null, + "socket_timeout": 5.0, + "socket_connect_timeout": 5.0, + "health_check_interval": null, + }); + let serde_json::Value::Object(defaults) = defaults else { + unreachable!("the defaults are an object") + }; + serde_json::Value::Object(defaults.into_iter().chain(fields).collect()) +} + +#[rstest] +#[case::host_defaults( + json!({"host": "redis"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0}))), +)] +#[case::empty_cluster_nodes_use_the_environment( + json!({"startup_nodes": "[{\"host\": \"node-a\", \"port\": 7000}]"}), + &[("startup_nodes", "REDIS_CLUSTER_NODES")], + None, + Resolves(expected_view(json!({"mode": "cluster", "nodes": ["node-a:7000"], "health_check_interval": 25.0}))), +)] +#[case::empty_sentinel_nodes_use_the_environment( + json!({"sentinel_nodes": "[[\"sentinel-a\", 26379]]", "service_name": "primary"}), + &[("sentinel_nodes", "REDIS_SENTINEL_NODES")], + None, + Resolves(expected_view(json!({"mode": "sentinel", "nodes": ["sentinel-a:26379"], "service_name": "primary", "sentinel_password": null, "db": 0}))), +)] +#[case::empty_service_name_uses_the_environment( + json!({"sentinel_nodes": [["sentinel-a", 26379]], "service_name": "primary"}), + &[("service_name", "REDIS_SERVICE_NAME")], + None, + Resolves(expected_view(json!({"mode": "sentinel", "nodes": ["sentinel-a:26379"], "service_name": "primary", "sentinel_password": null, "db": 0}))), +)] +#[case::plaintext_ignores_tls_requirements( + json!({"host": "redis", "ssl_cert_reqs": "strict"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0}))), +)] +#[case::an_ambiguous_flag_is_refused( + json!({"host": "redis", "ssl": "on"}), + &[], + None, + Refuses("invalid Redis setting ssl: expected a boolean"), +)] +#[case::empty_sentinel_nodes_without_the_environment_name_no_target( + json!({"sentinel_nodes": [], "service_name": "primary"}), + &[], + None, + Fails("either host or url must be specified for redis", Some(ValueError)), +)] +#[case::ssl_ca_path_is_unsupported( + json!({"host": "redis", "ssl": true, "ssl_ca_path": "/ca"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": "/ca", "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}}))), +)] +#[case::ssl_validate_ocsp_is_unsupported( + json!({"host": "redis", "ssl": true, "ssl_validate_ocsp": true}), + &[], + None, + Fails("ssl_validate_ocsp asks for OCSP revocation checks, which rustls cannot perform, so Redis is not connected without them", Some(TypeError)), +)] +#[case::ssl_include_verify_flags_is_unsupported( + json!({"host": "redis", "ssl": true, "ssl_include_verify_flags": ["VERIFY_CRL_CHECK_LEAF"]}), + &[], + None, + Fails("ssl_include_verify_flags asks for X.509 verify flags such as CRL checking, which rustls cannot perform, so Redis is not connected without them", Some(TypeError)), +)] +#[case::ssl_password_is_unsupported( + json!({"host": "redis", "ssl": true, "ssl_certfile": "/client.pem", "ssl_password": "key-secret"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": "/client.pem", "keyfile": null, "verify_certificate": true, "check_hostname": false}}))), +)] +#[case::host_with_every_basic_setting( + json!({"db": 2, "host": "redis", "password": "pass", "port": 6380, "socket_timeout": 1.5, "ssl": true, "username": "user"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6380, "db": 2, "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}, "auth": {"kind": "static", "username": "user", "password": "pass"}, "socket_timeout": 1.5, "socket_connect_timeout": 1.5}))), +)] +#[case::environment_host_port_and_password( + json!({"port": "6390", "ssl": true, "host": "environment", "password": "environment-secret"}), + &[("host", "REDIS_HOST"), ("password", "REDIS_PASSWORD"), ("port", "REDIS_PORT"), ("ssl", "REDIS_SSL")], + None, + Resolves(expected_view(json!({"mode": "host", "host": "environment", "port": 6390, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}, "auth": {"kind": "static", "username": null, "password": "environment-secret"}}))), +)] +#[case::string_numbers_are_coerced( + json!({"db": "3", "host": "redis", "port": "6380", "socket_timeout": "2.5"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6380, "db": 3, "socket_timeout": 2.5, "socket_connect_timeout": 2.5}))), +)] +#[case::padded_and_underscored_numbers_parse_like_python_int_and_float( + json!({"host": "redis", "port": "6380\n", "db": " 2 ", "socket_timeout": "1_0.5\n"}), + &[], + None, + Resolves(expected_view(json!({"db": 2, "host": "redis", "mode": "host", "port": 6380, "socket_timeout": 10.5, "socket_connect_timeout": 10.5}))), +)] +#[case::padded_environment_numbers_parse_like_python_int( + json!({"port": "6_380", "host": "redis", "db": "3\n"}), + &[("db", "REDIS_DB"), ("host", "REDIS_HOST"), ("port", "REDIS_PORT")], + None, + Resolves(expected_view(json!({"db": 3, "host": "redis", "mode": "host", "port": 6380}))), +)] +#[case::socket_timeout_from_the_environment( + json!({"socket_timeout": "0.75", "host": "redis"}), + &[("socket_timeout", "REDIS_SOCKET_TIMEOUT")], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "socket_timeout": 0.75, "socket_connect_timeout": 0.75}))), +)] +#[case::environment_url( + json!({"url": "redis://environment:6379/1"}), + &[("url", "REDIS_URL")], + None, + Resolves(expected_view(json!({"mode": "url", "url": "redis://environment:6379/1"}))), +)] +#[case::environment_url_beats_the_environment_host( + json!({"host": "ignored", "url": "redis://from-url:6379"}), + &[("host", "REDIS_HOST"), ("url", "REDIS_URL")], + None, + Resolves(expected_view(json!({"mode": "url", "url": "redis://from-url:6379"}))), +)] +#[case::an_explicit_host_drops_the_environment_url( + json!({"host": "configured"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "configured", "port": 6379, "db": 0}))), +)] +#[case::a_url_drops_host_db_and_password_and_keeps_the_username( + json!({"db": 4, "host": "dropped", "password": "dropped", "url": "rediss://cache:6380", "username": "user"}), + &[], + None, + Resolves(expected_view(json!({"mode": "url", "url": "rediss://cache:6380", "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}, "auth": {"kind": "static", "username": "user", "password": null}}))), +)] +#[case::a_url_ignores_tls_settings_outside_the_url( + json!({"ssl_ca_certs": "/ca.pem", "ssl_cert_reqs": "none", "ssl_check_hostname": true, "url": "rediss://cache:6380"}), + &[], + None, + Resolves(expected_view(json!({"mode": "url", "url": "rediss://cache:6380", "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}}))), +)] +#[case::startup_nodes_beat_a_url( + json!({"password": "cluster-secret", "startup_nodes": [{"host": "node-a", "port": 7000}], "url": "redis://single:6379"}), + &[], + None, + Resolves(expected_view(json!({"mode": "cluster", "nodes": ["node-a:7000"], "auth": {"kind": "static", "username": null, "password": "cluster-secret"}, "health_check_interval": 25.0}))), +)] +#[case::environment_cluster_nodes( + json!({"startup_nodes": "[{\"host\": \"node-a\", \"port\": \"7000\"}, {\"host\": \"node-b\", \"port\": 7001}]"}), + &[("startup_nodes", "REDIS_CLUSTER_NODES")], + None, + Resolves(expected_view(json!({"mode": "cluster", "nodes": ["node-a:7000", "node-b:7001"], "health_check_interval": 25.0}))), +)] +#[case::cluster_tls_options( + json!({"ssl": "true", "ssl_ca_certs": "/ca.pem", "ssl_check_hostname": true, "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + None, + Resolves(expected_view(json!({"mode": "cluster", "nodes": ["node-a:7000"], "tls": {"ca_certs": "/ca.pem", "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": true}, "health_check_interval": 25.0}))), +)] +#[case::sentinel_from_settings( + json!({"password": "data-secret", "sentinel_nodes": [["sentinel-a", 26379], ["sentinel-b", 26380]], "sentinel_password": "sentinel-secret", "service_name": "primary"}), + &[], + None, + Resolves(expected_view(json!({"mode": "sentinel", "nodes": ["sentinel-a:26379", "sentinel-b:26380"], "service_name": "primary", "sentinel_password": "sentinel-secret", "db": 0, "auth": {"kind": "static", "username": null, "password": "data-secret"}}))), +)] +#[case::sentinel_from_the_environment( + json!({"sentinel_nodes": "[[\"sentinel-a\", 26379]]", "sentinel_password": "sentinel-secret", "service_name": "primary"}), + &[("sentinel_nodes", "REDIS_SENTINEL_NODES"), ("sentinel_password", "REDIS_SENTINEL_PASSWORD"), ("service_name", "REDIS_SERVICE_NAME")], + None, + Resolves(expected_view(json!({"mode": "sentinel", "nodes": ["sentinel-a:26379"], "service_name": "primary", "sentinel_password": "sentinel-secret", "db": 0}))), +)] +#[case::a_host_breaks_sentinel( + json!({"host": "ignored", "sentinel_nodes": [["sentinel-a", 26379]], "service_name": "primary"}), + &[], + None, + Fails("host cannot be combined with sentinel_nodes, which redis-py rejects at startup", Some(TypeError)), +)] +#[case::a_url_beats_sentinels( + json!({"sentinel_nodes": [["sentinel-a", 26379]], "service_name": "primary", "url": "redis://single:6379"}), + &[], + None, + Resolves(expected_view(json!({"mode": "url", "url": "redis://single:6379"}))), +)] +#[case::an_explicit_credential_provider_beats_cloud_auth( + json!({"aws_iam_auth": true, "credential_provider": true, "gcp_service_account": "account", "host": "redis", "password": "dropped"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "auth": {"kind": "host"}}))), +)] +#[case::gcp_beats_azure_and_aws( + json!({"gcp_service_account": "projects/-/serviceAccounts/sa@project.iam.gserviceaccount.com", "aws_iam_auth": true, "azure_redis_ad_token": "True", "host": "redis", "password": "dropped"}), + &[("gcp_service_account", "REDIS_GCP_SERVICE_ACCOUNT")], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "auth": {"kind": "gcp_iam", "service_account": "projects/-/serviceAccounts/sa@project.iam.gserviceaccount.com"}}))), +)] +#[case::gcp_ca_certs_replace_the_configured_ones_under_ssl( + json!({"gcp_service_account": "account", "gcp_ssl_ca_certs": "/gcp-ca.pem", "host": "redis", "ssl": true, "ssl_ca_certs": "/configured-ca.pem"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": "/gcp-ca.pem", "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}, "auth": {"kind": "gcp_iam", "service_account": "account"}}))), +)] +#[case::azure_beats_aws_and_takes_the_environment_username( + json!({"aws_iam_auth": true, "azure_client_id": "client", "azure_client_secret": "secret", "azure_redis_ad_token": true, "azure_tenant_id": "tenant", "host": "redis", "username": "dropped"}), + &[], + Some("environment-user"), + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "auth": {"kind": "azure_ad", "username": "environment-user"}}))), +)] +#[case::an_empty_environment_username_sends_only_the_token( + json!({"azure_client_id": "client", "azure_client_secret": "secret", "azure_redis_ad_token": true, "azure_tenant_id": "tenant", "host": "redis", "username": ""}), + &[("username", "REDIS_USERNAME")], + Some(""), + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "auth": {"kind": "azure_ad", "username": null}}))), +)] +#[case::azure_without_a_username_sends_only_the_token( + json!({"azure_client_id": "client", "azure_client_secret": "secret", "azure_redis_ad_token": true, "azure_tenant_id": "tenant", "host": "redis"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "auth": {"kind": "azure_ad", "username": null}}))), +)] +#[case::azure_needs_the_literal_true( + json!({"azure_client_id": "client", "azure_client_secret": "secret", "azure_redis_ad_token": "1", "azure_tenant_id": "tenant", "host": "redis"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0}))), +)] +#[case::aws_iam_over_tls_with_the_default_region( + json!({"aws_iam_auth": "yes", "aws_iam_cache_name": "cache-1", "aws_iam_user_name": "iam-user", "host": "cache-1", "password": "dropped", "ssl": true, "aws_iam_region": "us-east-1"}), + &[("aws_iam_region", "AWS_DEFAULT_REGION")], + None, + Resolves(expected_view(json!({"mode": "host", "host": "cache-1", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}, "auth": {"kind": "aws_iam", "user_name": "iam-user", "cache_name": "cache-1", "region": "us-east-1", "serverless": false}}))), +)] +#[case::aws_region_beats_the_default_region( + json!({"aws_iam_auth": true, "aws_iam_cache_name": "cache-1", "aws_iam_user_name": "iam-user", "host": "cache-1", "ssl": true, "aws_iam_region": "eu-central-1"}), + &[("aws_iam_region", "AWS_REGION")], + None, + Resolves(expected_view(json!({"mode": "host", "host": "cache-1", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}, "auth": {"kind": "aws_iam", "user_name": "iam-user", "cache_name": "cache-1", "region": "eu-central-1", "serverless": false}}))), +)] +#[case::aws_iam_serverless_over_a_rediss_url_drops_its_userinfo( + json!({"aws_iam_auth": true, "aws_iam_cache_name": "cache-1", "aws_iam_region": "eu-west-1", "aws_iam_serverless": true, "aws_iam_user_name": "iam-user", "url": "rediss://user:pass@cache-1:6379"}), + &[], + None, + Resolves(expected_view(json!({"mode": "url", "url": "rediss://cache-1:6379", "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}, "auth": {"kind": "aws_iam", "user_name": "iam-user", "cache_name": "cache-1", "region": "eu-west-1", "serverless": true}}))), +)] +#[case::aws_iam_without_tls_fails( + json!({"aws_iam_auth": true, "aws_iam_cache_name": "cache-1", "aws_iam_region": "us-east-1", "aws_iam_user_name": "iam-user", "host": "cache-1"}), + &[], + None, + Fails("AWS ElastiCache IAM Redis authentication requires TLS", Some(ValueError)), +)] +#[case::aws_iam_without_a_region_fails( + json!({"aws_iam_auth": true, "aws_iam_cache_name": "cache-1", "aws_iam_user_name": "iam-user", "host": "cache-1", "ssl": true}), + &[], + None, + Fails("AWS ElastiCache IAM Redis authentication requires: aws_iam_region", Some(ValueError)), +)] +#[case::tls_options_from_settings( + json!({"host": "redis", "ssl": true, "ssl_ca_certs": "/ca.pem", "ssl_cert_reqs": "none", "ssl_check_hostname": true}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": "/ca.pem", "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": false, "check_hostname": true}}))), +)] +#[case::a_null_cert_requirement_skips_verification( + json!({"host": "redis", "ssl": true, "ssl_cert_reqs": null}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": false, "check_hostname": false}}))), +)] +#[case::a_verify_mode_of_none_skips_verification( + json!({"host": "redis", "ssl": true, "ssl_cert_reqs": 0}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": false, "check_hostname": false}}))), +)] +#[case::a_verify_mode_of_optional_verifies( + json!({"host": "redis", "ssl": true, "ssl_cert_reqs": 1}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}}))), +)] +#[case::a_cert_requirement_spelled_like_python_none_fails( + json!({"ssl": true, "ssl_cert_reqs": "None", "host": "redis"}), + &[("ssl", "REDIS_SSL"), ("ssl_cert_reqs", "REDIS_SSL_CERT_REQS")], + None, + Fails("invalid Redis setting REDIS_SSL_CERT_REQS: expected none, optional, required, null or an ssl.VerifyMode value", None), +)] +#[case::tls_options_from_the_environment( + json!({"ssl_ca_certs": "/env-ca.pem", "ssl": true, "ssl_cert_reqs": "optional", "host": "redis"}), + &[("ssl", "REDIS_SSL"), ("ssl_ca_certs", "REDIS_SSL_CA_CERTS"), ("ssl_cert_reqs", "REDIS_SSL_CERT_REQS")], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": "/env-ca.pem", "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}}))), +)] +#[case::connect_timeout_is_separate_from_the_socket_timeout( + json!({"host": "redis", "socket_connect_timeout": 0.25, "socket_timeout": 2.5}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "socket_timeout": 2.5, "socket_connect_timeout": 0.25}))), +)] +#[case::environment_references_resolve( + json!({"host": "referenced", "password": "referenced-secret"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "referenced", "port": 6379, "db": 0, "auth": {"kind": "static", "username": null, "password": "referenced-secret"}}))), +)] +#[case::no_target_fails( + json!({"port": 6379}), + &[], + None, + Fails("either host or url must be specified for redis", Some(ValueError)), +)] +#[case::client_certificates( + json!({"host": "redis", "ssl": true, "ssl_certfile": "/client.pem", "ssl_keyfile": "/client.key"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": "/client.pem", "keyfile": "/client.key", "verify_certificate": true, "check_hostname": false}}))), +)] +#[case::a_rediss_url_takes_tls_options_from_its_query( + json!({"url": "rediss://cache:6380/0?ssl_cert_reqs=none&ssl_ca_certs=/ca.pem&ssl_check_hostname=yes"}), + &[], + None, + Resolves(expected_view(json!({"mode": "url", "url": "rediss://cache:6380/0?ssl_cert_reqs=none&ssl_ca_certs=/ca.pem&ssl_check_hostname=yes", "tls": {"ca_certs": "/ca.pem", "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": false, "check_hostname": true}}))), +)] +#[case::a_url_query_carries_socket_timeout( + json!({"url": "redis://cache:6379/0?socket_timeout=2"}), + &[], + None, + Resolves(expected_view(json!({"mode": "url", "url": "redis://cache:6379/0?socket_timeout=2", "socket_timeout": 2.0, "socket_connect_timeout": 2.0}))), +)] +#[case::url_query_settings_win_over_the_kwargs( + json!({"health_check_interval": 4, "socket_connect_timeout": 7, "socket_timeout": 1, "url": "redis://cache:6379/0?socket_connect_timeout=0.5&health_check_interval=9&socket_timeout=3"}), + &[], + None, + Resolves(expected_view(json!({"mode": "url", "url": "redis://cache:6379/0?socket_connect_timeout=0.5&health_check_interval=9&socket_timeout=3", "socket_timeout": 3.0, "socket_connect_timeout": 0.5, "health_check_interval": 9.0}))), +)] +#[case::a_repeated_url_query_parameter_takes_its_first_value( + json!({"url": "redis://cache:6379/0?socket_timeout=2&socket_timeout=9"}), + &[], + None, + Resolves(expected_view(json!({"mode": "url", "url": "redis://cache:6379/0?socket_timeout=2&socket_timeout=9", "socket_timeout": 2.0, "socket_connect_timeout": 2.0}))), +)] +#[case::a_url_query_number_parse_url_cannot_read( + json!({"url": "redis://cache:6379/0?socket_timeout=slow"}), + &[], + None, + Fails("invalid Redis setting url ?socket_timeout: redis-py's parse_url cannot read it as a number", Some(ValueError)), +)] +#[case::tls_settings_outside_a_url_never_fail_it( + json!({"db": "99999999999999999999", "ssl_cert_reqs": "strict", "url": "rediss://cache:6380"}), + &[], + None, + Resolves(expected_view(json!({"mode": "url", "url": "rediss://cache:6380", "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}}))), +)] +#[case::db_is_not_read_for_a_cluster( + json!({"db": 99_999_999_999_i64, "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + None, + Fails("db must be 0 in cluster mode", Some(RedisClusterException)), +)] +#[case::cluster_db_true( + json!({"db": true, "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + None, + Fails("db must be 0 in cluster mode", Some(RedisClusterException)), +)] +#[case::cluster_db_false( + json!({"db": false, "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + None, + Resolves(expected_view(json!({"mode": "cluster", "nodes": ["node-a:7000"], "health_check_interval": 25.0}))), +)] +#[case::cluster_db_fraction( + json!({"db": 0.5, "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + None, + Fails("db must be 0 in cluster mode", Some(RedisClusterException)), +)] +#[case::cluster_db_float_zero( + json!({"db": 0.0, "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + None, + Resolves(expected_view(json!({"mode": "cluster", "nodes": ["node-a:7000"], "health_check_interval": 25.0}))), +)] +#[case::cluster_db_nonempty_list( + json!({"db": [1], "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + None, + Fails("db must be 0 in cluster mode", Some(RedisClusterException)), +)] +#[case::cluster_db_empty_list( + json!({"db": [], "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + None, + Resolves(expected_view(json!({"mode": "cluster", "nodes": ["node-a:7000"], "health_check_interval": 25.0}))), +)] +#[case::ssl_ciphers( + json!({"host": "redis", "ssl": true, "ssl_ciphers": "HIGH"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}}))), +)] +#[case::retry_on_timeout( + json!({"host": "redis", "retry_on_timeout": true}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0}))), +)] +#[case::malformed_port( + json!({"host": "redis", "port": "abc"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0}))), +)] +#[case::malformed_db( + json!({"db": "first", "host": "redis"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0}))), +)] +#[case::socket_timeout_not_a_number( + json!({"host": "redis", "socket_timeout": "slow"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0}))), +)] +#[case::malformed_port_from_the_environment( + json!({"port": "abc", "host": "redis"}), + &[("host", "REDIS_HOST"), ("port", "REDIS_PORT")], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0}))), +)] +#[case::protocol_3( + json!({"host": "redis", "protocol": 3}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0}))), +)] +#[case::retry_on_error_from_the_environment( + json!({"retry_on_error": true, "host": "redis"}), + &[("retry_on_error", "REDIS_RETRY_ON_ERROR")], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0}))), +)] +#[case::unix_socket_path( + json!({"host": "redis", "unix_socket_path": "/tmp/redis.sock"}), + &[], + None, + Resolves(expected_view(json!({"mode": "unix", "path": "/tmp/redis.sock"}))), +)] +#[case::unix_socket_path_keeps_tls_settings_out_of_the_target( + json!({"host": "redis", "ssl": true, "ssl_ca_certs": "/ca.pem", "unix_socket_path": "/tmp/redis.sock"}), + &[], + None, + Resolves(expected_view(json!({"mode": "unix", "path": "/tmp/redis.sock"}))), +)] +#[case::ssl_ca_data( + json!({"host": "redis", "ssl": true, "ssl_ca_data": "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----", "ca_path": null, "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}}))), +)] +#[case::ssl_ca_path_from_the_environment( + json!({"ssl": true, "ssl_ca_path": "/etc/ssl/private-cas", "host": "redis"}), + &[("ssl", "REDIS_SSL"), ("ssl_ca_path", "REDIS_SSL_CA_PATH")], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "tls": {"ca_certs": null, "ca_data": null, "ca_path": "/etc/ssl/private-cas", "certfile": null, "keyfile": null, "verify_certificate": true, "check_hostname": false}}))), +)] +#[case::health_check_interval_is_whole_seconds( + json!({"health_check_interval": "7", "host": "redis"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0, "health_check_interval": 7.0}))), +)] +#[case::health_check_interval_zero_is_off( + json!({"health_check_interval": 0, "host": "redis"}), + &[], + None, + Resolves(expected_view(json!({"mode": "host", "host": "redis", "port": 6379, "db": 0}))), +)] +#[case::health_check_interval_zero_keeps_a_cluster_off( + json!({"health_check_interval": 0, "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + None, + Resolves(expected_view(json!({"mode": "cluster", "nodes": ["node-a:7000"]}))), +)] +#[case::a_cluster_rejects_a_nonzero_db( + json!({"db": 3, "startup_nodes": [{"host": "node-a", "port": 7000}]}), + &[], + None, + Fails("db must be 0 in cluster mode", Some(RedisClusterException)), +)] +#[case::unix_socket_path_without_a_host_fails_like_python( + json!({"unix_socket_path": "/tmp/redis.sock"}), + &[], + None, + Fails("either host or url must be specified for redis", Some(ValueError)), +)] +fn resolves_like_python_redis( + #[case] merged: serde_json::Value, + #[case] sources: &[(&str, &str)], + #[case] redis_username: Option<&str>, + #[case] expected: Outcome, +) { + let outcome = interpret_merged(merged, sources, redis_username); + let rendered = outcome + .as_ref() + .map(view) + .map_err(|error| error.to_string()); + + match expected { + Resolves(expected) => assert_eq!(rendered, Ok(expected)), + Refuses(message) => assert_eq!(rendered, Err(message.to_owned())), + Fails(message, raised) => { + assert_eq!(rendered, Err(message.to_owned())); + assert_eq!( + outcome + .err() + .and_then(|error| error.raised_building_the_python_client()), + raised + ); + } + } +} diff --git a/litellm-rust/crates/redis-store/AGENTS.md b/litellm-rust/crates/redis-store/AGENTS.md new file mode 100644 index 00000000000..963f7ff168f --- /dev/null +++ b/litellm-rust/crates/redis-store/AGENTS.md @@ -0,0 +1,13 @@ +- One folder per Redis family (`router_cooldowns/`, `router_usage/`, `spend_counters/`, `auth_cache/`, `rate_limits/`), each with `mod.rs` (the store and its ops), `keys.rs` (key types and spellings) and `types.rs` (the family's values, with the `FromValue` / `IntoValue` codecs for what it stores in Redis). It is the only place that family's keys, value codec and scripts are written, and it takes domain ids, never keys. Shared pieces live once at the crate root: `Error` and `ErrorKind` (`error.rs`), `Memory` and `Clock` (`memory.rs`), `Pending` (`request.rs`), `Remote` and its attach-once `RemoteSlot`, which knows nothing of memory (`remote.rs`), `DualCache` (`dual_cache/`) and the Python codecs (`codec.rs`, `value.rs`) +- An op declared on a `Send` returns `Pending`, a lazy future built when the op is declared. An op that only means something on Redis (an `MGET` the caller falls back from, a script) returns `None` without one, and an op that replaced a `DualCache` call answers from memory. A post-call method returns `false` when there was no Redis to declare on, or `None` when it returns a settlement the caller waits on, like `SpendCounterStore::increment_many_post_call` +- A store that sends a pipeline of its own uses `RedisBatch::pipelined`, never `Pipeline` and raw `Value`s +- Python and Rust pods share these keys during a rolling deploy, so a module writes the bytes `main`'s Python writes and reads everything `main` reads. Python formats go through `litellm-python-compat` (`repr`, `literal_eval`, `json`), never a hand-rolled codec +- A family holds one `DualCache` per `DualCache` its Python held on `main`, and each `DualCache` method is the Python method it names, so a family op that was a `DualCache` call is one call here. Where `main`'s Python reached past `DualCache` to `in_memory_cache` or `redis_cache`, the family does the same through `memory()` and `redis()`, which keeps that reach visible. Caches a family's Python attached to one `RedisCache` share one `Arc`. Only a family that batch-reads builds its cache with `DualCache::batch_reading` from a `TierPolicy` the host passes in, and the Python host fills it from what `main` reads from the environment or `litellm` globals (`litellm/rust_bridge/host/settings.py`, `router_tiers`). Callers pass a `Remote` (or none), never a cache. Redis-only families are plain functions over `&RedisClient` +- Every family works with no Redis, answering from memory. `attach` adds the Redis tier at most once and keeps memory, as `DualCache.attach_redis_cache` does, so a host builds the store at startup and never rebuilds it to connect +- Where `main` swallows a Redis failure and serves memory, the family does the same and logs. It never turns a failure into a miss that `main` would not +- Key suffixes and wire spellings derive from strum on the family's enums. A value the family stores in Redis implements `FromValue` / `IntoValue` over `litellm_python_compat::Value`. An argument or reply is a typed record, never a positional tuple: a frozen Python dataclass read in `python-bridge` by a `#[derive(FromPyObject)]` struct with `#[pyo3(attribute)]` fields and converted into the family's type with `From`, and a reply returned as a frozen `#[pyclass]`. The bridge reads an enum through its strum spelling, never a hand-written match +- Wire contracts are hand-written cases with expected values, inlined as the same literals in this crate's test and in `tests/unit/redis/stores/`, checked here against real Redis and there against Python's own codec. Change both sides together +- `rate_limits.rs` keeps the v3 limiter's Lua byte for byte (`rate_limits/lua.rs`) so pods on either stack share one script SHA; the admission decisions stay in the Python limiter +- `RequestRedis` is one request's batches: a pre-call and a post-call `RedisBatch` per client, the post one flushed by `flush_post` or its deadline. Each op exists once and takes a `Send` (`Now`, `PreCall`, `PostCall`) picked by the caller, and the store calls `send.batch(&client)`. A store never encodes a phase in a method name. A post-call method exists only where a follow-up runs after the write settles (`token_increment_post_call`), as a thin wrapper over the shared op. `Send::Now` is an alone batch, so a script call outside a request goes out alone, like a redis-py `Script` outside a pipeline, so `NOSCRIPT` costs `SCRIPT LOAD` and one more `EVALSHA` the way it does on main. Only what main's Python puts on its request pipeline rides these batches. An op main sends on its own round trip, like a deployment's tpm and rpm increment, stays on its own round trip here +- Every store sends the commands main's `RedisCache` sends for that operation: `MGET` where Python reads several keys at once, `SET ... EX`, and `INCRBYFLOAT` with `EXPIRE` or `TTL`. A `MONITOR` diff against main is a parity check, and any difference is a defect +- Tests run against `litellm-redis-testing` on standalone and cluster. A test is single-topology only for outages diff --git a/litellm-rust/crates/redis-store/Cargo.toml b/litellm-rust/crates/redis-store/Cargo.toml new file mode 100644 index 00000000000..1dd61d76d3c --- /dev/null +++ b/litellm-rust/crates/redis-store/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "litellm-redis-store" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +futures-util.workspace = true +litellm-cache.workspace = true +litellm-cache-memory.workspace = true +litellm-python-compat.workspace = true +litellm-redis-client.workspace = true +strum.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } +tracing.workspace = true + +[dev-dependencies] +litellm-circuit-breaker.workspace = true +litellm-redis-testing.workspace = true +litellm-secret-value.workspace = true +litellm-tracing.workspace = true +rstest.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/redis-store/src/auth_cache/keys.rs b/litellm-rust/crates/redis-store/src/auth_cache/keys.rs new file mode 100644 index 00000000000..feb87a40b00 --- /dev/null +++ b/litellm-rust/crates/redis-store/src/auth_cache/keys.rs @@ -0,0 +1,8 @@ +/// `is_user_key_cache_key`: user-key objects are cached under a bare `hash_token` digest, every +/// other object under a prefixed key. +pub fn is_key_object(key: &str) -> bool { + key.len() == 64 + && key + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} diff --git a/litellm-rust/crates/redis-store/src/auth_cache/mod.rs b/litellm-rust/crates/redis-store/src/auth_cache/mod.rs new file mode 100644 index 00000000000..b5a83b5aec6 --- /dev/null +++ b/litellm-rust/crates/redis-store/src/auth_cache/mod.rs @@ -0,0 +1,258 @@ +mod keys; +mod types; + +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; + +use litellm_python_compat::{Value, json}; +use litellm_redis_client::{Declared, RedisBatch, commands}; + +pub use self::keys::is_key_object; +pub use self::types::Settings; +use crate::{ + Error, Number, Pending, RequestRedis, Send, + dual_cache::DualCache, + memory::{Clock, lock, system_clock}, + remote::{Remote, RemoteSlot}, + request::ready, +}; + +/// `RedisCache.async_increment_if_present`, byte for byte, sent as `EVAL` like it. +const INCREMENT_IF_PRESENT: &str = "if redis.call('EXISTS', KEYS[1]) == 0 then return false end return redis.call('INCRBYFLOAT', KEYS[1], ARGV[1])"; + +/// The proxy's auth object cache with `enable_redis_auth_cache` on: `UserApiKeyCache`'s two +/// `DualCache` partitions, so churn in management objects cannot evict user-key objects, in front of +/// one Redis shared by every pod. Keys are the logical cache keys `UserApiKeyCache` callers already +/// use; values are what `json.loads` returns, written as `json.dumps`, like `RedisCache.async_set_cache`. +pub struct AuthCache { + key_objects: DualCache, + objects: DualCache, + redis: Arc, + settings: Mutex, +} + +impl AuthCache { + pub fn new(remote: Option, settings: Settings) -> Self { + Self::with_clock(remote, settings, system_clock()) + } + + pub fn with_clock(remote: Option, settings: Settings, now: Clock) -> Self { + let redis = Arc::new(RemoteSlot::new(remote)); + let partition = || DualCache::new(redis.clone(), settings.memory(), now.clone()); + Self { + key_objects: partition(), + objects: partition(), + redis, + settings: Mutex::new(settings), + } + } + + pub fn has_redis(&self) -> bool { + self.redis.get().is_some() + } + + pub fn settings(&self) -> Settings { + *lock(&self.settings) + } + + /// `UserApiKeyCache.update_cache_ttl` and `update_in_memory_max_size` on both partitions, which + /// keep what they hold. + pub fn reconfigure(&self, settings: Settings) { + *lock(&self.settings) = settings; + self.key_objects.reconfigure(settings.memory()); + self.objects.reconfigure(settings.memory()); + } + + pub fn get_local(&self, key: &str) -> Result, Error> { + Ok(self.partition(key).memory().get(key)?) + } + + /// A memory write with `ttl`, or `default_in_memory_ttl` when it names none. + pub fn set_local(&self, key: &str, value: Value, ttl: Option) -> Result<(), Error> { + let partition = self.partition(key); + Ok(partition + .memory() + .set(key, value, ttl.or(partition.default_in_memory_ttl()))?) + } + + pub fn delete_local(&self, key: &str) -> Result<(), Error> { + Ok(self.partition(key).memory().delete(key)?) + } + + pub fn flush_local(&self) -> Result<(), Error> { + self.key_objects.memory().flush()?; + Ok(self.objects.memory().flush()?) + } + + /// Entries held by the management-object and the user-key partitions. + pub fn len_local(&self) -> Result<(usize, usize), Error> { + Ok(( + self.objects.memory().len()?, + self.key_objects.memory().len()?, + )) + } + + /// `DualCache.async_get_cache`. A key `request` already read as missing from Redis is not read + /// again. + pub async fn get( + &self, + request: Option<&RequestRedis>, + key: &str, + ) -> Result, Error> { + self.partition(key).get(request, key, None).await + } + + /// `DualCache.async_set_cache`, writing `json.dumps(value)`. + pub async fn set(&self, key: &str, value: Value, ttl: Option) -> Result<(), Error> { + self.partition(key).set(key, value, ttl).await + } + + /// `DualCache.set_cache`, whose `RedisCache.set_cache` writes `str(value)`. + pub async fn set_str( + &self, + key: &str, + value: Value, + ttl: Option, + ) -> Result<(), Error> { + self.partition(key).set_str(key, value, ttl).await + } + + /// `DualCache.async_set_cache_pre_call`: memory now, the `SET` on the batch `send` picks. + pub fn declare_set( + &self, + send: Send<'_>, + key: &str, + value: Value, + ttl: Option, + ) -> Result, Error> { + self.partition(key).declare_set(send, key, value, ttl) + } + + /// `DualCache.async_delete_cache_pre_call`, memory now and the `DEL` on the batch `send` picks. + pub fn declare_delete(&self, send: Send<'_>, key: &str) -> Result, Error> { + self.partition(key).declare_delete(send, key) + } + + /// `UserApiKeyCache.async_set_cache_pipeline`: the key-object entries in one pipeline, then + /// the rest in another, each with one TTL. + pub async fn set_many( + &self, + entries: Vec<(String, Value)>, + ttl: Option, + ) -> Result<(), Error> { + let (key_objects, others): (Vec<_>, Vec<_>) = + entries.into_iter().partition(|(key, _)| is_key_object(key)); + self.key_objects.set_pipeline(key_objects, ttl).await?; + self.objects.set_pipeline(others, ttl).await + } + + /// The auth prefetch write-back: memory first, an entry without a TTL taking + /// `default_in_memory_ttl`, then `RedisCache.async_set_cache_pipeline_with_ttls`, which encodes + /// every value before it sends anything and gives an entry without a TTL the connection's + /// default. On a request batch the `SET`s are declared and nothing is left to wait for. + /// [`Send::Now`] is one pipeline of its own rather than one round trip per entry. + pub fn write_back( + &self, + send: Send<'_>, + entries: Vec<(String, Value, Option)>, + ) -> Result, Error> { + let encoded = self + .redis + .get() + .map(|_| { + entries + .iter() + .map(|(_, value, _)| json::dumps(value)) + .collect::, _>>() + }) + .transpose(); + let writes = entries + .into_iter() + .map(|(key, value, ttl)| { + self.set_local(&key, value, ttl)?; + Ok((key, ttl)) + }) + .collect::, Error>>()?; + let (Some(remote), Some(encoded)) = (self.redis.get(), encoded?) else { + return Ok(ready(())); + }; + let sets = writes.iter().zip(encoded).map(|((key, ttl), value)| { + commands::set(key, value, ttl.unwrap_or(remote.default_ttl)) + }); + let Send::Now = send else { + let batch = send.batch(&remote.client); + sets.for_each(|set| drop(batch.add(set))); + return Ok(ready(())); + }; + let pipeline = RedisBatch::pipelined(remote.client.clone()); + let written: Vec<_> = sets.map(|set| pipeline.add(set)).collect(); + Ok(Box::pin(async move { + Declared::all(written) + .await + .into_iter() + .try_for_each(|reply| Ok(reply?)) + })) + } + + /// `UserApiKeyCache.async_delete_cache_keys`: the key-object partition and the rest as two + /// concurrent `DualCache.async_delete_cache_keys`. Both run even when one fails. + pub async fn delete(&self, keys: &[String]) -> Result<(), Error> { + let (key_objects, others): (Vec, Vec) = + keys.iter().cloned().partition(|key| is_key_object(key)); + let (key_objects, others) = tokio::join!( + self.key_objects.delete_keys(&key_objects), + self.objects.delete_keys(&others) + ); + key_objects.and(others) + } + + /// `RedisCache.async_batch_get_cache` for keys the caller found missing in memory, one `MGET` on + /// the batch `send` picks, noting the keys Redis lacks as the request's misses. `None` without + /// Redis. Any failed read or undecodable value fails the whole read, as a failed `MGET` does. + pub fn get_many(&self, send: Send<'_>, keys: &[String]) -> Option>>> { + let client = self.redis.client()?; + let misses = send.misses(client); + let read = send.batch(client).add(commands::mget::>( + &keys.iter().map(String::as_str).collect::>(), + )); + let keys = keys.to_vec(); + Some(Box::pin(async move { + let raw = read.resolve().await?; + if let Some(misses) = misses { + misses.note( + keys.into_iter() + .zip(&raw) + .filter(|(_, raw)| raw.is_none()) + .map(|(key, _)| key), + ); + } + raw.into_iter().map(|raw| Ok(types::decode(raw)?)).collect() + })) + } + + /// `RedisCache.async_increment_if_present`: `INCRBYFLOAT` only an entry Redis still holds. + /// `None` without Redis or for an absent entry. + pub async fn increment_if_present( + &self, + key: &str, + amount: Number, + ) -> Result, Error> { + let Some(client) = self.redis.client() else { + return Ok(None); + }; + let amount = litellm_redis_client::Amount::from(amount); + Ok(client + .send(commands::eval_source(INCREMENT_IF_PRESENT, &[key], amount)) + .await?) + } + + fn partition(&self, key: &str) -> &DualCache { + if is_key_object(key) { + &self.key_objects + } else { + &self.objects + } + } +} diff --git a/litellm-rust/crates/redis-store/src/auth_cache/types.rs b/litellm-rust/crates/redis-store/src/auth_cache/types.rs new file mode 100644 index 00000000000..e933ac19af9 --- /dev/null +++ b/litellm-rust/crates/redis-store/src/auth_cache/types.rs @@ -0,0 +1,35 @@ +use std::time::Duration; + +use litellm_python_compat::Value; + +use crate::{DecodeError, codec::python_value, dual_cache::MemorySettings}; + +/// `UserApiKeyCache`'s memory configuration: each partition's size and `default_in_memory_ttl`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Settings { + pub memory_entries: usize, + pub memory_ttl: Option, +} + +impl Default for Settings { + /// `InMemoryCache`'s 200 entries. + fn default() -> Self { + Self { + memory_entries: 200, + memory_ttl: None, + } + } +} + +impl Settings { + pub(super) fn memory(self) -> MemorySettings { + MemorySettings { + entries: Some(self.memory_entries), + ttl: self.memory_ttl, + } + } +} + +pub(super) fn decode(raw: Option>) -> Result, DecodeError> { + raw.as_deref().map(python_value).transpose() +} diff --git a/litellm-rust/crates/redis-store/src/codec.rs b/litellm-rust/crates/redis-store/src/codec.rs new file mode 100644 index 00000000000..21cde7d0b81 --- /dev/null +++ b/litellm-rust/crates/redis-store/src/codec.rs @@ -0,0 +1,74 @@ +use litellm_python_compat::{Value, json, literal, repr::to_str}; + +use crate::{Error, FromValue, IntoValue, Number}; + +#[derive(Debug, thiserror::Error)] +pub enum DecodeError { + #[error("Redis value is not UTF-8")] + NotUtf8, + #[error("Redis value is neither JSON nor a Python literal: {0}")] + Unparsed(#[source] litellm_python_compat::Error), + #[error("Redis value has the wrong shape: {0}")] + Shape(&'static str), +} + +/// `RedisCache._get_cache_logic`: `json.loads`, falling back to `ast.literal_eval`. +pub(crate) fn python_value(raw: &[u8]) -> Result { + let text = std::str::from_utf8(raw).map_err(|_| DecodeError::NotUtf8)?; + json::loads(text).or_else(|_| literal::literal_eval(text).map_err(DecodeError::Unparsed)) +} + +/// A cached value as `RedisCache._get_cache_logic` reads it. +pub(crate) fn decode(raw: &[u8]) -> Result { + V::from_value(&python_value(raw)?) +} + +/// `json.dumps(value)`, what `RedisCache.async_set_cache` writes. +pub(crate) fn dumps(value: V) -> Result { + Ok(json::dumps(&value.into_value())?) +} + +/// `str(value)`, what the sync `RedisCache.set_cache` writes. +pub(crate) fn str(value: V) -> String { + to_str(&value.into_value()) +} + +/// A number written by Python, `shape` naming what it was meant to be when it is not one. +pub(crate) fn number(raw: &[u8], shape: &'static str) -> Result { + Number::from_value(&python_value(raw)?).ok_or(DecodeError::Shape(shape)) +} + +#[cfg(test)] +mod tests { + use litellm_python_compat::{Value, json, repr::repr}; + use rstest::rstest; + + use super::{number, python_value}; + use crate::Number; + + // Expected values are the repr of main's RedisCache._get_cache_logic on these bytes + #[rstest] + #[case::nan(b"NaN", "nan")] + #[case::infinity(b"Infinity", "inf")] + #[case::negative_infinity(b"-Infinity", "-inf")] + #[case::nan_inside_a_document(br#"{"spend": NaN}"#, "{'spend': nan}")] + #[case::python_literal_fallback(b"{'a': (1, 2)}", "{'a': (1, 2)}")] + fn reads_what_python_reads(#[case] raw: &[u8], #[case] expected: &str) { + assert_eq!(repr(&python_value(raw).unwrap()), expected); + } + + #[rstest] + #[case::nan(f64::NAN)] + #[case::infinity(f64::INFINITY)] + #[case::negative_infinity(f64::NEG_INFINITY)] + fn a_non_finite_number_reads_back_as_written(#[case] value: f64) { + let written = json::dumps(&Value::Float(value)).unwrap(); + + let read = number(written.as_bytes(), "a number").unwrap(); + + assert_eq!( + repr(&read.to_value()), + repr(&Number::Float(value).to_value()) + ); + } +} diff --git a/litellm-rust/crates/redis-store/src/dual_cache/batch_read.rs b/litellm-rust/crates/redis-store/src/dual_cache/batch_read.rs new file mode 100644 index 00000000000..71dfda49233 --- /dev/null +++ b/litellm-rust/crates/redis-store/src/dual_cache/batch_read.rs @@ -0,0 +1,242 @@ +use std::{ + collections::{HashMap, VecDeque}, + sync::Mutex, + time::Duration, +}; + +use litellm_redis_client::{Declared, Error as RedisError, commands}; + +use super::{DualCache, MemorySettings}; +use crate::{Error, FromValue, Send, codec, memory::lock, request::Misses}; + +/// A batch-reading family's `DualCache` settings, as the host reads them from `litellm` globals. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TierPolicy { + /// Python's `redis_batch_cache_expiry`: a key memory misses is read from Redis at most this often. + pub refresh_after: Duration, + /// Python's `default_max_redis_batch_cache_size`: how many keys the refresh throttle tracks. + pub tracked_reads: usize, + /// `litellm.default_in_memory_ttl`, which a value read from Redis keeps in memory. + pub default_in_memory_ttl: Option, +} + +impl TierPolicy { + pub(crate) fn memory(&self) -> MemorySettings { + MemorySettings { + entries: None, + ttl: self.default_in_memory_ttl, + } + } +} + +/// `DualCache.last_redis_batch_access_time` with its expiry, which only `async_batch_get_cache` reads. +pub(crate) struct BatchReads { + refresh_after: Duration, + recent: Mutex, +} + +impl BatchReads { + pub(super) fn new(policy: TierPolicy) -> Self { + Self { + refresh_after: policy.refresh_after, + recent: Mutex::new(RecentReads::new(policy.tracked_reads)), + } + } +} + +impl DualCache { + /// `DualCache.async_batch_get_cache`: memory first, then one `MGET` for the misses whose refresh + /// is due, written back to memory. A failed Redis read serves memory, as `RedisCache` swallows + /// it, and only a breaker refusal releases the refresh it reserved. + pub(crate) async fn batch_get(&self, keys: &[String]) -> Result>, Error> { + let declared = self.declare_batch_get(Send::Now, keys)?; + self.resolve_batch_get(declared).await + } + + /// The first half of `batch_get`: reserves the due refreshes and declares their `MGET` on the + /// batch `send` picks, without sending anything. + pub(crate) fn declare_batch_get( + &self, + send: Send<'_>, + keys: &[String], + ) -> Result { + let fetch = match self.redis.client() { + None => None, + Some(client) => { + let reservation = self.reserve(keys, &self.read_memory(keys)?); + (!reservation.keys.is_empty()).then(|| { + let keys: Vec<&str> = reservation.keys.iter().map(String::as_str).collect(); + let replies = send.batch(client).add(commands::mget::>(&keys)); + Fetch { + reservation, + replies, + misses: send.misses(client), + } + }) + } + }; + Ok(DeclaredRead { + keys: keys.to_vec(), + fetch, + }) + } + + /// The second half of `batch_get`. Memory is read again here, so a value written to memory + /// since the declaration wins over what Redis answered. The keys Redis lacks are noted as the + /// request's misses. An undecodable value serves memory for the whole read, where `main`'s + /// `RedisCache` returns the raw bytes and `DualCache` backfills memory with them. + pub(crate) async fn resolve_batch_get( + &self, + declared: DeclaredRead, + ) -> Result>, Error> { + let DeclaredRead { keys, fetch } = declared; + let Some(Fetch { + reservation, + replies, + misses, + }) = fetch + else { + return self.read_memory(&keys); + }; + let fetched = replies.resolve().await; + let local = self.read_memory(&keys)?; + let fetched = match fetched { + Ok(fetched) => fetched, + Err(RedisError::BreakerOpen) => { + self.release(reservation); + return Ok(local); + } + Err(error) => { + crate::failure_log::redis_failure!( + &error, + "Redis read failed, serving the memory tier" + ); + return Ok(local); + } + }; + if let Some(misses) = misses { + misses.note( + reservation + .keys + .iter() + .zip(&fetched) + .filter(|(_, raw)| raw.is_none()) + .map(|(key, _)| key.clone()), + ); + } + let decoded = match fetched + .iter() + .map(|raw| raw.as_deref().map(codec::decode).transpose()) + .collect::, _>>() + { + Ok(decoded) => decoded, + Err(error) => { + tracing::warn!(%error, "a Redis value did not decode, serving the memory tier"); + return Ok(local); + } + }; + let read = reservation + .keys + .into_iter() + .zip(decoded) + .collect::>(); + keys.iter() + .zip(local) + .map(|(key, local)| match local { + Some(value) => Ok(Some(value)), + None => self.backfill(key, read.get(key).cloned().flatten(), None), + }) + .collect() + } + + fn read_memory(&self, keys: &[String]) -> Result>, Error> { + keys.iter().map(|key| Ok(self.memory.get(key)?)).collect() + } + + fn reserve(&self, keys: &[String], local: &[Option]) -> Reservation { + let now = self.now(); + let refresh_after = self.reads.refresh_after; + let mut recent = lock(&self.reads.recent); + let due = keys + .iter() + .zip(local) + .filter(|(_, value)| value.is_none()) + .filter_map(|(key, _)| { + let previous = recent.get(key); + let due = previous.is_none_or(|read| now.saturating_sub(read) >= refresh_after); + due.then(|| { + recent.insert(key.clone(), now); + (key.clone(), previous) + }) + }) + .collect::>(); + let (keys, previous) = due.into_iter().unzip(); + Reservation { keys, previous } + } + + fn release(&self, reservation: Reservation) { + let mut recent = lock(&self.reads.recent); + for (key, previous) in reservation.keys.into_iter().zip(reservation.previous) { + match previous { + Some(read) => recent.insert(key, read), + None => recent.remove(&key), + } + } + } +} + +/// The `MGET` `declare_batch_get` put on a batch, with the refreshes it reserved. +pub(crate) struct DeclaredRead { + keys: Vec, + fetch: Option, +} + +struct Fetch { + reservation: Reservation, + replies: Declared>>>, + misses: Option, +} + +struct Reservation { + keys: Vec, + previous: Vec>, +} + +/// Python's `LimitedSizeOrderedDict`: insertion-ordered, and any write at capacity drops the +/// oldest entry first, even when the written key is already present. +struct RecentReads { + capacity: usize, + order: VecDeque, + times: HashMap, +} + +impl RecentReads { + fn new(capacity: usize) -> Self { + Self { + capacity, + order: VecDeque::new(), + times: HashMap::new(), + } + } + + fn get(&self, key: &str) -> Option { + self.times.get(key).copied() + } + + fn insert(&mut self, key: String, time: Duration) { + if self.times.len() >= self.capacity + && let Some(oldest) = self.order.pop_front() + { + self.times.remove(&oldest); + } + if self.times.insert(key.clone(), time).is_none() { + self.order.push_back(key); + } + } + + fn remove(&mut self, key: &str) { + if self.times.remove(key).is_some() { + self.order.retain(|tracked| tracked != key); + } + } +} diff --git a/litellm-rust/crates/redis-store/src/dual_cache/increment.rs b/litellm-rust/crates/redis-store/src/dual_cache/increment.rs new file mode 100644 index 00000000000..705bc71b2c1 --- /dev/null +++ b/litellm-rust/crates/redis-store/src/dual_cache/increment.rs @@ -0,0 +1,129 @@ +use std::time::Duration; + +use futures_util::future::join_all; +use litellm_redis_client::{Declared, Expiry, RedisBatch, commands}; + +use super::DualCache; +use crate::{Error, Number, RequestRedis, Send, failure_log::redis_failure, memory::Counter}; + +const INCREMENT_FAILED: &str = + "Redis async_increment_cache failed, falling back to in-memory result"; +const PIPELINE_FAILED: &str = + "Redis async_increment_cache_pipeline failed, falling back to in-memory result"; + +/// A `RedisPipelineIncrementOperation`. +pub(crate) struct PipelineIncrement { + pub(crate) key: String, + pub(crate) amount: Number, + pub(crate) ttl: Option, +} + +impl DualCache { + /// `DualCache.async_increment_cache`: memory, then `INCRBYFLOAT` keeping an existing TTL, else + /// setting `ttl` or Redis's default. `None` when memory refuses the increment, which then never + /// reaches Redis, and the memory count when Redis fails. Both only log, as there. + pub(crate) async fn increment( + &self, + key: &str, + amount: Number, + ttl: Option, + ) -> Option { + let local = self + .memory + .increment(key, amount, ttl) + .map_err(|error| redis_failure!(&Error::from(error), INCREMENT_FAILED)) + .ok()?; + let Some(remote) = self.redis.get() else { + return Some(local); + }; + let expiry = Expiry::KeepExisting(ttl.unwrap_or(remote.default_ttl)); + let increment = commands::incr_by_float(key, amount.into(), Some(expiry)); + match remote.client.send(increment).await { + Ok(total) => Some(Number::Float(total)), + Err(error) => { + redis_failure!(&error, INCREMENT_FAILED); + Some(local) + } + } + } + + /// `DualCache.async_increment_cache_pipeline`: memory, then one pipeline of `INCRBYFLOAT` with + /// `EXPIRE` per increment. `None` when memory refuses an increment, which leaves the ones before + /// it applied and sends nothing, and the memory totals when Redis fails. + pub(crate) async fn increment_pipeline( + &self, + increments: &[PipelineIncrement], + ) -> Option> { + let local = increments + .iter() + .map(|increment| { + self.memory + .increment(&increment.key, increment.amount, increment.ttl) + }) + .collect::, _>>() + .map_err(|error| redis_failure!(&Error::from(error), PIPELINE_FAILED)) + .ok()?; + let Some(client) = self.redis.client() else { + return Some(local); + }; + let pipeline = RedisBatch::pipelined(client.clone()); + let totals = increments + .iter() + .map(|increment| { + pipeline.add(commands::incr_by_float( + &increment.key, + increment.amount.into(), + increment.ttl.map(Expiry::Refresh), + )) + }) + .collect::>(); + match Declared::all(totals) + .await + .into_iter() + .collect::, _>>() + { + Ok(totals) => Some(totals.into_iter().map(Number::Float).collect()), + Err(error) => { + redis_failure!(&error, PIPELINE_FAILED); + Some(local) + } + } + } + + /// `DualCache.async_increment_cache_pipeline_post_call`: memory now, stopping at the first + /// increment memory refuses and only logging it, then each increment on the request's post-call + /// batch, whose failure is logged once it settles. + pub(crate) fn increment_pipeline_post_call( + &self, + request: &RequestRedis, + increments: &[PipelineIncrement], + ) { + if let Err(error) = increments.iter().try_for_each(|increment| { + self.memory + .increment(&increment.key, increment.amount, increment.ttl) + .map(drop) + }) { + tracing::warn!(%error, "in-memory increment failed"); + } + let Some(batch) = self.redis.batch(Send::PostCall(request)) else { + return; + }; + let declared = increments + .iter() + .map(|increment| { + batch.add(commands::incr_by_float( + &increment.key, + increment.amount.into(), + increment.ttl.map(Expiry::KeepExisting), + )) + }) + .collect::>(); + request.spawn(async move { + join_all(declared.into_iter().map(Declared::settled)) + .await + .into_iter() + .filter_map(Result::err) + .for_each(|error| redis_failure!(&error, "deferred Redis increment failed")); + }); + } +} diff --git a/litellm-rust/crates/redis-store/src/dual_cache/mod.rs b/litellm-rust/crates/redis-store/src/dual_cache/mod.rs new file mode 100644 index 00000000000..b2c277a118e --- /dev/null +++ b/litellm-rust/crates/redis-store/src/dual_cache/mod.rs @@ -0,0 +1,279 @@ +mod batch_read; +mod increment; +mod read; +mod write; + +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; + +pub use self::batch_read::TierPolicy; +pub(crate) use self::{batch_read::BatchReads, increment::PipelineIncrement}; +use crate::{ + memory::{Clock, Memory, lock}, + remote::RemoteSlot, +}; + +/// The memory tier's `InMemoryCache` size and the `default_in_memory_ttl` a write or a Redis +/// backfill takes when it names no TTL. `None` keeps `InMemoryCache`'s own default. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct MemorySettings { + pub(crate) entries: Option, + pub(crate) ttl: Option, +} + +/// Python's `DualCache`: a per-worker `InMemoryCache` in front of the Redis every pod shares. Each +/// method is the `DualCache` method it names. A family whose Python reached past `DualCache` to its +/// `in_memory_cache` or `redis_cache` uses `memory` and `redis` the same way. +pub(crate) struct DualCache { + memory: Memory, + redis: Arc, + default_in_memory_ttl: Mutex>, + now: Clock, + reads: R, +} + +impl DualCache { + /// `redis` may be shared with the family's other caches, as one `RedisCache` is attached to each. + pub(crate) fn new(redis: Arc, memory: MemorySettings, now: Clock) -> Self { + Self { + memory: Memory::new(memory.entries, &now), + redis, + default_in_memory_ttl: Mutex::new(memory.ttl), + now, + reads: (), + } + } + + /// A cache that batch-reads, with the `async_batch_get_cache` throttle `policy` configures. + pub(crate) fn batch_reading( + redis: Arc, + policy: TierPolicy, + now: Clock, + ) -> DualCache { + let Self { + memory, + redis, + default_in_memory_ttl, + now, + reads: (), + } = Self::new(redis, policy.memory(), now); + DualCache { + memory, + redis, + default_in_memory_ttl, + now, + reads: BatchReads::new(policy), + } + } +} + +impl DualCache { + pub(crate) fn memory(&self) -> &Memory { + &self.memory + } + + pub(crate) fn redis(&self) -> &RemoteSlot { + &self.redis + } + + pub(crate) fn now(&self) -> Duration { + (self.now)() + } + + pub(crate) fn default_in_memory_ttl(&self) -> Option { + *lock(&self.default_in_memory_ttl) + } + + /// `DualCache.update_cache_ttl` and `update_in_memory_max_size` together: both apply in place, + /// and the entries memory already holds stay. + pub(crate) fn reconfigure(&self, memory: MemorySettings) { + *lock(&self.default_in_memory_ttl) = memory.ttl; + self.memory.update_max_size(memory.entries); + } +} + +#[cfg(test)] +mod tests { + use std::{ + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, + }; + + use rstest::rstest; + use tokio::runtime::Handle; + + use super::{DualCache, MemorySettings, PipelineIncrement}; + use crate::{Clock, Number, RequestRedis, memory::Counter, remote::RemoteSlot}; + + #[derive(Clone, Debug, PartialEq)] + enum Entry { + Number(Number), + Text, + } + + impl From for Entry { + fn from(number: Number) -> Self { + Self::Number(number) + } + } + + impl Counter for Entry { + fn as_number(&self) -> Option { + match self { + Self::Number(number) => Some(*number), + Self::Text => None, + } + } + } + + fn clock() -> (Clock, Arc) { + let seconds = Arc::new(AtomicU64::new(1_000)); + let read = seconds.clone(); + let clock: Clock = Arc::new(move || Duration::from_secs(read.load(Ordering::SeqCst))); + (clock, seconds) + } + + fn without_redis( + default_in_memory_ttl: Option, + now: Clock, + ) -> DualCache { + DualCache::new( + Arc::new(RemoteSlot::new(None)), + MemorySettings { + entries: None, + ttl: default_in_memory_ttl, + }, + now, + ) + } + + fn increment(key: &str, amount: Number) -> PipelineIncrement { + PipelineIncrement { + key: key.to_owned(), + amount, + ttl: None, + } + } + + // InMemoryCache's own default TTL is 600 s, which an entry outlives here only when the cache + // configured none + #[rstest] + #[case::default_in_memory_ttl(Some(Duration::from_secs(5)), None)] + #[case::in_memory_cache_default(None, Some(Number::Int(1)))] + #[tokio::test] + async fn a_set_without_a_ttl_takes_default_in_memory_ttl( + #[case] default_in_memory_ttl: Option, + #[case] after_six_seconds: Option, + ) { + let (now, seconds) = clock(); + let cache = without_redis(default_in_memory_ttl, now); + + cache.set("k", Number::Int(1), None).await.unwrap(); + seconds.fetch_add(6, Ordering::SeqCst); + + assert_eq!(cache.memory().get("k").unwrap(), after_six_seconds); + } + + #[rstest] + #[case::absent_starts_at_int_zero(None, Number::Int(2), Number::Int(2))] + #[case::int_plus_int_stays_int(Some(Number::Int(1)), Number::Int(2), Number::Int(3))] + #[case::a_float_makes_a_float(Some(Number::Int(1)), Number::Float(0.5), Number::Float(1.5))] + fn a_local_increment_adds_as_python_adds( + #[case] stored: Option, + #[case] amount: Number, + #[case] expected: Number, + ) { + let (now, _) = clock(); + let cache = without_redis::(None, now); + if let Some(stored) = stored { + cache.memory().set("k", stored, None).unwrap(); + } + + let total = cache.memory().increment("k", amount, None).unwrap(); + + assert_eq!(total, expected); + assert_eq!(cache.memory().get("k").unwrap(), Some(expected)); + } + + #[test] + fn a_local_increment_of_a_non_number_fails_and_keeps_the_entry() { + let (now, _) = clock(); + let cache = without_redis::(None, now); + cache.memory().set("k", Entry::Text, None).unwrap(); + + let total = cache.memory().increment("k", Number::Int(1), None); + + assert!(total.is_err()); + assert_eq!(cache.memory().get("k").unwrap(), Some(Entry::Text)); + } + + // async_increment_cache and async_increment_cache_pipeline catch the in-memory failure with the + // Redis one, so the caller gets None and nothing is sent + #[tokio::test] + async fn an_increment_memory_refuses_answers_none() { + let (now, _) = clock(); + let cache = without_redis::(None, now); + cache.memory().set("text", Entry::Text, None).unwrap(); + + let single = cache.increment("text", Number::Int(1), None).await; + let pipelined = cache + .increment_pipeline(&[ + increment("before", Number::Int(1)), + increment("text", Number::Int(1)), + ]) + .await; + + assert_eq!(single, None); + assert_eq!(pipelined, None); + assert_eq!(cache.memory().get("text").unwrap(), Some(Entry::Text)); + } + + // update_cache_ttl and update_in_memory_max_size change the settings in place, so what memory + // already holds survives and the next write takes the new TTL + #[tokio::test] + async fn reconfigure_keeps_entries_and_applies_the_new_ttl() { + let (now, seconds) = clock(); + let cache = without_redis(None, now); + cache.set("kept", Number::Int(1), None).await.unwrap(); + + cache.reconfigure(MemorySettings { + entries: Some(10), + ttl: Some(Duration::from_secs(5)), + }); + cache.set("short", Number::Int(2), None).await.unwrap(); + seconds.fetch_add(6, Ordering::SeqCst); + + assert_eq!(cache.memory().get("kept").unwrap(), Some(Number::Int(1))); + assert_eq!(cache.memory().get("short").unwrap(), None); + } + + // async_increment_cache_pipeline_post_call: InMemoryCache.async_increment_pipeline raises at the + // first entry it cannot add to, so later increments never reach memory, and DualCache only logs + #[tokio::test] + async fn a_post_call_increment_stops_at_the_first_entry_memory_refuses() { + let (now, _) = clock(); + let cache = without_redis::(None, now); + let request = RequestRedis::new(Handle::current(), Duration::from_secs(60)); + cache.memory().set("text", Entry::Text, None).unwrap(); + + cache.increment_pipeline_post_call( + &request, + &[ + increment("before", Number::Int(1)), + increment("text", Number::Int(1)), + increment("after", Number::Int(1)), + ], + ); + + assert_eq!( + cache.memory().get("before").unwrap(), + Some(Entry::Number(Number::Int(1))) + ); + assert_eq!(cache.memory().get("after").unwrap(), None); + } +} diff --git a/litellm-rust/crates/redis-store/src/dual_cache/read.rs b/litellm-rust/crates/redis-store/src/dual_cache/read.rs new file mode 100644 index 00000000000..14156483c6f --- /dev/null +++ b/litellm-rust/crates/redis-store/src/dual_cache/read.rs @@ -0,0 +1,55 @@ +use std::time::Duration; + +use litellm_redis_client::commands; + +use super::DualCache; +use crate::{Error, FromValue, RequestRedis, codec, failure_log::redis_failure}; + +impl DualCache { + /// `DualCache.async_get_cache`: memory, else `None` for a key `request` already read as missing + /// from Redis, else a `GET` written back to memory for `ttl` or `default_in_memory_ttl`. A failed + /// or undecodable Redis read is a miss, as there. + pub(crate) async fn get( + &self, + request: Option<&RequestRedis>, + key: &str, + ttl: Option, + ) -> Result, Error> { + if let Some(value) = self.memory.get(key)? { + return Ok(Some(value)); + } + let Some(client) = self.redis.client() else { + return Ok(None); + }; + if request.is_some_and(|request| request.misses(client).contains(key)) { + return Ok(None); + } + let value = client + .send(commands::get::>(key)) + .await + .map_err(Error::from) + .and_then(|raw| Ok(raw.as_deref().map(codec::decode).transpose()?)) + .unwrap_or_else(|error| { + redis_failure!(&error, "LiteLLM Cache: exception in async_get_cache"); + None + }); + self.backfill(key, value, ttl) + } +} + +impl DualCache { + /// `_backfill_kwargs`: a value read from Redis stays in memory for `ttl` or + /// `default_in_memory_ttl`. + pub(super) fn backfill( + &self, + key: &str, + value: Option, + ttl: Option, + ) -> Result, Error> { + if let Some(value) = &value { + self.memory + .set(key, value.clone(), ttl.or(self.default_in_memory_ttl()))?; + } + Ok(value) + } +} diff --git a/litellm-rust/crates/redis-store/src/dual_cache/write.rs b/litellm-rust/crates/redis-store/src/dual_cache/write.rs new file mode 100644 index 00000000000..6fcc6f32741 --- /dev/null +++ b/litellm-rust/crates/redis-store/src/dual_cache/write.rs @@ -0,0 +1,178 @@ +use std::time::Duration; + +use litellm_redis_client::{Declared, RedisBatch, commands}; + +use super::DualCache; +use crate::{ + Error, IntoValue, Pending, Send, codec, + failure_log::redis_failure, + request::{pending, ready}, +}; + +/// `DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE`, the chunk `async_delete_cache_keys` deletes at once. +const DELETE_CHUNK: usize = 1000; + +impl DualCache { + /// `DualCache.async_set_cache`: memory, then a `SET` of `json.dumps(value)`. A Redis failure, or + /// a value `json.dumps` refuses, only logs. + pub(crate) async fn set( + &self, + key: &str, + value: V, + ttl: Option, + ) -> Result<(), Error> { + self.set_now(key, value, ttl, codec::dumps).await + } + + /// `DualCache.set_cache`, whose sync `RedisCache.set_cache` writes `str(value)`. + pub(crate) async fn set_str( + &self, + key: &str, + value: V, + ttl: Option, + ) -> Result<(), Error> { + self.set_now(key, value, ttl, |value| Ok(codec::str(value))) + .await + } + + /// `DualCache._set_on_batch`: memory now, the `SET` on the batch `send` picks. Memory and Redis + /// both take `ttl`, else `default_in_memory_ttl`, and Redis falls back to its own default. + pub(crate) fn declare_set( + &self, + send: Send<'_>, + key: &str, + value: V, + ttl: Option, + ) -> Result, Error> { + self.declare_encoded(send, key, value, ttl, codec::dumps) + } + + /// `DualCache.async_set_cache_pipeline`: memory, then one pipeline of `SET`s. A value + /// `json.dumps` refuses aborts the whole pipeline, and any failure only logs, as `RedisCache` + /// does. + pub(crate) async fn set_pipeline( + &self, + entries: Vec<(String, V)>, + ttl: Option, + ) -> Result<(), Error> { + let ttl = ttl.or(self.default_in_memory_ttl()); + let encoded = self.redis.get().map(|_| { + entries + .iter() + .map(|(_, value)| codec::dumps(value.clone())) + .collect::, Error>>() + }); + let keys = entries + .into_iter() + .map(|(key, value)| { + self.memory.set(&key, value, ttl)?; + Ok(key) + }) + .collect::, Error>>()?; + let (Some(remote), Some(encoded)) = (self.redis.get(), encoded) else { + return Ok(()); + }; + let failure = match encoded { + Err(error) => Some(error), + Ok(values) => { + let pipeline = RedisBatch::pipelined(remote.client.clone()); + let ttl = ttl.unwrap_or(remote.default_ttl); + let written = keys + .iter() + .zip(values) + .map(|(key, value)| pipeline.add(commands::set(key, value, ttl))) + .collect::>(); + Declared::all(written) + .await + .into_iter() + .find_map(Result::err) + .map(Error::from) + } + }; + if let Some(error) = failure { + redis_failure!( + &error, + "LiteLLM Redis Caching: async set_cache_pipeline() - Got exception from REDIS", + ); + } + Ok(()) + } + + async fn set_now( + &self, + key: &str, + value: V, + ttl: Option, + encode: impl FnOnce(V) -> Result, + ) -> Result<(), Error> { + if let Err(error) = self + .declare_encoded(Send::Now, key, value, ttl, encode)? + .await + { + redis_failure!(&error, "LiteLLM Cache: exception in async add_cache"); + } + Ok(()) + } + + fn declare_encoded( + &self, + send: Send<'_>, + key: &str, + value: V, + ttl: Option, + encode: impl FnOnce(V) -> Result, + ) -> Result, Error> { + let ttl = ttl.or(self.default_in_memory_ttl()); + let encoded = self.redis.get().map(|_| encode(value.clone())); + self.memory.set(key, value, ttl)?; + let (Some(remote), Some(encoded)) = (self.redis.get(), encoded) else { + return Ok(ready(())); + }; + let encoded = match encoded { + Ok(encoded) => encoded, + Err(error) => { + redis_failure!(&error, "a cache value did not encode, kept in memory only"); + return Ok(ready(())); + } + }; + if let Some(misses) = send.misses(&remote.client) { + misses.forget(key); + } + Ok(pending(send.batch(&remote.client).add(commands::set( + key, + encoded, + ttl.unwrap_or(remote.default_ttl), + )))) + } +} + +impl DualCache { + /// `DualCache.async_delete_cache_pre_call`: memory now, the `DEL` on the batch `send` picks, + /// after which the request reads the key as missing. + pub(crate) fn declare_delete(&self, send: Send<'_>, key: &str) -> Result, Error> { + self.memory.delete(key)?; + let Some(client) = self.redis.client() else { + return Ok(ready(())); + }; + if let Some(misses) = send.misses(client) { + misses.note([key.to_owned()]); + } + let deleted = send.batch(client).add(commands::del(&[key])); + Ok(Box::pin( + async move { Ok(deleted.resolve().await.map(drop)?) }, + )) + } + + /// `DualCache.async_delete_cache_keys`: memory, then one multi-key `DEL` per chunk, in order. + pub(crate) async fn delete_keys(&self, keys: &[String]) -> Result<(), Error> { + keys.iter().try_for_each(|key| self.memory.delete(key))?; + let Some(client) = self.redis.client() else { + return Ok(()); + }; + for chunk in keys.chunks(DELETE_CHUNK) { + let chunk: Vec<&str> = chunk.iter().map(String::as_str).collect(); + client.send(commands::del(&chunk)).await?; + } + Ok(()) + } +} diff --git a/litellm-rust/crates/redis-store/src/error.rs b/litellm-rust/crates/redis-store/src/error.rs new file mode 100644 index 00000000000..fdc722855bb --- /dev/null +++ b/litellm-rust/crates/redis-store/src/error.rs @@ -0,0 +1,63 @@ +use crate::DecodeError; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error(transparent)] + Redis(#[from] litellm_redis_client::Error), + #[error(transparent)] + Memory(#[from] litellm_cache::Error), + #[error(transparent)] + Decode(#[from] DecodeError), + #[error("the value is not JSON serializable: {0}")] + Encode(#[from] litellm_python_compat::Error), +} + +impl Error { + pub fn kind(&self) -> ErrorKind { + match self { + Self::Redis(error) => error.into(), + Self::Memory(_) => ErrorKind::Memory, + Self::Decode(_) => ErrorKind::Decode, + Self::Encode(_) => ErrorKind::Encode, + } + } +} + +/// The failure class a host reports with a store error, `timeout` and `breaker_open` being the ones a +/// caller may drop. +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr)] +pub enum ErrorKind { + #[strum(serialize = "breaker_open")] + BreakerOpen, + #[strum(serialize = "timeout")] + Timeout, + #[strum(serialize = "transport")] + Transport, + #[strum(serialize = "rejected")] + Rejected, + #[strum(serialize = "decode")] + Decode, + #[strum(serialize = "encode")] + Encode, + #[strum(serialize = "memory")] + Memory, +} + +impl From<&Error> for ErrorKind { + fn from(error: &Error) -> Self { + error.kind() + } +} + +impl From<&litellm_redis_client::Error> for ErrorKind { + fn from(error: &litellm_redis_client::Error) -> Self { + use litellm_redis_client::Error; + match error { + Error::BreakerOpen => Self::BreakerOpen, + Error::Transport(error) if error.is_timeout() => Self::Timeout, + Error::Transport(_) => Self::Transport, + Error::Server(_) => Self::Rejected, + Error::Decode(_) => Self::Decode, + } + } +} diff --git a/litellm-rust/crates/redis-store/src/failure_log.rs b/litellm-rust/crates/redis-store/src/failure_log.rs new file mode 100644 index 00000000000..e6bf5187cfd --- /dev/null +++ b/litellm-rust/crates/redis-store/src/failure_log.rs @@ -0,0 +1,18 @@ +/// `log_redis_failure`: an open breaker is DEBUG, anything else WARNING. The event's `redis.failure` +/// field names the class, so the host can rate-limit timeout lines with the throttle redis-py shares. +/// A macro so the event carries its caller's target, file and line, as `stacklevel=2` does in Python. +macro_rules! redis_failure { + ($error:expr, $message:expr $(,)?) => {{ + let error = $error; + let kind = $crate::ErrorKind::from(error); + let failure: &'static str = kind.into(); + match kind { + $crate::ErrorKind::BreakerOpen => { + tracing::debug!(redis.failure = failure, "{}: {}", $message, error) + } + _ => tracing::warn!(redis.failure = failure, "{}: {}", $message, error), + } + }}; +} + +pub(crate) use redis_failure; diff --git a/litellm-rust/crates/redis-store/src/lib.rs b/litellm-rust/crates/redis-store/src/lib.rs new file mode 100644 index 00000000000..69600aac2a6 --- /dev/null +++ b/litellm-rust/crates/redis-store/src/lib.rs @@ -0,0 +1,24 @@ +pub mod auth_cache; +mod codec; +mod dual_cache; +mod error; +mod failure_log; +mod memory; +mod number; +pub mod rate_limits; +mod remote; +mod request; +pub mod router_cooldowns; +pub mod router_usage; +pub mod sensitive_route_pins; +pub mod spend_counters; +mod value; + +pub use codec::DecodeError; +pub use dual_cache::TierPolicy; +pub use error::{Error, ErrorKind}; +pub use memory::{Clock, system_clock}; +pub use number::Number; +pub use remote::Remote; +pub use request::{Pending, RequestRedis, Send}; +pub use value::{FromValue, IntoValue}; diff --git a/litellm-rust/crates/redis-store/src/memory.rs b/litellm-rust/crates/redis-store/src/memory.rs new file mode 100644 index 00000000000..f89d5926760 --- /dev/null +++ b/litellm-rust/crates/redis-store/src/memory.rs @@ -0,0 +1,122 @@ +use std::{ + sync::{Arc, Mutex, MutexGuard, PoisonError}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use litellm_cache::Error; +use litellm_cache_memory::InMemoryCache; + +use crate::Number; + +pub type Clock = Arc Duration + Send + Sync>; + +pub fn system_clock() -> Clock { + Arc::new(|| { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + }) +} + +/// A store's memory tier: Python's `InMemoryCache`, with its 200 entries and 600 s TTL unless the +/// store names others, plus one lock for every read-modify-write on it. +pub struct Memory { + cache: InMemoryCache, + updates: Mutex<()>, +} + +impl Memory { + pub fn new(entries: Option, now: &Clock) -> Self { + let now = now.clone(); + Self { + cache: InMemoryCache::with_clock(entries, None, move || now()), + updates: Mutex::new(()), + } + } + + pub fn get(&self, key: &str) -> Result, Error> { + self.cache.get_cache(key) + } + + pub fn set(&self, key: &str, value: V, ttl: Option) -> Result<(), Error> { + self.cache.set_cache(key, value, ttl).map(drop) + } + + pub fn delete(&self, key: &str) -> Result<(), Error> { + self.cache.delete_cache(key) + } + + pub fn flush(&self) -> Result<(), Error> { + self.cache.flush_cache() + } + + pub fn len(&self) -> Result { + self.cache.len() + } + + pub fn expires_at(&self, key: &str) -> Result, Error> { + self.cache.expires_at(key) + } + + pub fn entries(&self) -> Result, Error> { + self.cache.entries() + } + + /// `DualCache.update_in_memory_max_size`: `None` restores the 200-entry default. + pub fn update_max_size(&self, entries: Option) { + self.cache.update_max_size_in_memory(entries); + } + + /// Read-modify-write of one entry, atomic against every other update of this tier. + pub fn update( + &self, + key: &str, + ttl: Option, + update: impl FnOnce(Option) -> Result, + ) -> Result { + let _guard = self.guard(); + let value = update(self.get(key)?)?; + self.set(key, value.clone(), ttl)?; + Ok(value) + } + + /// Held across a read and a conditional write that `update` cannot express. + pub fn guard(&self) -> MutexGuard<'_, ()> { + lock(&self.updates) + } +} + +/// A value `InMemoryCache.increment_cache` can add to. +pub(crate) trait Counter: Clone + From { + fn as_number(&self) -> Option; +} + +impl Counter for Number { + fn as_number(&self) -> Option { + Some(*self) + } +} + +impl Memory { + /// `InMemoryCache.increment_cache`: `(current or 0) + amount`, refusing an entry that is not a + /// number and leaving it in place. + pub(crate) fn increment( + &self, + key: &str, + amount: Number, + ttl: Option, + ) -> Result { + let _guard = self.guard(); + let current = match self.get(key)? { + None => Number::Int(0), + Some(value) => value.as_number().ok_or(Error::InvalidEntry(None))?, + }; + let total = current + amount; + self.set(key, V::from(total), ttl)?; + Ok(total) + } +} + +pub(crate) fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} diff --git a/litellm-rust/crates/redis-store/src/number.rs b/litellm-rust/crates/redis-store/src/number.rs new file mode 100644 index 00000000000..7f3e9f4d10b --- /dev/null +++ b/litellm-rust/crates/redis-store/src/number.rs @@ -0,0 +1,83 @@ +use std::{cmp::Ordering, ops::Add}; + +use litellm_python_compat::{BigInt, Value, json, repr}; + +/// A Python `int` or `float`, kept apart because `repr` and `json.dumps` write them differently. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Number { + Int(i64), + Float(f64), +} + +impl From for litellm_redis_client::Amount { + fn from(number: Number) -> Self { + match number { + Number::Int(value) => Self::Int(value), + Number::Float(value) => Self::Float(value), + } + } +} + +impl Number { + pub fn as_f64(self) -> f64 { + match self { + Self::Int(value) => value as f64, + Self::Float(value) => value, + } + } + + /// Python's `int(value)`, which truncates toward zero. + pub fn truncate(self) -> i64 { + match self { + Self::Int(value) => value, + Self::Float(value) => value.trunc() as i64, + } + } + + pub fn to_value(self) -> Value { + match self { + Self::Int(value) => Value::Int(BigInt::from(value)), + Self::Float(value) => Value::Float(value), + } + } + + /// `json.dumps`, which `RedisCache.async_set_cache` writes. + pub fn to_json(self) -> String { + let value = self.to_value(); + json::dumps(&value).unwrap_or_else(|_| repr::to_str(&value)) + } + + pub fn from_value(value: &Value) -> Option { + match value { + Value::Int(value) => i64::try_from(value).ok().map(Self::Int), + Value::Float(value) => Some(Self::Float(*value)), + _ => None, + } + } + + /// Python's `min` over numbers: the first of the smallest, keeping its type. + pub fn min(values: impl IntoIterator) -> Option { + values + .into_iter() + .fold(None, |smallest, value| match smallest { + Some(smallest) + if value.as_f64().partial_cmp(&smallest.as_f64()) != Some(Ordering::Less) => + { + Some(smallest) + } + _ => Some(value), + }) + } +} + +/// Python's `+`: two ints stay an int, anything with a float is a float. +impl Add for Number { + type Output = Self; + + fn add(self, other: Self) -> Self { + match (self, other) { + (Self::Int(left), Self::Int(right)) => Self::Int(left + right), + (left, right) => Self::Float(left.as_f64() + right.as_f64()), + } + } +} diff --git a/litellm-rust/crates/redis-store/src/rate_limits/keys.rs b/litellm-rust/crates/redis-store/src/rate_limits/keys.rs new file mode 100644 index 00000000000..7f5714a19ad --- /dev/null +++ b/litellm-rust/crates/redis-store/src/rate_limits/keys.rs @@ -0,0 +1,74 @@ +/// A rate-limit descriptor, `{descriptor:value}`, the hash tag every key of its window shares. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Scope { + pub descriptor: String, + pub value: String, +} + +impl Scope { + pub(super) fn key(&self, suffix: &str) -> String { + format!("{{{}:{}}}:{suffix}", self.descriptor, self.value) + } + + pub(super) fn window_key(&self) -> String { + self.key(WINDOW) + } + + pub(super) fn gauge_key(&self) -> String { + self.key(GAUGE) + } +} + +const WINDOW: &str = "window"; +const GAUGE: &str = "max_parallel_requests"; + +/// A counter's key suffix, which is also how the Python limiter names it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, strum::EnumString, strum::IntoStaticStr)] +pub enum Kind { + #[strum(serialize = "requests")] + Requests, + #[strum(serialize = "tokens")] + Tokens, +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Counter { + pub scope: Scope, + pub kind: Kind, +} + +impl Counter { + pub(super) fn key(&self) -> String { + self.scope.key(self.kind.into()) + } +} + +/// One value of the limiter's keyspace, as both tiers address it. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum Entry { + Window(Scope), + Counter(Counter), + Gauge(Scope), +} + +impl Entry { + /// The entry the Python limiter names by its key suffix: `window`, `max_parallel_requests`, or a + /// counter's `Kind`. + pub fn new(suffix: &str, scope: Scope) -> Result { + match suffix { + WINDOW => Ok(Self::Window(scope)), + GAUGE => Ok(Self::Gauge(scope)), + kind => kind + .parse() + .map(|kind| Self::Counter(Counter { scope, kind })), + } + } + + pub(super) fn key(&self) -> String { + match self { + Self::Window(scope) => scope.window_key(), + Self::Counter(counter) => counter.key(), + Self::Gauge(scope) => scope.gauge_key(), + } + } +} diff --git a/litellm-rust/crates/redis-store/src/rate_limits/lua.rs b/litellm-rust/crates/redis-store/src/rate_limits/lua.rs new file mode 100644 index 00000000000..96451212d35 --- /dev/null +++ b/litellm-rust/crates/redis-store/src/rate_limits/lua.rs @@ -0,0 +1,260 @@ +//! The limiter's Lua, byte for byte as `main`'s Python registers it, so both share one script SHA. + +pub(super) const BATCH_RATE_LIMITER: &str = r#" +local results = {} +local now = tonumber(ARGV[1]) +local window_size = tonumber(ARGV[2]) + +-- Process each window/counter pair +for i = 1, #KEYS, 2 do + local window_key = KEYS[i] + local counter_key = KEYS[i + 1] + local increment_value = 1 + + -- Check if window exists and is valid + local window_start = redis.call('GET', window_key) + if not window_start or (now - tonumber(window_start)) >= window_size then + -- Reset window and counter + local prefix = string.sub(window_key, 1, -(#':window') - 1) + redis.call('DEL', prefix .. ':requests', prefix .. ':tokens') + redis.call('SET', window_key, tostring(now)) + redis.call('SET', counter_key, increment_value) + redis.call('EXPIRE', window_key, window_size) + redis.call('EXPIRE', counter_key, window_size) + table.insert(results, tostring(now)) -- window_start + table.insert(results, increment_value) -- counter + else + local counter = redis.call('INCR', counter_key) + -- This happens when window_key exists but counter_key doesn't (e.g., tokens key + -- created after requests key when both share the same window_key) + local current_ttl = redis.call('TTL', counter_key) + if current_ttl == -1 then + redis.call('EXPIRE', counter_key, window_size) + end + table.insert(results, window_start) -- window_start + table.insert(results, counter) -- counter + end +end + +return results +"#; + +pub(super) const BATCH_COUNTER_READ: &str = r#"return redis.call('MGET', unpack(KEYS))"#; + +pub(super) const CHECK_AND_INCREMENT_BY_N: &str = r#" +-- Atomic check-and-increment-by-N across one or more descriptors. +-- All-or-nothing: if any descriptor would exceed its limit, no counter is +-- modified. +-- +-- Uses Redis server time (`redis.call('TIME')`) instead of a client-supplied +-- timestamp so that window resets are deterministic across replicas with +-- skewed wall-clocks. This prevents a clock-skew-induced reopening of the +-- TOCTOU window across multi-replica deployments. +-- +-- KEYS layout: pairs of (window_key, counter_key), one pair per descriptor. +-- ARGV layout: per-descriptor 4-tuple, starting at ARGV[1]: +-- ARGV[(i-1)*4 + 1] = limit +-- ARGV[(i-1)*4 + 2] = increment +-- ARGV[(i-1)*4 + 3] = ttl_seconds (counter TTL when window resets) +-- ARGV[(i-1)*4 + 4] = window_size_seconds (sliding-window length) +-- +-- Return on success: +-- { 0, new_counter_1, window_start_1, new_counter_2, window_start_2, ... } +-- Return on over-limit: { 1, descriptor_index, current_counter, limit } +local time_reply = redis.call('TIME') +local now = tonumber(time_reply[1]) +local descriptor_count = #KEYS / 2 +local reset_windows = {} + +-- Pass 1: read state, validate. Abort without writing if any over limit. +local descriptor_state = {} +for i = 1, descriptor_count do + local window_key = KEYS[(i - 1) * 2 + 1] + local counter_key = KEYS[(i - 1) * 2 + 2] + local arg_base = (i - 1) * 4 + 1 + local limit = tonumber(ARGV[arg_base]) + local increment = tonumber(ARGV[arg_base + 1]) + local window_size = tonumber(ARGV[arg_base + 3]) + + local window_start = redis.call('GET', window_key) + local window_expired = (not window_start) or + ((now - tonumber(window_start)) >= window_size) + + local current_counter + if window_expired then + current_counter = 0 + else + current_counter = tonumber(redis.call('GET', counter_key) or 0) + end + + local blocked + if increment > 0 then + blocked = current_counter + increment > limit + else + blocked = current_counter >= limit + end + if blocked then + return { 1, i, current_counter, limit } + end + + descriptor_state[i] = { window_expired, current_counter, window_start } +end + +-- Pass 2: all checks passed. Apply increments. +local results = { 0 } +for i = 1, descriptor_count do + local window_key = KEYS[(i - 1) * 2 + 1] + local counter_key = KEYS[(i - 1) * 2 + 2] + local arg_base = (i - 1) * 4 + 1 + local increment = tonumber(ARGV[arg_base + 1]) + local ttl = tonumber(ARGV[arg_base + 2]) + local window_size = tonumber(ARGV[arg_base + 3]) + + local window_expired = descriptor_state[i][1] + local active_window_start + + if window_expired then + active_window_start = now + if not reset_windows[window_key] then + local prefix = string.sub(window_key, 1, -(#':window') - 1) + redis.call('DEL', prefix .. ':requests', prefix .. ':tokens') + reset_windows[window_key] = true + end + redis.call('SET', window_key, tostring(now)) + redis.call('SET', counter_key, increment) + redis.call('EXPIRE', window_key, window_size) + if ttl > 0 then + redis.call('EXPIRE', counter_key, ttl) + end + table.insert(results, increment) + else + active_window_start = tonumber(descriptor_state[i][3]) + local new_counter = redis.call('INCRBY', counter_key, increment) + local current_ttl = redis.call('TTL', counter_key) + if current_ttl == -1 and ttl > 0 then + redis.call('EXPIRE', counter_key, ttl) + end + table.insert(results, new_counter) + end + table.insert(results, active_window_start) +end + +return results +"#; + +pub(super) const WINDOW_GUARDED_TOKEN_INCREMENT: &str = r#" +local results = {} +for i = 1, #KEYS, 2 do + local window_key = KEYS[i] + local counter_key = KEYS[i + 1] + local arg_base = ((i - 1) / 2) * 3 + 1 + local expected_window_start = ARGV[arg_base] + local increment = tonumber(ARGV[arg_base + 1]) + local ttl = tonumber(ARGV[arg_base + 2]) + local active_window_start = redis.call('GET', window_key) + + if active_window_start and active_window_start == expected_window_start then + local new_counter = redis.call('INCRBY', counter_key, increment) + local current_ttl = redis.call('TTL', counter_key) + if current_ttl == -1 and ttl > 0 then + redis.call('EXPIRE', counter_key, ttl) + end + table.insert(results, 1) + table.insert(results, new_counter) + else + table.insert(results, 0) + table.insert(results, tonumber(redis.call('GET', counter_key) or 0)) + end +end +return results +"#; + +pub(super) const PARALLEL_ACQUIRE: &str = r#" +-- Atomic check-and-acquire for the max_parallel_requests concurrency gauge. +-- Each gauge key is a sorted set of per-request slot ids scored by acquire +-- time (Redis server clock). In-flight requests are counted by ZCARD after +-- pruning slots older than the slot TTL, so unlike the windowed RPM/TPM +-- counters the gauge is never reset while requests are in flight, a +-- rejected request never occupies a slot, and a slot leaked by a crashed +-- worker self-heals after the slot TTL even under continuous traffic. +-- +-- KEYS: one gauge zset key per descriptor. +-- ARGV: per-key triples (limit, slot_ttl_seconds, slot_id). +-- Success: { 0, in_flight_1, ... }. Over-limit: { 1, key_index, in_flight, limit }. +local time_reply = redis.call('TIME') +local now = tonumber(time_reply[1]) +for i = 1, #KEYS do + local limit = tonumber(ARGV[(i - 1) * 3 + 1]) + local slot_ttl = tonumber(ARGV[(i - 1) * 3 + 2]) + redis.call('ZREMRANGEBYSCORE', KEYS[i], '-inf', now - slot_ttl) + local in_flight = redis.call('ZCARD', KEYS[i]) + if in_flight + 1 > limit then + return { 1, i, in_flight, limit } + end +end +local results = { 0 } +for i = 1, #KEYS do + local slot_ttl = tonumber(ARGV[(i - 1) * 3 + 2]) + local slot_id = ARGV[(i - 1) * 3 + 3] + redis.call('ZADD', KEYS[i], now, slot_id) + redis.call('EXPIRE', KEYS[i], slot_ttl) + table.insert(results, redis.call('ZCARD', KEYS[i])) +end +return results +"#; + +pub(super) const PARALLEL_RELEASE: &str = r#" +-- Release one slot per gauge key by removing this request's slot id. +-- ZREM of an absent member (or key) is a no-op, so a release without a +-- matching acquire (proxy-side rejection, double-fired callback, slot +-- already expired) can never free a slot owned by another request. +-- KEYS: gauge zset keys. ARGV: per-key slot_id. +-- Returns the remaining in-flight count per key. +local results = {} +for i = 1, #KEYS do + redis.call('ZREM', KEYS[i], ARGV[i]) + table.insert(results, redis.call('ZCARD', KEYS[i])) +end +return results +"#; + +pub(super) const PARALLEL_COUNT: &str = r#" +-- Read the current in-flight count per gauge key (prunes expired slots +-- first so leaked slots do not inflate the reading). +-- KEYS: gauge zset keys. ARGV: per-key slot_ttl_seconds. +local time_reply = redis.call('TIME') +local now = tonumber(time_reply[1]) +local results = {} +for i = 1, #KEYS do + redis.call('ZREMRANGEBYSCORE', KEYS[i], '-inf', now - tonumber(ARGV[i])) + table.insert(results, redis.call('ZCARD', KEYS[i])) +end +return results +"#; + +pub(super) const TOKEN_INCREMENT: &str = r#" +local results = {} + +-- Process each key/increment_value/ttl triplet +for i = 1, #KEYS do + local key = KEYS[i] + local increment_value = tonumber(ARGV[i * 2 - 1]) + local ttl_seconds = tonumber(ARGV[i * 2]) + + -- Increment the value + local new_value = redis.call('INCRBYFLOAT', key, increment_value) + + -- Handle TTL: only set expire if ttl_seconds > 0 and key has no current TTL + -- ttl_seconds can be 0 (no TTL) or positive (set TTL) + if ttl_seconds and ttl_seconds > 0 then + local current_ttl = redis.call('TTL', key) + if current_ttl == -1 then + redis.call('EXPIRE', key, ttl_seconds) + end + end + + table.insert(results, new_value) +end + +return results +"#; diff --git a/litellm-rust/crates/redis-store/src/rate_limits/mod.rs b/litellm-rust/crates/redis-store/src/rate_limits/mod.rs new file mode 100644 index 00000000000..71b67d2ecc0 --- /dev/null +++ b/litellm-rust/crates/redis-store/src/rate_limits/mod.rs @@ -0,0 +1,304 @@ +mod keys; +mod lua; +mod scripts; +mod types; + +use std::{sync::Arc, time::Duration}; + +use litellm_redis_client::{Declared, RedisBatch, RedisClient, Slot}; + +pub use self::keys::{Counter, Entry, Kind, Scope}; +pub use self::types::{ + AcquireReply, Applied, AtomicCounter, AtomicReply, Gauge, GuardedIncrement, GuardedReply, + Increment, Local, WindowCount, +}; +use crate::{ + Error, Number, Pending, RequestRedis, Send, + dual_cache::{DualCache, MemorySettings, PipelineIncrement}, + memory::{Clock, system_clock}, + remote::{Remote, RemoteSlot}, + request::pending, +}; + +/// The v3 rate limiter's keyspace: its memory tier, its Lua scripts and the plain counter writes +/// `DualCache` made for it on `main`. The admission logic stays with the caller. A script needs +/// Redis, so without one it declares nothing and answers `None`. +pub struct RateLimits { + cache: DualCache, +} + +impl RateLimits { + /// `backfill_ttl` is the `default_in_memory_ttl` of the `DualCache` this tier replaces. An + /// increment without a TTL takes the remote's default. + pub fn new(remote: Option, backfill_ttl: Option) -> Self { + Self::with_clock(remote, backfill_ttl, system_clock()) + } + + pub fn with_clock(remote: Option, backfill_ttl: Option, now: Clock) -> Self { + Self { + cache: DualCache::new( + Arc::new(RemoteSlot::new(remote)), + MemorySettings { + entries: None, + ttl: backfill_ttl, + }, + now, + ), + } + } + + pub fn has_redis(&self) -> bool { + self.cache.redis().get().is_some() + } + + /// `_group_keys_by_hash_tag`: indices of `scopes` sharing a cluster slot, in first-seen order, + /// or one group when the client is not a cluster and grouping is not forced. + pub fn slot_groups(&self, scopes: &[Scope], force: bool) -> Vec> { + let by_slot = force + || self + .cache + .redis() + .client() + .is_some_and(RedisClient::is_cluster); + if !by_slot { + return vec![(0..scopes.len()).collect()]; + } + scopes + .iter() + .enumerate() + .fold( + Vec::<(Slot, Vec)>::new(), + |mut groups, (index, scope)| { + let slot = Slot::for_key(scope.window_key()); + match groups.iter_mut().find(|(seen, _)| *seen == slot) { + Some((_, members)) => members.push(index), + None => groups.push((slot, vec![index])), + } + groups + }, + ) + .into_iter() + .map(|(_, members)| members) + .collect() + } + + pub fn get_local(&self, entry: &Entry) -> Result, Error> { + Ok(self.cache.memory().get(&entry.key())?) + } + + pub fn set_local( + &self, + entry: &Entry, + value: Local, + ttl: Option, + ) -> Result<(), Error> { + Ok(self.cache.memory().set(&entry.key(), value, ttl)?) + } + + /// `DualCache.async_get_cache`, backfilling memory for `backfill_ttl` or the store's default. + /// A key `request` already read as missing from Redis is not read again. + pub async fn get( + &self, + request: Option<&RequestRedis>, + entry: &Entry, + backfill_ttl: Option, + ) -> Result, Error> { + self.cache.get(request, &entry.key(), backfill_ttl).await + } + + pub fn sliding_window( + &self, + send: Send<'_>, + counters: &[Counter], + now: i64, + window_size: i64, + ) -> Option>> { + Some(pending(scripts::sliding_window( + &self.batch(send)?, + counters, + now, + window_size, + ))) + } + + pub fn read_counters( + &self, + send: Send<'_>, + counters: &[Counter], + ) -> Option>>> { + Some(pending(scripts::read_counters( + &self.batch(send)?, + counters, + ))) + } + + pub fn check_and_increment( + &self, + send: Send<'_>, + counters: &[AtomicCounter], + ) -> Option> { + Some(pending(scripts::check_and_increment( + &self.batch(send)?, + counters, + ))) + } + + pub fn guarded_increment( + &self, + send: Send<'_>, + increment: &GuardedIncrement, + ) -> Option> { + Some(pending(scripts::guarded_increment( + &self.batch(send)?, + increment, + ))) + } + + pub fn parallel_acquire( + &self, + send: Send<'_>, + gauges: &[Gauge], + slot_ttl: Duration, + slot_id: &str, + ) -> Option> { + Some(pending(scripts::parallel_acquire( + &self.batch(send)?, + gauges, + slot_ttl, + slot_id, + ))) + } + + pub fn parallel_release( + &self, + send: Send<'_>, + gauges: &[Scope], + slot_id: &str, + ) -> Option>> { + Some(pending(self.declare_release(send, gauges, slot_id)?)) + } + + pub fn parallel_count( + &self, + send: Send<'_>, + gauges: &[Scope], + slot_ttl: Duration, + ) -> Option>> { + Some(pending(scripts::parallel_count( + &self.batch(send)?, + gauges, + slot_ttl, + ))) + } + + pub fn token_increment( + &self, + send: Send<'_>, + increments: &[Increment], + ) -> Option>> { + Some(pending(self.declare_token_increment(send, increments)?)) + } + + /// `TOKEN_INCREMENT_SCRIPT` on the request's post-call batch. A failed script falls back to + /// `increment_many` for these increments once the batch has gone out. `false` without Redis. + pub fn token_increment_post_call( + self: &Arc, + request: &RequestRedis, + increments: Vec, + ) -> bool { + let Some(declared) = self.declare_token_increment(Send::PostCall(request), &increments) + else { + return false; + }; + let store = Arc::clone(self); + request.spawn(async move { + let Err(error) = declared.settled().await else { + return; + }; + crate::failure_log::redis_failure!( + &error, + "TTL preservation failed, falling back to regular pipeline" + ); + store.increment_many(&increments).await; + }); + true + } + + /// `PARALLEL_RELEASE_SCRIPT` on the request's post-call batch; the caller has already freed + /// the slot in memory. `false` without Redis. + pub fn parallel_release_post_call( + &self, + request: &RequestRedis, + gauges: &[Scope], + slot_id: &str, + ) -> bool { + let Some(declared) = self.declare_release(Send::PostCall(request), gauges, slot_id) else { + return false; + }; + request.spawn(async move { + if let Err(error) = declared.settled().await { + crate::failure_log::redis_failure!( + &error, + "parallel_release_script failed, the slot stays released in memory only" + ); + } + }); + true + } + + /// `DualCache.async_increment_cache`. A Redis failure answers with the memory count, and an + /// entry memory cannot add to with `None`. + pub async fn increment(&self, increment: &Increment) -> Option { + self.cache + .increment(&increment.counter.key(), increment.amount, increment.ttl) + .await + } + + /// `DualCache.async_increment_cache_pipeline`, whose failures only log. + pub async fn increment_many(&self, increments: &[Increment]) { + self.cache.increment_pipeline(&pipeline(increments)).await; + } + + /// `DualCache.async_increment_cache_pipeline_post_call`: memory now, the Redis increments on + /// the request's post-call batch. + pub fn increment_many_post_call(&self, request: &RequestRedis, increments: &[Increment]) { + self.cache + .increment_pipeline_post_call(request, &pipeline(increments)); + } + + fn declare_release( + &self, + send: Send<'_>, + gauges: &[Scope], + slot_id: &str, + ) -> Option>> { + Some(scripts::parallel_release( + &self.batch(send)?, + gauges, + slot_id, + )) + } + + fn declare_token_increment( + &self, + send: Send<'_>, + increments: &[Increment], + ) -> Option>> { + Some(scripts::token_increment(&self.batch(send)?, increments)) + } + + fn batch(&self, send: Send<'_>) -> Option { + self.cache.redis().batch(send) + } +} + +fn pipeline(increments: &[Increment]) -> Vec { + increments + .iter() + .map(|increment| PipelineIncrement { + key: increment.counter.key(), + amount: increment.amount, + ttl: increment.ttl, + }) + .collect() +} diff --git a/litellm-rust/crates/redis-store/src/rate_limits/scripts.rs b/litellm-rust/crates/redis-store/src/rate_limits/scripts.rs new file mode 100644 index 00000000000..04bdad38268 --- /dev/null +++ b/litellm-rust/crates/redis-store/src/rate_limits/scripts.rs @@ -0,0 +1,303 @@ +use std::{sync::LazyLock, time::Duration}; + +use litellm_python_compat::repr::float_repr; +use litellm_redis_client::{ + Command, Declared, Error, RedisBatch, Script, Value, + commands::{self, unexpected_reply}, +}; + +use super::{ + keys::{Counter, Scope}, + lua, + types::{ + AcquireReply, Applied, AtomicCounter, AtomicReply, Gauge, GuardedIncrement, GuardedReply, + Increment, WindowCount, + }, +}; +use crate::Number; + +static BATCH_RATE_LIMITER: LazyLock', + "deprecation_source": "javascript:alert(1)", + } + ) + email = ModelDeprecationEmail( + model=model, + usage_model=model.litellm_model, + recipient_email="owner@example.com", + milestone=DeprecationEmailMilestone.THIRTY_DAYS, + key_count=3, + requests=12, + team_names=("Research",), + start_date=date(2026, 10, 3), + end_date=NOW.date(), + ) + rendered = render_model_deprecation_email(email, base_url="https://gateway.example") + assert "/public/assets/logos/litellm_logo.png" in rendered.html + assert "litellm-listing.s3.amazonaws.com" not in rendered.html + assert "