diff --git a/.circleci/config.yml b/.circleci/config.yml index 0bfbbcb4405..0ebf9127033 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,8 +1,8 @@ version: 2.1 orbs: codecov: codecov/codecov@4.0.1 - node: circleci/node@5.1.0 # Add this line to declare the node orb - win: circleci/windows@5.0 # Add Windows orb + node: circleci/node@5.1.0 # Add this line to declare the node orb + win: circleci/windows@5.0 # Add Windows orb commands: setup_google_dns: @@ -50,7 +50,7 @@ jobs: name: Run Windows-specific test command: | python -m pytest tests/windows_tests/test_litellm_on_windows.py -v - + mypy_linting: docker: - image: cimg/python:3.12 @@ -500,7 +500,7 @@ jobs: paths: - litellm_router_coverage.xml - litellm_router_coverage - + litellm_router_unit_testing: # Runs all tests with the "router" keyword docker: - image: cimg/python:3.11 @@ -532,7 +532,7 @@ jobs: command: | pwd ls - python -m pytest -vv tests/router_unit_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + python -m pytest -vv tests/router_unit_tests --cov=litellm --cov-report=xml -x -s --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 120m - run: name: Rename the coverage files @@ -563,8 +563,9 @@ jobs: - run: name: Install Docker CLI (In case it's not already installed) command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version - run: name: Install Python 3.13 command: | @@ -1041,6 +1042,92 @@ jobs: paths: - llm_responses_api_coverage.xml - llm_responses_api_coverage + ocr_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml ocr_coverage.xml + mv .coverage ocr_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - ocr_coverage.xml + - ocr_coverage + search_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml search_coverage.xml + mv .coverage search_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - search_coverage.xml + - search_coverage litellm_mapped_tests: docker: - image: cimg/python:3.11 @@ -1078,7 +1165,7 @@ jobs: command: | pwd ls - python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -s -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 8 + python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 8 no_output_timeout: 120m - run: name: Rename the coverage files @@ -1310,7 +1397,7 @@ jobs: command: | pwd ls - python -m pytest -vv tests/image_gen_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + python -m pytest -vv tests/image_gen_tests --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 120m - run: name: Rename the coverage files @@ -1376,6 +1463,49 @@ jobs: paths: - logging_coverage.xml - logging_coverage + audio_testing: + docker: + - image: cimg/python:3.11 + auth: + username: ${DOCKERHUB_USERNAME} + password: ${DOCKERHUB_PASSWORD} + working_directory: ~/project + + steps: + - checkout + - setup_google_dns + - run: + name: Install Dependencies + command: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + pip install "pytest==7.3.1" + pip install "pytest-retry==1.6.3" + pip install "pytest-cov==5.0.0" + pip install "pytest-asyncio==0.21.1" + pip install "respx==0.22.0" + # Run pytest and generate JUnit XML report + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: 120m + - run: + name: Rename the coverage files + command: | + mv coverage.xml audio_coverage.xml + mv .coverage audio_coverage + + # Store test results + - store_test_results: + path: test-results + - persist_to_workspace: + root: . + paths: + - audio_coverage.xml + - audio_coverage installing_litellm_on_python: docker: - image: circleci/python:3.8 @@ -1442,7 +1572,7 @@ jobs: python -m pytest -vv tests/local_testing/test_basic_python_version.py helm_chart_testing: machine: - image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker + image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker resource_class: medium working_directory: ~/project @@ -1454,7 +1584,7 @@ jobs: name: Install Helm command: | curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - + # Install kind - run: name: Install Kind @@ -1462,7 +1592,7 @@ jobs: curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 chmod +x ./kind sudo mv ./kind /usr/local/bin/kind - + # Install kubectl - run: name: Install kubectl @@ -1470,19 +1600,19 @@ jobs: curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" chmod +x kubectl sudo mv kubectl /usr/local/bin/ - + # Create kind cluster - run: name: Create Kind Cluster command: | kind create cluster --name litellm-test - + # Run helm lint - run: name: Run helm lint command: | helm lint ./deploy/charts/litellm-helm - + # Run helm tests - run: name: Run helm tests @@ -1491,22 +1621,21 @@ jobs: # Wait for pod to be ready echo "Waiting 30 seconds for pod to be ready..." sleep 30 - + # Print pod logs before running tests echo "Printing pod logs..." kubectl logs $(kubectl get pods -l app.kubernetes.io/name=litellm -o jsonpath="{.items[0].metadata.name}") - + # Run the helm tests helm test litellm --logs helm test litellm --logs - + # Cleanup - run: name: Cleanup command: | kind delete cluster --name litellm-test - when: always # This ensures cleanup runs even if previous steps fail - + when: always # This ensures cleanup runs even if previous steps fail check_code_and_doc_quality: docker: @@ -1580,6 +1709,7 @@ jobs: pip install "pytest==7.3.1" pip install "pytest-asyncio==0.21.1" pip install aiohttp + pip install apscheduler - run: name: Build Docker image command: | @@ -1617,7 +1747,7 @@ jobs: echo "=== Printing Full Container Startup Logs ===" docker logs my-app echo "=== End of Full Container Startup Logs ===" - + if docker logs my-app 2>&1 | grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two"; then echo "Expected message found in logs. Test passed." else @@ -1630,7 +1760,6 @@ jobs: python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 no_output_timeout: 120m - build_and_test: machine: image: ubuntu-2204:2023.10.1 @@ -1642,8 +1771,9 @@ jobs: - run: name: Install Docker CLI (In case it's not already installed) command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version - run: name: Install Python 3.9 command: | @@ -1780,8 +1910,9 @@ jobs: - run: name: Install Docker CLI (In case it's not already installed) command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version - run: name: Install Python 3.9 command: | @@ -1922,8 +2053,9 @@ jobs: - run: name: Install Docker CLI (In case it's not already installed) command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version - run: name: Install Python 3.9 command: | @@ -2106,8 +2238,9 @@ jobs: - run: name: Install Docker CLI (In case it's not already installed) command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version - run: name: Install Python 3.9 command: | @@ -2214,8 +2347,9 @@ jobs: - run: name: Install Docker CLI (In case it's not already installed) command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version - run: name: Install Python 3.9 command: | @@ -2347,8 +2481,10 @@ jobs: - run: name: Install Docker CLI (In case it's not already installed) command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + sudo systemctl restart docker - run: name: Install Python 3.9 command: | @@ -2434,8 +2570,7 @@ jobs: pwd ls python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5 - no_output_timeout: - 120m + no_output_timeout: 120m - run: name: Stop and remove containers command: | @@ -2446,7 +2581,7 @@ jobs: when: always - store_test_results: path: test-results - + proxy_build_from_pip_tests: # Change from docker to machine executor machine: @@ -2480,6 +2615,7 @@ jobs: pip install "pytest-mock==3.12.0" pip install "pytest-asyncio==0.21.1" pip install "mypy==1.18.2" + pip install apscheduler - run: name: Build Docker image command: | @@ -2555,8 +2691,9 @@ jobs: - run: name: Install Docker CLI (In case it's not already installed) command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version - run: name: Install Python 3.9 command: | @@ -2633,8 +2770,8 @@ jobs: -e GEMINI_API_KEY=$GEMINI_API_KEY \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \ - -e AZURE_API_KEY_PASSHROUGH=$AZURE_API_KEY_PASSHROUGH \ - -e AZURE_API_BASE_PASSHROUGH=$AZURE_API_BASE_PASSHROUGH \ + -e AZURE_API_KEY=$AZURE_API_KEY \ + -e AZURE_API_BASE=$AZURE_API_BASE \ -e USE_DDTRACE=True \ -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ @@ -2663,17 +2800,17 @@ jobs: curl -sSL https://rvm.io/mpapis.asc | gpg --import - curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - } - + # Install Ruby version manager (RVM) curl -sSL https://get.rvm.io | bash -s stable - + # Source RVM from the correct location source $HOME/.rvm/scripts/rvm - + # Install Ruby 3.2.2 rvm install 3.2.2 rvm use 3.2.2 --default - + # Install latest Bundler gem install bundler @@ -2741,7 +2878,7 @@ jobs: python -m venv venv . venv/bin/activate pip install coverage - coverage combine llm_translation_coverage llm_responses_api_coverage mcp_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage + coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage coverage xml - codecov/upload: file: ./coverage.xml @@ -2827,32 +2964,32 @@ jobs: python -m pip install toml # Get current version from pyproject.toml CURRENT_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])") - + # Get last published version from PyPI LAST_VERSION=$(curl -s https://pypi.org/pypi/litellm-proxy-extras/json | python -c "import json, sys; print(json.load(sys.stdin)['info']['version'])") - + echo "Current version: $CURRENT_VERSION" echo "Last published version: $LAST_VERSION" - + # Compare versions using Python's packaging.version VERSION_COMPARE=$(python -c "from packaging import version; print(1 if version.parse('$CURRENT_VERSION') < version.parse('$LAST_VERSION') else 0)") - + echo "Version compare: $VERSION_COMPARE" if [ "$VERSION_COMPARE" = "1" ]; then echo "Error: Current version ($CURRENT_VERSION) is less than last published version ($LAST_VERSION)" exit 1 fi - + # If versions are equal or current is greater, check contents pip download --no-deps litellm-proxy-extras==$LAST_VERSION -d /tmp - + echo "Contents of /tmp directory:" ls -la /tmp - + # Find the downloaded file (could be .whl or .tar.gz) DOWNLOADED_FILE=$(ls /tmp/litellm_proxy_extras-*) echo "Downloaded file: $DOWNLOADED_FILE" - + # Extract based on file extension if [[ "$DOWNLOADED_FILE" == *.whl ]]; then echo "Extracting wheel file..." @@ -2863,10 +3000,10 @@ jobs: tar -xzf "$DOWNLOADED_FILE" -C /tmp EXTRACTED_DIR="/tmp/litellm_proxy_extras-$LAST_VERSION" fi - + echo "Contents of extracted package:" ls -R "$EXTRACTED_DIR" - + # Compare contents if ! diff -r "$EXTRACTED_DIR/litellm_proxy_extras" ./litellm_proxy_extras; then if [ "$CURRENT_VERSION" = "$LAST_VERSION" ]; then @@ -2932,23 +3069,24 @@ jobs: export NVM_DIR="/opt/circleci/.nvm" source "$NVM_DIR/nvm.sh" source "$NVM_DIR/bash_completion" - + # Install and use Node version nvm install v20 nvm use v20 - + cd ui/litellm-dashboard - + # Install dependencies first npm install - + # Now source the build script source ./build_ui.sh - run: - name: Install Docker CLI (In case it's not already installed) + name: Upgrade Docker to v24.x (API 1.44+) command: | - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version - run: name: Install Python 3.9 command: | @@ -2996,10 +3134,10 @@ jobs: source "$NVM_DIR/nvm.sh" nvm install 20 nvm use 20 - + cd ui/litellm-dashboard npm ci || npm install - + # CI run, with both LCOV (Codecov) and HTML (artifact you can click) CI=true npm run test -- --run --coverage \ --coverage.provider=v8 \ @@ -3007,7 +3145,6 @@ jobs: --coverage.reporter=html \ --coverage.reportsDirectory=coverage/html - - run: name: Build Docker image command: docker build -t my-app:latest -f ./docker/Dockerfile.database . @@ -3289,6 +3426,18 @@ workflows: only: - main - /litellm_.*/ + - ocr_testing: + filters: + branches: + only: + - main + - /litellm_.*/ + - search_testing: + filters: + branches: + only: + - main + - /litellm_.*/ - litellm_mapped_enterprise_tests: filters: branches: @@ -3331,6 +3480,12 @@ workflows: only: - main - /litellm_.*/ + - audio_testing: + filters: + branches: + only: + - main + - /litellm_.*/ - upload-coverage: requires: - llm_translation_testing @@ -3338,6 +3493,8 @@ workflows: - google_generate_content_endpoint_testing - guardrails_testing - llm_responses_api_testing + - ocr_testing + - search_testing - litellm_mapped_tests - litellm_mapped_enterprise_tests - batches_testing @@ -3345,6 +3502,7 @@ workflows: - pass_through_unit_testing - image_gen_testing - logging_testing + - audio_testing - litellm_router_testing - litellm_router_unit_testing - caching_unit_tests @@ -3400,6 +3558,8 @@ workflows: - mcp_testing - google_generate_content_endpoint_testing - llm_responses_api_testing + - ocr_testing + - search_testing - litellm_mapped_tests - litellm_mapped_enterprise_tests - batches_testing @@ -3407,6 +3567,7 @@ workflows: - pass_through_unit_testing - image_gen_testing - logging_testing + - audio_testing - litellm_router_testing - litellm_router_unit_testing - caching_unit_tests @@ -3428,4 +3589,3 @@ workflows: - check_code_and_doc_quality - publish_proxy_extras - guardrails_testing - diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt index 8e0f1dfe7e9..2294c84813c 100644 --- a/.circleci/requirements.txt +++ b/.circleci/requirements.txt @@ -15,4 +15,5 @@ fastapi-sso==0.16.0 uvloop==0.21.0 mcp==1.10.1 # for MCP server semantic_router==0.1.10 # for auto-routing with litellm -fastuuid==0.12.0 \ No newline at end of file +fastuuid==0.12.0 +responses==0.25.7 # for proxy client tests \ No newline at end of file diff --git a/.dockerignore b/.dockerignore index 89c3c34bd71..76e31546c2f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,9 +4,51 @@ cookbook .github tests .git -.github -.circleci .devcontainer *.tgz log.txt docker/Dockerfile.* + +# Claude Flow generated files (must be excluded from Docker build) +.claude/ +.claude-flow/ +.swarm/ +.hive-mind/ +memory/ +coordination/ +claude-flow +.mcp.json +hive-mind-prompt-*.txt + +# Python virtual environments and version managers +.venv/ +venv/ +**/.venv/ +**/venv/ +.python-version +.pyenv/ +__pycache__/ +**/__pycache__/ +*.pyc +.mypy_cache/ +.pytest_cache/ +.ruff_cache/ +**/pyvenv.cfg + +# Common project exclusions +.vscode +*.pyo +*.pyd +.Python +env/ +.pytest_cache +.coverage +htmlcov/ +dist/ +build/ +*.egg-info/ +.DS_Store +node_modules/ +*.log +.env +.env.local diff --git a/.github/workflows/interpret_load_test.py b/.github/workflows/interpret_load_test.py index 6b5e6535d79..0b5df738626 100644 --- a/.github/workflows/interpret_load_test.py +++ b/.github/workflows/interpret_load_test.py @@ -88,6 +88,7 @@ def get_docker_run_command(release_version): if __name__ == "__main__": + return csv_file = "load_test_stats.csv" # Change this to the path of your CSV file markdown_table = interpret_results(csv_file) diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index b7f4a25d593..1d9bd201fa8 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -33,6 +33,7 @@ jobs: poetry run pip install "google-genai==1.22.0" poetry run pip install "google-cloud-aiplatform>=1.38" poetry run pip install "fastapi-offline==1.7.3" + poetry run pip install "python-multipart==0.0.18" - name: Setup litellm-enterprise as local package run: | cd enterprise @@ -40,4 +41,4 @@ jobs: cd .. - name: Run tests run: | - poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 + poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 diff --git a/.gitignore b/.gitignore index e1045032d46..aa973201fd1 100644 --- a/.gitignore +++ b/.gitignore @@ -99,3 +99,4 @@ litellm/proxy/to_delete_loadtest_work/* update_model_cost_map.py tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py litellm/proxy/_experimental/out/guardrails/index.html +scripts/test_vertex_ai_search.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ad58a4976d6..3e835809b71 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -258,7 +258,7 @@ docker run \ If you need help: - 💬 [Join our Discord](https://discord.gg/wuPM9dRgDw) -- 💬 [Join our Slack](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3) +- 💬 [Join our Slack](https://www.litellm.ai/support) - 📧 Email us: ishaan@berri.ai / krrish@berri.ai - 🐛 [Create an issue](https://github.com/BerriAI/litellm/issues/new) diff --git a/Dockerfile b/Dockerfile index 6ab78d85e33..d9ea0d9a471 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,6 +65,10 @@ COPY --from=builder /wheels/ /wheels/ # Install the built wheel using pip; again using a wildcard if it's the only file RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels +# Remove test files and keys from dependencies +RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \ + find /usr/lib -type d -path "*/tornado/test" -delete + # Install semantic_router and aurelio-sdk using script RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh diff --git a/Makefile b/Makefile index 159fe4fa2ef..a79a397f945 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ install-proxy-dev-ci: install-test-deps: install-proxy-dev poetry run pip install "pytest-retry==1.6.3" poetry run pip install pytest-xdist - cd enterprise && python -m pip install -e . && cd .. + cd enterprise && poetry run pip install -e . && cd .. install-helm-unittest: helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists" diff --git a/README.md b/README.md index c785ee82ffa..b29c86a1125 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ LiteLLM manages: - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) - Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy) +LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks)) + [**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#litellm-proxy-server-llm-gateway---docs)
[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs) @@ -132,11 +134,15 @@ print(response) ## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream)) -liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response. +LiteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response. Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.) ```python from litellm import completion + +messages = [{"content": "Hello, how are you?", "role": "user"}] + +# gpt-4o response = completion(model="openai/gpt-4o", messages=messages, stream=True) for part in response: print(part.choices[0].delta.content or "") @@ -301,52 +307,108 @@ curl 'http://0.0.0.0:4000/key/generate' \ } ``` -## Supported Providers ([Docs](https://docs.litellm.ai/docs/providers)) +## Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers)) -| Provider | [Completion](https://docs.litellm.ai/docs/#basic-usage) | [Streaming](https://docs.litellm.ai/docs/completion/stream#streaming-responses) | [Async Completion](https://docs.litellm.ai/docs/completion/stream#async-completion) | [Async Streaming](https://docs.litellm.ai/docs/completion/stream#async-streaming) | [Async Embedding](https://docs.litellm.ai/docs/embedding/supported_embedding) | [Async Image Generation](https://docs.litellm.ai/docs/image_generation) | -|-------------------------------------------------------------------------------------|---------------------------------------------------------|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------|-------------------------------------------------------------------------------|-------------------------------------------------------------------------| -| [openai](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| [Meta - Llama API](https://docs.litellm.ai/docs/providers/meta_llama) | ✅ | ✅ | ✅ | ✅ | | | -| [azure](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| [AI/ML API](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| [aws - sagemaker](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ | ✅ | | -| [aws - bedrock](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ | ✅ | | -| [google - vertex_ai](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| [google - palm](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | ✅ | | | -| [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | ✅ | | | -| [mistral ai api](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | ✅ | | -| [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | ✅ | | | -| [CompactifAI](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ | ✅ | | | -| [cohere](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | ✅ | | -| [anthropic](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | ✅ | | | -| [empower](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | ✅ | -| [huggingface](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ | ✅ | | -| [replicate](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | ✅ | | | -| [together_ai](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | ✅ | | | -| [openrouter](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | ✅ | | | -| [ai21](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | ✅ | | | -| [baseten](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | ✅ | | | -| [vllm](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | ✅ | | | -| [nlp_cloud](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | ✅ | | | -| [aleph alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | ✅ | | | -| [petals](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | ✅ | | | -| [ollama](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ | ✅ | | -| [deepinfra](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | ✅ | | | -| [perplexity-ai](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | ✅ | | | -| [Groq AI](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | ✅ | | | -| [Deepseek](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | ✅ | | | -| [anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | ✅ | | | -| [IBM - watsonx.ai](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | ✅ | | -| [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ | | -| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ | | -| [FriendliAI](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ | ✅ | | | -| [Galadriel](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ | ✅ | | | -| [GradientAI](https://docs.litellm.ai/docs/providers/gradient_ai) | ✅ | ✅ | | | | | -| [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | ✅ | | | -| [Featherless AI](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | ✅ | | | -| [Nebius AI Studio](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | ✅ | | -| [Heroku](https://docs.litellm.ai/docs/providers/heroku) | ✅ | ✅ | | | | | -| [OVHCloud AI Endpoints](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | | | | | +| Provider | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` | +|-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------| +| [AI/ML API (`aiml`)](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | +| [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | | +| [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | | | | | | | | +| [Aleph Alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | | | | | | | | +| [Anthropic (`anthropic`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | | | | | | ✅ | | +| [Anthropic Text (`anthropic_text`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | | | | | | ✅ | | +| [Anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | | | | | | | | +| [AssemblyAI (`assemblyai`)](https://docs.litellm.ai/docs/pass_through/assembly_ai) | ✅ | ✅ | ✅ | | | ✅ | | | | | +| [Auto Router (`auto_router`)](https://docs.litellm.ai/docs/proxy/auto_routing) | ✅ | ✅ | ✅ | | | | | | | | +| [AWS - Bedrock (`bedrock`)](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ | +| [AWS - Sagemaker (`sagemaker`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ | | | | | | | +| [Azure (`azure`)](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | +| [Azure AI (`azure_ai`)](https://docs.litellm.ai/docs/providers/azure_ai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | +| [Azure Text (`azure_text`)](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | | +| [Baseten (`baseten`)](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | | | | | | | | +| [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | | +| [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | | +| [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | | +| [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | | | | | | | | +| [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | | +| [Cohere (`cohere`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ | +| [Cohere Chat (`cohere_chat`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | | | | | | | | +| [CometAPI (`cometapi`)](https://docs.litellm.ai/docs/providers/cometapi) | ✅ | ✅ | ✅ | ✅ | | | | | | | +| [CompactifAI (`compactifai`)](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ | | | | | | | | +| [Custom (`custom`)](https://docs.litellm.ai/docs/providers/custom_llm_server) | ✅ | ✅ | ✅ | | | | | | | | +| [Custom OpenAI (`custom_openai`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | | +| [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ | | | | | | | | +| [Databricks (`databricks`)](https://docs.litellm.ai/docs/providers/databricks) | ✅ | ✅ | ✅ | | | | | | | | +| [DataRobot (`datarobot`)](https://docs.litellm.ai/docs/providers/datarobot) | ✅ | ✅ | ✅ | | | | | | | | +| [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ | | | ✅ | | | | | +| [DeepInfra (`deepinfra`)](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | | | | | | | | +| [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | | | | | | | | +| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | | ✅ | | | | +| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | | +| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | | +| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | | +| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | | +| [FriendliAI (`friendliai`)](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ | | | | | | | | +| [Galadriel (`galadriel`)](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ | | | | | | | | +| [GitHub Copilot (`github_copilot`)](https://docs.litellm.ai/docs/providers/github_copilot) | ✅ | ✅ | ✅ | | | | | | | | +| [GitHub Models (`github`)](https://docs.litellm.ai/docs/providers/github) | ✅ | ✅ | ✅ | | | | | | | | +| [Google - PaLM](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | | | | | | | | +| [Google - Vertex AI (`vertex_ai`)](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | +| [Google AI Studio - Gemini (`gemini`)](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | | | | | | | | +| [GradientAI (`gradient_ai`)](https://docs.litellm.ai/docs/providers/gradient_ai) | ✅ | ✅ | ✅ | | | | | | | | +| [Groq AI (`groq`)](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | | | | | | | | +| [Heroku (`heroku`)](https://docs.litellm.ai/docs/providers/heroku) | ✅ | ✅ | ✅ | | | | | | | | +| [Hosted VLLM (`hosted_vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | | +| [Huggingface (`huggingface`)](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ | +| [Hyperbolic (`hyperbolic`)](https://docs.litellm.ai/docs/providers/hyperbolic) | ✅ | ✅ | ✅ | | | | | | | | +| [IBM - Watsonx.ai (`watsonx`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | | | | | | | +| [Infinity (`infinity`)](https://docs.litellm.ai/docs/providers/infinity) | | | | ✅ | | | | | | | +| [Jina AI (`jina_ai`)](https://docs.litellm.ai/docs/providers/jina_ai) | | | | ✅ | | | | | | | +| [Lambda AI (`lambda_ai`)](https://docs.litellm.ai/docs/providers/lambda_ai) | ✅ | ✅ | ✅ | | | | | | | | +| [Lemonade (`lemonade`)](https://docs.litellm.ai/docs/providers/lemonade) | ✅ | ✅ | ✅ | | | | | | | | +| [LiteLLM Proxy (`litellm_proxy`)](https://docs.litellm.ai/docs/providers/litellm_proxy) | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | +| [Llamafile (`llamafile`)](https://docs.litellm.ai/docs/providers/llamafile) | ✅ | ✅ | ✅ | | | | | | | | +| [LM Studio (`lm_studio`)](https://docs.litellm.ai/docs/providers/lm_studio) | ✅ | ✅ | ✅ | | | | | | | | +| [Maritalk (`maritalk`)](https://docs.litellm.ai/docs/providers/maritalk) | ✅ | ✅ | ✅ | | | | | | | | +| [Meta - Llama API (`meta_llama`)](https://docs.litellm.ai/docs/providers/meta_llama) | ✅ | ✅ | ✅ | | | | | | | | +| [Mistral AI API (`mistral`)](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | | | | | | | +| [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ | | | | | | | | +| [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ | | | | | | | | +| [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | | | | | | | +| [NLP Cloud (`nlp_cloud`)](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | | | | | | | | +| [Novita AI (`novita`)](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | | | | | | | | +| [Nscale (`nscale`)](https://docs.litellm.ai/docs/providers/nscale) | ✅ | ✅ | ✅ | | | | | | | | +| [Nvidia NIM (`nvidia_nim`)](https://docs.litellm.ai/docs/providers/nvidia_nim) | ✅ | ✅ | ✅ | | | | | | | | +| [OCI (`oci`)](https://docs.litellm.ai/docs/providers/oci) | ✅ | ✅ | ✅ | | | | | | | | +| [Ollama (`ollama`)](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ | | | | | | | +| [Ollama Chat (`ollama_chat`)](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | | | | | | | | +| [Oobabooga (`oobabooga`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | | +| [OpenAI (`openai`)](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | +| [OpenAI-like (`openai_like`)](https://docs.litellm.ai/docs/providers/openai_compatible) | | | | ✅ | | | | | | | +| [OpenRouter (`openrouter`)](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | | | | | | | | +| [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ | | | | | | | | +| [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | | | | | | | | +| [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | | | | | | | | +| [Predibase (`predibase`)](https://docs.litellm.ai/docs/providers/predibase) | ✅ | ✅ | ✅ | | | | | | | | +| [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) | | | | | ✅ | | | | | | +| [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | | | | | | | | +| [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | | | | | | | | +| [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | | +| [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | | +| [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | | +| [Text Completion OpenAI (`text-completion-openai`)](https://docs.litellm.ai/docs/providers/text_completion_openai) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | | +| [Together AI (`together_ai`)](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | | | | | | | | +| [Topaz (`topaz`)](https://docs.litellm.ai/docs/providers/topaz) | ✅ | ✅ | ✅ | | | | | | | | +| [Triton (`triton`)](https://docs.litellm.ai/docs/providers/triton-inference-server) | ✅ | ✅ | ✅ | | | | | | | | +| [V0 (`v0`)](https://docs.litellm.ai/docs/providers/v0) | ✅ | ✅ | ✅ | | | | | | | | +| [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | | +| [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | | +| [Volcengine (`volcengine`)](https://docs.litellm.ai/docs/providers/volcano) | ✅ | ✅ | ✅ | | | | | | | | +| [Voyage AI (`voyage`)](https://docs.litellm.ai/docs/providers/voyage) | | | | ✅ | | | | | | | +| [WandB Inference (`wandb`)](https://docs.litellm.ai/docs/providers/wandb_inference) | ✅ | ✅ | ✅ | | | | | | | | +| [Watsonx Text (`watsonx_text`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | | | | | | | | +| [xAI (`xai`)](https://docs.litellm.ai/docs/providers/xai) | ✅ | ✅ | ✅ | | | | | | | | +| [Xinference (`xinference`)](https://docs.litellm.ai/docs/providers/xinference) | | | | ✅ | | | | | | | [**Read the Docs**](https://docs.litellm.ai/docs/) diff --git a/VERTEX_ENV_SETUP.md b/VERTEX_ENV_SETUP.md new file mode 100644 index 00000000000..93a631c82f1 --- /dev/null +++ b/VERTEX_ENV_SETUP.md @@ -0,0 +1,261 @@ +# Vertex AI Environment Variables Setup Guide + +## Overview + +LiteLLM can load Vertex AI credentials from environment variables instead of storing them in config files. This is more secure and easier to manage for local development. + +## Environment Variables + +LiteLLM looks for these environment variables (in order of precedence): + +### 1. **DEFAULT_VERTEXAI_PROJECT** (Required) +Your GCP project ID that has Vertex AI enabled. + +```bash +export DEFAULT_VERTEXAI_PROJECT="my-gcp-project-id" +``` + +### 2. **DEFAULT_VERTEXAI_LOCATION** (Required) +The region/location for Vertex AI services. + +```bash +export DEFAULT_VERTEXAI_LOCATION="global" +# or +export DEFAULT_VERTEXAI_LOCATION="us-central1" +``` + +Common locations: +- `global` - For Discovery Engine and global services +- `us-central1` - US Central region +- `us-east1` - US East region +- `europe-west1` - Europe West region +- `asia-southeast1` - Asia Southeast region + +### 3. **DEFAULT_GOOGLE_APPLICATION_CREDENTIALS** (Required) +Path to your service account JSON key file. + +```bash +export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json" +``` + +### 4. **GOOGLE_APPLICATION_CREDENTIALS** (Fallback) +Standard Google Cloud environment variable (used as fallback). + +```bash +export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json" +``` + +## Quick Setup + +### Option 1: Interactive Script + +```bash +chmod +x setup_vertex_env.sh +source setup_vertex_env.sh +``` + +### Option 2: Manual Setup + +1. **Set environment variables** (for current session): + +```bash +export DEFAULT_VERTEXAI_PROJECT="your-project-id" +export DEFAULT_VERTEXAI_LOCATION="global" +export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json" +export GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json" +``` + +2. **Make them persistent** (add to `~/.zshrc` or `~/.bashrc`): + +```bash +echo 'export DEFAULT_VERTEXAI_PROJECT="your-project-id"' >> ~/.zshrc +echo 'export DEFAULT_VERTEXAI_LOCATION="global"' >> ~/.zshrc +echo 'export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"' >> ~/.zshrc +echo 'export GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"' >> ~/.zshrc +``` + +3. **Reload your shell**: + +```bash +source ~/.zshrc +``` + +## Service Account Setup + +### 1. Create a Service Account + +```bash +gcloud iam service-accounts create litellm-vertex-sa \ + --display-name="LiteLLM Vertex AI Service Account" +``` + +### 2. Grant Necessary Permissions + +For Discovery Engine (vector stores): +```bash +gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \ + --member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \ + --role="roles/discoveryengine.viewer" + +gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \ + --member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \ + --role="roles/discoveryengine.dataStoreEditor" +``` + +For general Vertex AI: +```bash +gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \ + --member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \ + --role="roles/aiplatform.user" +``` + +### 3. Create and Download Key + +```bash +gcloud iam service-accounts keys create ~/service-account-key.json \ + --iam-account=litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com +``` + +## Verify Setup + +### Check Environment Variables + +```bash +python3 << 'EOF' +import os +print("✓ Environment Variables:") +print(f" DEFAULT_VERTEXAI_PROJECT: {os.getenv('DEFAULT_VERTEXAI_PROJECT')}") +print(f" DEFAULT_VERTEXAI_LOCATION: {os.getenv('DEFAULT_VERTEXAI_LOCATION')}") +print(f" DEFAULT_GOOGLE_APPLICATION_CREDENTIALS: {os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS')}") +print(f" GOOGLE_APPLICATION_CREDENTIALS: {os.getenv('GOOGLE_APPLICATION_CREDENTIALS')}") + +# Check if credentials file exists +creds_path = os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS') +if creds_path and os.path.exists(creds_path): + print(f"\n✅ Credentials file found at: {creds_path}") +else: + print(f"\n❌ Credentials file NOT found at: {creds_path}") +EOF +``` + +### Test Authentication + +```bash +python3 << 'EOF' +import os +import json +from google.oauth2 import service_account +from google.auth.transport.requests import Request + +creds_path = os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS') +project = os.getenv('DEFAULT_VERTEXAI_PROJECT') + +try: + # Load credentials + credentials = service_account.Credentials.from_service_account_file( + creds_path, + scopes=['https://www.googleapis.com/auth/cloud-platform'] + ) + + # Get access token + credentials.refresh(Request()) + + print("✅ Authentication successful!") + print(f" Project: {project}") + print(f" Service Account: {credentials.service_account_email}") + print(f" Token expiry: {credentials.expiry}") + +except Exception as e: + print(f"❌ Authentication failed: {e}") +EOF +``` + +## Using with Vector Store Passthrough + +Once your environment is set up, the vector store passthrough will work in two ways: + +### 1. **With Vector Store Config** (Priority 1) +If you have a vector store configured with its own credentials in `litellm_params`, those will be used first: + +```yaml +vector_stores: + - vector_store_id: test-store-123 + custom_llm_provider: vertex_ai + litellm_params: + vertex_project: "specific-project" + vertex_location: "us-central1" + vertex_credentials: "{...}" # Inline credentials +``` + +### 2. **Environment Variables Fallback** (Priority 2) +If the vector store doesn't have explicit credentials, it falls back to your environment variables: + +```yaml +vector_stores: + - vector_store_id: test-store-123 + custom_llm_provider: vertex_ai + # No litellm_params - will use DEFAULT_VERTEXAI_PROJECT, DEFAULT_VERTEXAI_LOCATION, etc. +``` + +### 3. **Model Config Fallback** (Priority 3) +If neither above work, it looks for credentials in your model configuration. + +## Troubleshooting + +### "No credentials found" + +Check that all environment variables are set: +```bash +env | grep -E "(DEFAULT_VERTEXAI|GOOGLE_APPLICATION_CREDENTIALS)" +``` + +### "Authentication failed" + +Verify your service account key is valid: +```bash +cat $DEFAULT_GOOGLE_APPLICATION_CREDENTIALS | python3 -m json.tool +``` + +### "Permission denied" + +Ensure your service account has the necessary roles: +```bash +gcloud projects get-iam-policy YOUR_PROJECT_ID \ + --flatten="bindings[].members" \ + --filter="bindings.members:serviceAccount:litellm-vertex-sa@*" +``` + +### Different Credentials for Different Projects + +If you need to use different credentials for different vector stores, configure them explicitly in the vector store config rather than relying on environment variables. + +## Start LiteLLM Proxy + +Once your environment is configured: + +```bash +# Start the proxy (it will automatically load env vars) +litellm --config proxy_server_config.yaml + +# Or with debug logging +export LITELLM_LOG=DEBUG +litellm --config proxy_server_config.yaml +``` + +You should see logs like: +``` +Vertex: Loading vertex credentials from /path/to/service-account.json +Found credentials for vertex_ai_default +``` + +## Test the Endpoint + +```bash +curl -X POST http://0.0.0.0:4000/vertex_ai/discovery/v1/projects/fake-project/locations/global/dataStores/test-store-123/servingConfigs/default_config:search \ + -H 'Authorization: Bearer YOUR_LITELLM_API_KEY' \ + -H 'Content-Type: application/json' \ + -d '{"query": "test query"}' +``` + +The proxy will use your environment credentials to make the request to Vertex AI! + diff --git a/batch_small.jsonl b/batch_small.jsonl new file mode 100644 index 00000000000..36792f79dec --- /dev/null +++ b/batch_small.jsonl @@ -0,0 +1,4 @@ +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello, how are you?"}]}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "What is the weather today?"}]}} +{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Tell me a short joke"}]}} + diff --git a/cookbook/LiteLLM_CometAPI.ipynb b/cookbook/LiteLLM_CometAPI.ipynb new file mode 100644 index 00000000000..bdd916c5bfe --- /dev/null +++ b/cookbook/LiteLLM_CometAPI.ipynb @@ -0,0 +1,474 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "iFEmsVJI_2BR" + }, + "source": [ + "# LiteLLM CometAPI Cookbook" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "cBlUhCEP_xj4" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: litellm in /Users/xmx/.miniforge3/lib/python3.12/site-packages (1.78.2)\n", + "Requirement already satisfied: aiohttp>=3.10 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.11.18)\n", + "Requirement already satisfied: click in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.3.0)\n", + "Requirement already satisfied: fastuuid>=0.13.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.13.3)\n", + "Requirement already satisfied: httpx>=0.23.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.28.1)\n", + "Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.6.1)\n", + "Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.1.6)\n", + "Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (4.25.1)\n", + "Requirement already satisfied: openai>=1.99.5 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n", + "Requirement already satisfied: pydantic<3.0.0,>=2.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (2.11.10)\n", + "Requirement already satisfied: python-dotenv>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.1.1)\n", + "Requirement already satisfied: tiktoken>=0.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.12.0)\n", + "Requirement already satisfied: tokenizers in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.22.1)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (2.6.1)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.4.0)\n", + "Requirement already satisfied: attrs>=17.3.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (25.3.0)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.6.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (6.6.3)\n", + "Requirement already satisfied: propcache>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (0.3.1)\n", + "Requirement already satisfied: yarl<2.0,>=1.17.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.20.0)\n", + "Requirement already satisfied: anyio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (4.11.0)\n", + "Requirement already satisfied: certifi in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (2025.10.5)\n", + "Requirement already satisfied: httpcore==1.* in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (1.0.9)\n", + "Requirement already satisfied: idna in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (3.10)\n", + "Requirement already satisfied: h11>=0.16 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpcore==1.*->httpx>=0.23.0->litellm) (0.16.0)\n", + "Requirement already satisfied: zipp>=3.20 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from importlib-metadata>=6.8.0->litellm) (3.21.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jinja2<4.0.0,>=3.1.2->litellm) (3.0.3)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2025.9.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.36.2)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.27.1)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (1.9.0)\n", + "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (0.11.0)\n", + "Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (1.3.1)\n", + "Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (4.67.1)\n", + "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (4.15.0)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.33.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (2.33.2)\n", + "Requirement already satisfied: typing-inspection>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.4.2)\n", + "Requirement already satisfied: regex>=2022.1.18 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tiktoken>=0.7.0->litellm) (2025.9.18)\n", + "Requirement already satisfied: requests>=2.26.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tiktoken>=0.7.0->litellm) (2.32.2)\n", + "Requirement already satisfied: huggingface-hub<2.0,>=0.16.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tokenizers->litellm) (0.25.2)\n", + "Requirement already satisfied: filelock in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (3.15.4)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (2025.9.0)\n", + "Requirement already satisfied: packaging>=20.9 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (25.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (6.0.3)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken>=0.7.0->litellm) (3.4.0)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken>=0.7.0->litellm) (1.26.20)\n" + ] + } + ], + "source": [ + "!pip install litellm" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Completion" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "p-MQqWOT_1a7" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ['COMETAPI_KEY'] = \"Your_CometAPI_Key_Here\"\n", + "api_key = os.getenv('COMETAPI_KEY')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Ze8JqMqWAARO", + "outputId": "64f3e836-69fa-4f8e-fb35-088a913bbe98" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "ModelResponse(id='msg_017L3DDDit8AkEgHRe2DQBc9', created=1760589916, model='claude-sonnet-4-5-20250929', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='I\\'ll create a simple Python script that says hi.\\n\\n\\nhello.py\\n#!/usr/bin/env python3\\n\"\"\"\\nA simple script that says hi!\\n\"\"\"\\n\\ndef say_hi(name=None):\\n \"\"\"Say hi to someone, or just say hi generally.\"\"\"\\n if name:\\n print(f\"Hi, {name}!\")\\n else:\\n print(\"Hi!\")\\n\\nif __name__ == \"__main__\":\\n # Say hi generally\\n say_hi()\\n \\n # Say hi to someone specific\\n say_hi(\"World\")\\n\\n\\n\\nI\\'ve created a simple Python script called `hello.py` that:\\n\\n1. Defines a `say_hi()` function that can optionally take a name parameter\\n2. Prints \"Hi!\" if no name is provided\\n3. Prints \"Hi, [name]!\" if a name is provided\\n4. Demonstrates both usages when run\\n\\nYou can run it with:\\n```bash\\npython hello.py\\n```\\n\\nThis will output:\\n```\\nHi!\\nHi, World!\\n```\\n\\nWould you like me to create versions in other programming languages, or modify this in any way?', role='assistant', tool_calls=None, function_call=None, provider_specific_fields=None), provider_specific_fields={})], usage=Usage(completion_tokens=290, prompt_tokens=26, total_tokens=316, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=None, text_tokens=None, image_tokens=None, cached_tokens_details={})))" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from litellm import completion\n", + "response = completion(\n", + " model=\"cometapi/claude-sonnet-4-5-20250929\",\n", + " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", + ")\n", + "response" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "-LnhELrnAM_J", + "outputId": "d51c7ab7-d761-4bd1-f849-1534d9df4cd0" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "ModelResponse(id='chatcmpl-CRA9Uo6nsQ9C7kMJv1J4kyNDFJym7', created=1760589916, model='gpt-5-chat-latest', object='chat.completion', system_fingerprint='fp_2da73a467a', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Sure! I can help you write a simple code that prints out \"Hi\" in different programming languages. \\n\\nHere’s an example in **Python**:\\n\\n```python\\n# Simple Python program to say \"Hi\"\\nprint(\"Hi\")\\n```\\n\\nExample in **JavaScript**:\\n\\n```javascript\\n// Simple JavaScript program to say \"Hi\"\\nconsole.log(\"Hi\");\\n```\\n\\nExample in **C**:\\n\\n```c\\n#include \\n\\nint main() {\\n printf(\"Hi\\\\n\");\\n return 0;\\n}\\n```\\n\\nExample in **Java**:\\n\\n```java\\npublic class SayHi {\\n public static void main(String[] args) {\\n System.out.println(\"Hi\");\\n }\\n}\\n```\\n\\nWhich language would you like me to focus on, or do you want me to make it interactive so the program greets the user by name?', role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}, annotations=[]), provider_specific_fields={})], usage=Usage(completion_tokens=174, prompt_tokens=12, total_tokens=186, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=0, cached_tokens=0, text_tokens=None, image_tokens=None)))" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response = completion(\n", + " model=\"cometapi/gpt-5-chat-latest\",\n", + " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", + ")\n", + "response" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "dJBOUYdwCEn1", + "outputId": "ffa18679-ec15-4dad-fe2b-68665cdf36b0" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "ModelResponse(id='02176058998406949c23b3bf3d52941de23f13565a086747738f0', created=1760589991, model='deepseek-v3.2-exp', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='Here are several ways to say \"hi\" in different programming languages:\\n\\n## Python\\n```python\\nprint(\"Hi!\")\\n```\\n\\n## JavaScript (Browser)\\n```javascript\\nconsole.log(\"Hi!\");\\n// or\\nalert(\"Hi!\");\\n```\\n\\n## JavaScript (Node.js)\\n```javascript\\nconsole.log(\"Hi!\");\\n```\\n\\n## Java\\n```java\\npublic class Hello {\\n public static void main(String[] args) {\\n System.out.println(\"Hi!\");\\n }\\n}\\n```\\n\\n## C\\n```c\\n#include \\n\\nint main() {\\n printf(\"Hi!\\\\n\");\\n return 0;\\n}\\n```\\n\\n## C++\\n```cpp\\n#include \\n\\nint main() {\\n std::cout << \"Hi!\" << std::endl;\\n return 0;\\n}\\n```\\n\\n## C#\\n```csharp\\nusing System;\\n\\nclass Program {\\n static void Main() {\\n Console.WriteLine(\"Hi!\");\\n }\\n}\\n```\\n\\n## PHP\\n```php\\n\\n```\\n\\n## Ruby\\n```ruby\\nputs \"Hi!\"\\n```\\n\\n## Go\\n```go\\npackage main\\n\\nimport \"fmt\"\\n\\nfunc main() {\\n fmt.Println(\"Hi!\")\\n}\\n```\\n\\n## Rust\\n```rust\\nfn main() {\\n println!(\"Hi!\");\\n}\\n```\\n\\n## Swift\\n```swift\\nprint(\"Hi!\")\\n```\\n\\n## Kotlin\\n```kotlin\\nfun main() {\\n println(\"Hi!\")\\n}\\n```\\n\\n## HTML (webpage)\\n```html\\n\\n\\n\\n Hi Page\\n\\n\\n

Hi!

\\n\\n\\n```\\n\\nThe Python version is probably the simplest if you\\'re just getting started!', role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}), provider_specific_fields={})], usage=Usage(completion_tokens=347, prompt_tokens=10, total_tokens=357, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=None, audio_tokens=None, reasoning_tokens=0, rejected_prediction_tokens=None, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None)), service_tier='default')" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response = completion(\n", + " model=\"cometapi/deepseek-v3.2-exp\",\n", + " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", + ")\n", + "response" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Streaming" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Streaming Responses" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "I'm\n", + " doing\n", + " well\n", + " —\n", + " thanks\n", + " for\n", + " asking\n", + "!\n", + " How\n", + " can\n", + " I\n", + " help\n", + " you\n", + " today\n", + "?\n", + "\n" + ] + } + ], + "source": [ + "messages = [{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}]\n", + "response = completion(model=\"cometapi/gpt-5-mini\", messages=messages, stream=True)\n", + "for part in response:\n", + " print(part.choices[0].delta.content or \"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Async Completion" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ModelResponse(id='chatcmpl-CRAAkmfczlmCEnCM55D9CKexbRenn', created=1760589994, model='gpt-5-mini-2025-08-07', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"I'm doing well, thanks — how are you? How can I help today?\", role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}, annotations=[]), provider_specific_fields={'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'protected_material_code': {'filtered': False, 'detected': False}, 'protected_material_text': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}})], usage=Usage(completion_tokens=26, prompt_tokens=12, total_tokens=38, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=0, cached_tokens=0, text_tokens=None, image_tokens=None)), prompt_filter_results=[{'prompt_index': 0, 'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'jailbreak': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}}])\n" + ] + } + ], + "source": [ + "from litellm import acompletion\n", + "import asyncio\n", + "\n", + "async def test_get_response():\n", + " user_message = \"Hello, how are you?\"\n", + " messages = [{\"content\": user_message, \"role\": \"user\"}]\n", + " response = await acompletion(model=\"cometapi/gpt-5-mini\", messages=messages)\n", + " return response\n", + "\n", + "response = await test_get_response()\n", + "print(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Async Streaming" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "test acompletion + streaming\n", + "response: \n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='Hi', role='assistant', function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' —', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' I', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='’m', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' doing', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' well', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=',', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' thanks', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='!', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' How', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' are', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' you', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='?', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' What', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' can', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' I', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' help', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' you', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' with', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' today', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='?', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", + "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason='stop', index=0, delta=Delta(provider_specific_fields=None, content=None, role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None)\n" + ] + } + ], + "source": [ + "from litellm import acompletion\n", + "import asyncio, os, traceback\n", + "\n", + "async def completion_call():\n", + " try:\n", + " print(\"test acompletion + streaming\")\n", + " response = await acompletion(\n", + " model=\"cometapi/gpt-5-mini\", \n", + " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}], \n", + " stream=True\n", + " )\n", + " print(f\"response: {response}\")\n", + " async for chunk in response:\n", + " print(chunk)\n", + " except:\n", + " print(f\"error occurred: {traceback.format_exc()}\")\n", + " pass\n", + "\n", + "await completion_call()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Embedding" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "EmbeddingResponse(model='text-embedding-3-small', data=[{'object': 'embedding', 'index': 0, 'embedding': [-0.018048199, 0.0047550877, -0.013976435, -0.021936804, -0.038773336, -0.03708264, 0.03854791, -0.0007172257, 0.026473511, -0.0027438616, -0.019823432, -0.011947598, -0.013426959, -0.0059914105, 0.020485623, 0.04269012, -0.028276922, -0.015216281, -0.03325039, 0.045057096, 0.0037477135, 0.015793936, -0.005188329, -0.0071713766, 0.008446445, 0.0070938864, 0.0027632343, 0.025656339, 0.022091785, -0.026797561, -0.029840818, -0.0542714, 0.017907308, -0.03454659, -0.014582269, 0.0429719, 0.03575826, 0.007939235, -0.010123054, -0.029587213, 0.018921727, 0.022556728, 0.019598005, 0.008819807, 0.01655475, 0.043310042, -0.034321167, 0.004441604, 0.032686826, 0.047226828, -0.0043253684, 0.006681779, 0.008995921, 0.06593721, 0.0066078105, -0.023190739, -0.00777721, 0.049875587, -0.028023317, -0.019386668, -0.0013252605, 0.009214303, -0.0055828253, -0.007432026, -0.01199691, 0.0054384116, -0.024247425, 0.047255006, -0.013743965, 0.012799991, 0.009883538, -0.005579303, -0.012616833, -0.014709071, -0.004473305, -0.022204498, 0.010454148, -0.02202134, 0.034631126, 0.0097567355, 0.041478455, -0.010750021, -0.0005340668, -0.041450277, -0.04516981, 0.014709071, -0.06142869, 0.021640932, -0.0008752884, -0.0012671428, -0.045254346, -0.004216178, -0.028220564, -0.011074071, 0.010693664, -0.0029481545, -0.039590508, -0.030178957, -0.019062618, -0.03539194, 0.019978413, 0.019161243, -0.025924034, 0.0058329077, -0.029305428, 0.028840488, 0.02731886, -0.0008048426, -0.056582022, 0.003043256, -0.08459125, -0.012560476, 0.00081100664, 0.0015771041, 0.015188103, -0.0049206354, 0.046888687, -0.027107522, 0.019245777, -0.011412211, 0.039336905, -0.018611765, 0.0429719, -0.04302826, -0.018132735, -0.0074883825, -0.02035882, -0.073038146, -0.042380158, 0.04485985, 0.03361671, -0.033842135, -0.0017259207, -0.0017091898, -0.049199305, -0.024148801, -0.044803493, 0.040943068, -0.03093977, 0.045057096, -0.0186963, -0.014053926, -0.009848315, -0.0070480965, -0.0060054995, -0.02813603, -0.022105874, 0.010235767, -0.004476827, 0.027220234, -0.026290352, 0.0216832, -0.05559578, 0.042464696, 0.0253182, -0.0031594916, 0.02266944, -0.030798879, -0.02896729, -0.0005891025, 0.004061197, -0.042295624, -0.008383043, 0.017146494, -0.025797231, -0.046522368, 0.044127215, 0.021105545, -0.04302826, 0.024430584, -0.014934498, -0.01556851, -0.075743265, -0.022415835, -0.028586883, 0.032743182, 0.03539194, -0.0034483192, -0.04669144, 0.051932603, 0.021626843, 0.03127791, 0.015610777, -0.016470214, -0.0056744046, -0.012743635, 0.060132485, 0.017428277, -0.0039942735, -0.017118316, 0.025106862, 0.008974788, 0.018513141, 0.0016035212, -0.0049593803, 0.0017514573, 0.031221554, -0.034856554, -0.011461522, 0.04621241, 0.044239927, 0.034715664, -0.0121941585, -0.012053267, -0.08526753, -0.011813751, -0.025769053, 0.0125182085, -0.0046670306, 0.038266126, 0.1187997, 0.005787118, -0.030038064, -0.054553185, -0.041506633, -0.024078354, 0.0071854657, -0.013391736, 0.03192601, -0.059625275, 0.0023458432, 0.027924692, 0.09163582, 0.030967949, 0.017639615, 0.01489223, 0.029559033, 0.042943723, 0.0003306547, 0.0047198646, 0.029897174, 0.00012603184, 0.0046811197, -0.0427183, 0.01789322, 0.018175002, -0.0081857955, 0.02581132, 0.009890582, 0.03840702, -0.094453655, -0.01076411, 0.06858598, 0.041647524, 0.033532172, 0.007467249, -0.008235107, -0.030967949, 0.0151317455, 0.027361127, 0.011834885, 0.008707094, 0.008178751, -0.022458103, 0.02844599, 0.003605061, -0.02399382, 0.05212985, 0.06041427, -0.023317542, -0.013335379, -0.044099037, -0.040802173, -0.0047656544, -0.023909286, 0.017315563, 0.017428277, 0.00736158, -0.0016070436, -0.055454887, -0.038012523, -0.020626513, 0.018273626, 0.03260229, -0.016991513, 0.038463376, -0.022458103, -0.0109613575, -0.021810003, 0.04846667, -0.042521052, 0.008601425, -0.019259866, -0.0040048403, -0.03308132, -0.02499415, 0.026783472, -0.032884073, 0.021824092, 0.013145176, -0.009186125, -0.01769597, 0.03240504, -0.015343083, -0.012539342, 0.03578644, -0.012299826, 0.011898286, 0.035730083, 0.058441788, 0.032010544, 0.048720278, 0.012926794, -0.0015207474, -0.03313768, 0.014540002, 0.020189751, 0.00029058868, 0.011531969, -0.022514459, 0.019752987, -0.037956167, 0.005272864, -0.042295624, -0.08521117, -0.03494109, 0.053313337, 0.029981708, -0.008150573, -0.053200625, 0.059681635, -0.035476476, 0.034828376, 0.00087881065, 0.025712697, 0.018668123, 0.03212326, 0.008474623, 0.017836861, 0.004910068, 0.016174342, -0.059681635, 0.04004136, 0.00753065, 0.008150573, -0.038012523, 0.0051178834, 0.012525253, 0.022119964, 0.030235313, 0.008242152, -0.01835816, -0.003150686, 0.010714797, -0.0033162334, -0.028882755, -0.06836055, 0.056159347, 0.013624207, 0.0077349427, 0.0066183778, 0.018583586, -0.008883208, -0.046550546, 0.046945043, -0.07393985, -0.017343743, 0.029530855, -0.010982491, 0.008129438, 0.009700378, -0.024613742, -0.0030097943, 0.0078053884, -0.006438741, 0.04770586, -0.008221018, 0.01654066, 0.02498006, -0.015793936, -0.010827511, 0.02399382, 0.03192601, 0.022923045, -0.029192716, 0.006724046, -0.04601516, 0.038519733, 0.031531516, 0.019443026, 0.000109466084, 0.03525105, -0.027248414, -0.038125236, 0.011771483, -0.007467249, 0.010285079, 0.01670973, -0.007861745, 0.026868006, 0.052327096, 0.026374886, -0.03905512, 0.031193376, -0.053566944, 0.04257741, -0.004670553, 0.0168788, 0.035166513, -0.057878222, 0.07095295, -0.009749691, -0.0137510095, -0.02151413, -0.02431787, 0.010073741, -0.05176353, -0.02083785, 0.003959051, -0.02682574, 0.062104966, -0.011461522, 0.04170388, 0.0076363184, 0.026177637, 0.0144413775, 0.014821785, -0.00046890447, 0.0050544823, 0.00032228927, -0.038970586, -0.011355854, -0.056300238, -0.04302826, -0.003545182, 0.04021043, 0.0051108385, -0.048438493, 0.00252548, -0.07692675, -0.0012433673, 0.0054278444, 0.029305428, -0.016188432, -0.003263399, -0.046156053, 6.031917e-05, 0.060977835, -0.016611107, -0.010637308, -0.012602744, 0.016442036, -0.051509928, -0.016991513, 0.0019407802, 0.019161243, 0.045282524, 0.031869654, -0.036941748, -0.035814617, -0.017850952, -0.027192056, -0.049734693, -0.020964652, 0.0228526, -0.025050506, 0.023472521, 0.025740875, -0.017738238, -0.009813092, -0.030883415, -0.012405495, -0.03277136, -0.029502677, 0.016780175, -0.04421175, -0.0020816717, 0.010341435, 0.059230782, -0.041901127, -0.04119667, 0.025924034, 0.02334572, -0.0008435878, 0.020654691, -0.022753974, 0.010700708, -0.013856677, -0.0121941585, -0.011391076, 0.006590199, 0.0050227814, -0.007960369, 0.0008418266, -0.0198657, 0.10781016, -0.0384352, -0.019147152, 0.0057237167, -0.0038181592, -0.047424074, -0.009341106, 0.018499052, -0.016906979, 0.005642704, -0.01837225, -0.038125236, -0.024895526, -0.010285079, -0.055708494, 0.014173684, -0.019724809, 0.00024215724, 0.04500074, 0.048804812, 0.009777869, -0.006572588, 0.008277375, 0.012328005, -0.012609788, 0.026079014, -0.012990195, 0.017963665, -0.007312268, -0.0015682983, 0.05446865, -0.01258161, 0.00035376972, -0.011299497, -0.036321826, -0.0071854657, 0.012969062, 0.026558045, -0.051819887, -0.0029146927, -0.044606246, -0.010383703, -0.03919601, 0.013624207, 0.0030978515, 0.0121941585, -0.0022225631, 0.0512845, -0.0029780937, -0.025191398, -0.015751667, -0.021006921, 0.0039520063, 0.04418357, 0.020570157, 0.00083390146, 0.020541979, -0.004807922, -0.0114263, -0.036152754, 0.018428607, -0.032658648, -0.002035882, 0.013828499, 0.03144698, -0.0003275727, -0.029756282, 0.008488712, 0.0041879993, 0.027826069, 0.0007273523, -0.018949905, -0.0029023646, 0.007861745, 0.011398122, 0.0125322975, 0.014976765, 0.006318983, -0.0066536004, -0.042915545, 0.025867676, -0.015272637, 0.034602948, 0.050241902, 0.014582269, 0.005987888, 0.015244459, -0.050213724, -0.003212326, 0.01315222, -0.022866689, -0.004772699, 0.035673723, -0.024796901, -0.00699174, -0.002072866, 0.022077696, 0.021147812, 0.005093227, -0.039618686, -0.0049241576, 0.012264604, -0.062104966, -0.0022613083, -0.004339458, 0.065486364, -0.0033320836, 0.02944632, 0.017498722, 0.0033039053, -0.020260196, -0.0154980635, -0.05460954, -0.03626547, 0.0072629564, 0.0028900367, 1.2954037e-05, 0.01769597, -0.0045930627, 0.022260854, 0.0027192058, -0.0010566862, -0.0005212985, 0.012158935, -0.0017312041, -0.035110157, -0.0036032998, -0.02317665, -0.01639977, 0.010327346, 0.018259536, -0.011095204, -0.00061904197, -0.023134382, -0.011989865, 0.0025924034, 0.0056708823, 0.03110884, -0.013462181, -0.021105545, 0.010376658, -0.010017385, -0.025106862, 0.026093103, 0.018456785, -0.02134506, 0.0066993902, 0.011891241, -0.010017385, -0.012687278, -0.017132405, 0.04717047, 0.012475941, -0.018752657, -0.008657781, 0.005276386, -0.02582541, 0.02913636, -0.0193444, -0.01101067, 0.029305428, 0.011736261, 0.043140974, 0.02135915, 0.00089422066, 0.009827181, 0.013638296, 0.013884856, -0.014004614, 0.010285079, 0.008108305, -0.04035132, -0.02978446, 0.008481667, -0.022289034, 0.01621661, -0.0057941624, -0.019090796, -0.01852723, -0.022923045, 0.0077208537, -0.039985005, -0.017428277, -0.009460864, 0.018301804, 0.0014397348, 0.04815671, -0.012187114, 0.018879458, 0.021739556, 0.018414518, -0.013462181, -0.06368295, 0.0057096276, 0.013088819, 0.0061640027, 0.031193376, -0.008728228, -0.019245777, 0.010735931, 0.012454808, 0.0397314, -0.017597347, 0.012278693, 0.0130465515, -0.025473181, -0.03215144, -0.0053292206, -0.0027068777, 0.014068015, -0.028079674, 0.016498392, 0.015159924, -0.009207259, -0.02334572, -0.0013710503, 0.008488712, 0.0012231142, 0.0020464489, -0.025149131, -0.021063277, -0.014427288, -0.035222873, 0.051030897, 0.016103897, 0.0063401167, -0.03093977, -0.004684642, -0.0070199184, -0.008495756, -0.0038674714, -0.012222337, -0.022556728, 0.0036015387, -0.040943068, -0.011362898, 0.016794264, 0.017766416, -0.014194817, 0.0011755633, -0.039759576, 0.011384032, -0.0006318103, 0.008298509, 0.04449353, 0.0004838742, 0.016935157, -0.011341765, 0.016864711, -0.00027892113, 0.0009140335, -0.031306088, -0.049452912, -0.0068367594, -0.00011216283, -0.005079138, -0.014420244, 0.01803411, 0.03984411, -0.026276262, -0.0011077593, -0.00063313113, -0.006301372, -0.019992502, 0.0064316965, -0.024289692, 0.0120039545, 0.0068649375, -0.017724149, -0.015667133, -0.0036490895, -0.007953324, 0.024627833, 0.024402406, 0.021810003, -0.015977094, 0.010524594, -0.0060054995, 0.0414221, -0.048551206, 0.01472316, 0.015427617, 0.0029217373, 0.012666144, 0.0048995013, 0.007326357, -0.04187295, -0.0064176074, -0.00674518, 0.0047762212, -0.053059734, -0.09541172, 0.022063607, 0.029530855, 0.01556851, 0.011292453, -0.0038709936, -0.0055370354, -0.016005272, -0.0035170037, -0.0572583, 0.038632445, 0.007981502, -0.005434889, -0.023895197, 0.0021380284, -0.0015084195, 0.016117986, 0.005434889, -0.014694982, -0.007104453, 0.011595369, -0.055229463, 0.0036455672, 0.0027104, -0.010052607, -0.023697948, -0.016315235, -0.002757951, 0.039505973, 0.011095204, 0.0002681341, 0.058948997, -0.0074883825, 0.0050122146, 0.040604927, 0.012912705, -0.025078684, 0.040464036, -0.008925476, -0.00876345, -0.040633105, -0.009024099, 0.024796901, 0.03592733, 0.03626547, -0.029474499, -0.00055431994, 0.0010839839, 0.016737908, 0.013286067, -0.005441934, 0.0059420983, -0.0121941585, 0.015089478, -0.010186454, -0.03477202, -0.0076363184, -0.0087141385, 0.0018439173, 0.028065585, -0.022331301, 0.0029516767, -0.045789734, 0.0010672531, 0.018287715, -0.015948916, 0.04849485, 0.0057589393, 0.0066219, 0.002196146, -0.047255006, 0.012116668, 0.02085194, 0.025924034, -0.0036737456, -0.02877004, 0.016906979, -0.037336245, -0.016258877, 0.010883868, -0.003765325, -0.0049523357, -0.002613537, -0.03263047, 0.023204828, 0.0049946033, -0.007692675, -0.034236632, 0.034095738, 0.020133393, 0.019259866, -0.014103238, 0.024599653, 0.005889264, 0.02430378, 0.0111233825, -0.018780835, -0.00040550332, 0.020232018, 0.03806888, 0.009890582, 0.032376863, 0.031052483, 0.01871039, 0.03891423, -0.0009739124, 0.002759712, 0.017498722, -0.01158128, -0.0045578396, 0.02744566, 0.06497915, 0.024853257, 0.004709298, 0.016667463, -0.00066263025, -0.018132735, -0.013138131, -0.01124314, -0.0125182085, -0.0038111147, 0.03361671, -0.007270001, 0.0012011, -0.01771006, -0.00039999973, 0.024021998, 0.0027896515, 0.0024744067, 0.0013965869, -0.05939985, 0.0014150789, -0.0052517303, 0.052524347, 0.015779847, -0.03327857, 0.042633764, 0.0059420983, -0.023387987, 0.0039097387, -0.028023317, -0.011863063, 0.004378203, 0.02052789, -0.063626595, -0.014864052, 0.014293442, -0.00015938349, -0.007932191, -0.0010954313, 0.023528878, -0.007467249, 0.0059667546, 0.017132405, 0.005730761, -0.00020495309, -0.032038722, 0.0036631785, 0.042915545, -0.029925352, 0.015667133, 0.018935816, -0.0072065997, 0.01556851, -0.025473181, 0.017625526, -0.0026698937, -0.007446115, -0.008622559, -0.043422755, -0.020133393, -0.0039801844, 0.01489223, -0.021655021, 0.015357172, -0.03640636, -0.005663838, -0.028530527, 0.0022648307, -0.00043015933, 0.043591827, -0.015526242, 0.011870108, -0.02530411, -0.016315235, -0.00032316984, -0.030150779, -0.0052552526, 0.020372909, 0.0075024716, 0.0104330145, -0.00055608107, -0.026248084, -0.015202192, -0.03341946, 0.031559695, -0.0012046222, 0.07185466, -0.039590508, 0.022979401, 0.05810365, 0.014025748, -0.029756282, -0.022866689, 0.0073897582, 0.037618026, -0.004180955, -0.0051566283, 0.009728557, -0.03604004, 0.040633105, 0.0026963109, -0.0054172776, 0.034095738, -0.00595971, 0.040943068, -0.031390622, 0.055962097, 0.02117599, -0.012912705, -0.019626183, 0.055877563, 0.017343743, -0.0035416598, 0.013257889, -0.0186963, 0.01656884, -0.06396473, -0.0055405577, 0.020767406, -0.0046564634, 0.045085277, -0.009221348, 0.013645341, 0.008777539, 0.004730432, -0.018625854, -0.011067026, 0.021500042, -0.015047211, 0.004600107, -0.0014344514, -0.0023740216, -0.016188432, 0.006209792, 0.0011993388, 0.004180955, -0.017160583, 0.014497734, 0.015371261, 0.018259536, -0.028333278, -0.008390088, 0.041929305, 0.003923828, 0.02550136, -0.003300383, -0.008058993, -0.010418925, 0.058216363, 0.01885128, -0.02020384, 0.002858336, -0.009806047, -0.022274945, 0.0070445742, 0.026670758, 0.008213974, -0.035307407, -0.027713355, 0.042915545, -0.039675042, -0.0029217373, 0.012053267, -0.003853382, 0.01133472, -0.010073741, 0.005878697, 0.0070938864, -0.035673723, 0.024205158, 0.005896309, 0.030573452, 0.02416289, -0.0072911344, 0.01738601, 0.017005602, -0.02846008, 0.0030344503, 0.018794924, -0.0148076955, -0.0344057, 0.025430914, 0.033503994, -0.0050580045, 0.0077138087, 0.03243322, 0.01372283, -0.005441934, 0.0073404466, -0.0007832686, -0.04767768, 0.0070480965, 0.015145835, 0.026233995, -0.01670973, -0.019513471, -0.014849963, 0.007953324, -0.0032176094, 0.006572588, -0.0012477703, 0.004230267, 0.004476827, -0.021810003, -0.030009886, -0.019273955, -0.0030414949, -0.002918215, 0.060639694, 0.024641922, 0.010327346, 0.026558045, 0.018921727, -0.025867676, -0.016117986, 0.023881108, 0.025360467, 0.009770825, 0.03792799, -0.022429924, 0.033363104, -0.0018914682, 0.04040768, 0.018484963, 0.0070199184, -0.017583257, 0.016258877, 0.010954313, -0.008939565, -0.024148801, -0.02498006, -0.007889924, 0.02748793, 0.0307707, 0.029756282, 0.0051425393, 0.0045719286, -0.03046074, 0.013596028, 0.025684519, -0.0033197557, 0.006967084, 0.03677268, 0.0120039545, -0.0032792494, -0.0032211316, -0.02399382, -0.026924362, -0.013920079, -0.0042197, 0.025346378, -0.0027015943, -0.016991513, 0.0031594916, -0.007579962, 0.018978084, 0.017681882, 0.0126591, 0.028939111, 0.008833896, 0.10183637, 0.0059632324, -0.05196078, -0.023697948, 0.011045893, -0.008777539, -0.013807366, 0.019273955, -0.025346378, 0.0074742935, 0.009961028, -0.010813422, 0.018597675, 0.009636978, 0.014948587, 0.024064265, -0.008693005, -0.020570157, 0.014194817, -0.026219906, -0.02299349, 0.011067026, 0.032066904, 0.013391736, -0.05148175, -0.009489042, -0.03062981, -0.0012847543, 0.07286908, 0.026529867, -0.00025008238, 0.013638296, 0.016089808, 0.018654034, -0.0020394044, -0.024543297, 0.0147795165, -0.009601755, 0.0018791402, -0.040520392, -0.003360262, 0.02216223, 0.0137650985, 0.0059914105, 0.0048361, -0.0009844792, 0.016977424, -0.00934815, 0.024233336, -0.013088819, -0.017555078, -0.0050263037, 0.010595039, -0.027516108, 0.0071537653, -0.023247095, -0.0017655464, -0.015948916, 0.058160007, -0.025966302, 0.0121941585, -0.012384362, -0.0015612538, 0.009946939, 0.00628376, 0.011327676, 0.0109613575, 0.008601425, -0.018329982, 0.055680316, -0.012778858, -0.0100807855, -0.011067026, -0.0036490895, -0.01356785, 0.0073193125, -0.014272308, -0.027403394, -0.030742522, 0.02862915, 0.03062981, -0.014596358, 0.021697288, 0.0042408337, 0.027572464, 0.0019601528, -0.037138995, -0.031306088, 0.041929305, 0.017738238, 0.004857234, 0.008256241, 0.0118278405, 0.021753646, -0.00160176, -0.0018333505, -0.0047374764, 0.042239267, 0.0058329077, -0.026459422, 0.015075389, 0.021147812, -0.005212985, 0.01281408, 0.017738238, 0.008242152, -0.020372909, -0.011081115, -0.011017715, 0.007706764, 0.01834407, 0.01954165, 0.037477136, -0.010278034, 0.015808025, 0.00031590514, -0.017681882, -0.008967743, -0.020612424, -0.025416825, -0.0037970257, -0.029868996, 0.01720285, -0.0144554665, 0.026727116, 0.00414221, 0.0040154075, 0.05838543, 0.0005622451, -0.025219576, 0.004180955, -0.002932304, -0.0090663675, 0.011574236, 0.02450103, -0.012553431, -0.020612424, -0.032095082, 0.015526242, 0.008974788, 0.0053151315, -0.0003112821, -0.017935487, -0.0076222294, 0.03358853, 0.029474499, -0.011496745, -0.012835215, -0.020739228, -0.012482986, -0.037871633, 0.0052517303, -0.012926794, -0.0025237189, 0.0020323596, 0.045113456, -0.04835396, -0.027755624, -0.0079955915, 0.007896968, 0.0072559114, 0.015047211, -0.0014573464, -0.014032792, 0.021091456, -0.0046071517, -0.0065232757, -0.02582541, -0.035870973, -0.015343083, 0.03254593, -0.028431902, -0.003286294, 0.014328664, 0.008840941, 0.015948916, 0.012835215, 0.019400757, -0.012342094, -0.010693664, 0.004772699, -0.03254593, 0.010707753, -0.016822444, -0.0032827717, 0.021246437, -0.04485985, -0.04384543, -0.015906649, -0.009707424, 0.02299349, 0.019513471, -0.010151232, 0.018963994, -0.0057976847, 0.05739919, -0.019922055, -0.029108182, -0.0106232185, 0.021077367, 0.0036455672, -0.026614401, 0.04497256, -0.04446535, -0.0004556959, -0.004578973, 0.003962573, -0.004910068, 0.015089478, -0.0301226, 0.007664497, 0.008375999, 0.031982366, 0.006135824, 0.02152822, -0.015469885, -0.007210122, 0.034715664, -0.01233505, 0.0004490916, -0.0144413775, -0.003150686, -0.02003477, -0.027924692, -0.0015850292, -0.009376328, -0.0035997776, -0.03240504, -0.010912046, 0.0031999978, 0.022303123, -0.008988877, 0.00024633997, -0.0035698381, 0.0070974086, -0.002599448, -0.042267445, -0.016935157, -0.0002481011, -0.041393917, 0.014483645, 0.019006262, -0.02813603, 0.0072030774, -7.3032425e-05, 0.01802002, -0.017188761, 0.015991183, 0.020401087, 0.03542012, 0.04469078, 0.04071764, 0.011095204, -0.031390622, -0.03254593, 0.014187773, 0.016272966, -0.009721513, -0.026388975, -0.014849963, -0.005642704, -0.022556728, 0.0064457855, -0.043450933, 0.010834555, -0.015977094, 0.020880118, -0.02385293, -0.054806788, 0.03789981, 0.0013516777, -0.026431242, -0.015540331, 0.016695641, -0.037167173, -0.021190079, 0.023881108, -0.0045860177, 0.0064105624, -0.007763121, -0.013053596, 0.024472851, -0.0004962022, -0.00976378, 0.060019772, -0.0057624616, -0.04384543, 0.010313257, 0.0076715415, 0.0025888812, -0.03589915, 0.008791628, -0.012785902, 0.01042597, 0.015653044, 0.04767768, -0.009869449, 0.0064457855, -0.010947268, -0.0077349427, -0.032715004, -0.023867019, -0.011327676, -0.00046274049, -0.036998104, 0.013913034, 0.012250515, -0.009996251, 0.021204168, 0.020091126, -0.003740669, -0.0049769916, -0.0140891485, 0.024064265, 0.0038815604, 0.025684519, 0.041788414, -0.013553761, 0.006681779, -0.0050826604, -0.018175002, 0.008228063, -0.006230926, -0.018907638, 0.0154839745, -0.028713685, -0.015047211, -0.019682541, 0.02516322, 0.040802173, 0.007213644, 0.011743305, -0.015963005, -0.03818159, 0.01191942, -0.031728763, -0.011863063, 0.023881108, 0.0053116092, -0.020992832, -0.017991843, -0.00405063, -0.017780505, -0.0057659843, 0.02978446, 0.031165197, 0.0014221234, 0.021316882, 0.026008569, -0.0018544842, -0.032658648, 0.028474169, 0.013109953, 0.018076377, 0.0007991189, -0.0042373114, 0.028910933, -0.0029358263, 0.021866359, 0.024472851, -0.002576553, -0.033532172, 0.01920351, -0.0095665315, -0.03093977, 0.0034817809, 0.018654034, -0.0074038478, 0.021443684, 0.0038604268, -0.02745975, 0.031587873, 0.0061146906, 0.022711707, -0.019795254, -0.016991513, -0.04471896, -0.007875834, -0.0034941088, -0.043789074, 0.021091456, 0.024909616, -0.013194487, -0.0042690123, 0.027896514, -0.018414518, -0.023303451, -0.025797231, -0.009524264]}], object='list', usage=Usage(completion_tokens=0, prompt_tokens=3, total_tokens=3, completion_tokens_details=None, prompt_tokens_details=None))\n" + ] + } + ], + "source": [ + "import litellm\n", + "\n", + "\n", + "async def main():\n", + " response = await litellm.aembedding(\n", + " model=\"cometapi/text-embedding-3-small\", # The model name must include prefix \"openai\" + the model name from ai/ml api\n", + " api_key=api_key, # your aiml api-key\n", + " api_base=\"https://api.cometapi.com/v1\", # 👈 the URL has changed from v2 to v1\n", + " input=\"Your text string\",\n", + " )\n", + " print(response)\n", + "\n", + "await main()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "EmbeddingResponse(model='text-embedding-3-small', data=[{'object': 'embedding', 'index': 0, 'embedding': [-0.018048199, 0.0047550877, -0.013976435, -0.021936804, -0.038773336, -0.03708264, 0.03854791, -0.0007172257, 0.026473511, -0.0027438616, -0.019823432, -0.011947598, -0.013426959, -0.0059914105, 0.020485623, 0.04269012, -0.028276922, -0.015216281, -0.03325039, 0.045057096, 0.0037477135, 0.015793936, -0.005188329, -0.0071713766, 0.008446445, 0.0070938864, 0.0027632343, 0.025656339, 0.022091785, -0.026797561, -0.029840818, -0.0542714, 0.017907308, -0.03454659, -0.014582269, 0.0429719, 0.03575826, 0.007939235, -0.010123054, -0.029587213, 0.018921727, 0.022556728, 0.019598005, 0.008819807, 0.01655475, 0.043310042, -0.034321167, 0.004441604, 0.032686826, 0.047226828, -0.0043253684, 0.006681779, 0.008995921, 0.06593721, 0.0066078105, -0.023190739, -0.00777721, 0.049875587, -0.028023317, -0.019386668, -0.0013252605, 0.009214303, -0.0055828253, -0.007432026, -0.01199691, 0.0054384116, -0.024247425, 0.047255006, -0.013743965, 0.012799991, 0.009883538, -0.005579303, -0.012616833, -0.014709071, -0.004473305, -0.022204498, 0.010454148, -0.02202134, 0.034631126, 0.0097567355, 0.041478455, -0.010750021, -0.0005340668, -0.041450277, -0.04516981, 0.014709071, -0.06142869, 0.021640932, -0.0008752884, -0.0012671428, -0.045254346, -0.004216178, -0.028220564, -0.011074071, 0.010693664, -0.0029481545, -0.039590508, -0.030178957, -0.019062618, -0.03539194, 0.019978413, 0.019161243, -0.025924034, 0.0058329077, -0.029305428, 0.028840488, 0.02731886, -0.0008048426, -0.056582022, 0.003043256, -0.08459125, -0.012560476, 0.00081100664, 0.0015771041, 0.015188103, -0.0049206354, 0.046888687, -0.027107522, 0.019245777, -0.011412211, 0.039336905, -0.018611765, 0.0429719, -0.04302826, -0.018132735, -0.0074883825, -0.02035882, -0.073038146, -0.042380158, 0.04485985, 0.03361671, -0.033842135, -0.0017259207, -0.0017091898, -0.049199305, -0.024148801, -0.044803493, 0.040943068, -0.03093977, 0.045057096, -0.0186963, -0.014053926, -0.009848315, -0.0070480965, -0.0060054995, -0.02813603, -0.022105874, 0.010235767, -0.004476827, 0.027220234, -0.026290352, 0.0216832, -0.05559578, 0.042464696, 0.0253182, -0.0031594916, 0.02266944, -0.030798879, -0.02896729, -0.0005891025, 0.004061197, -0.042295624, -0.008383043, 0.017146494, -0.025797231, -0.046522368, 0.044127215, 0.021105545, -0.04302826, 0.024430584, -0.014934498, -0.01556851, -0.075743265, -0.022415835, -0.028586883, 0.032743182, 0.03539194, -0.0034483192, -0.04669144, 0.051932603, 0.021626843, 0.03127791, 0.015610777, -0.016470214, -0.0056744046, -0.012743635, 0.060132485, 0.017428277, -0.0039942735, -0.017118316, 0.025106862, 0.008974788, 0.018513141, 0.0016035212, -0.0049593803, 0.0017514573, 0.031221554, -0.034856554, -0.011461522, 0.04621241, 0.044239927, 0.034715664, -0.0121941585, -0.012053267, -0.08526753, -0.011813751, -0.025769053, 0.0125182085, -0.0046670306, 0.038266126, 0.1187997, 0.005787118, -0.030038064, -0.054553185, -0.041506633, -0.024078354, 0.0071854657, -0.013391736, 0.03192601, -0.059625275, 0.0023458432, 0.027924692, 0.09163582, 0.030967949, 0.017639615, 0.01489223, 0.029559033, 0.042943723, 0.0003306547, 0.0047198646, 0.029897174, 0.00012603184, 0.0046811197, -0.0427183, 0.01789322, 0.018175002, -0.0081857955, 0.02581132, 0.009890582, 0.03840702, -0.094453655, -0.01076411, 0.06858598, 0.041647524, 0.033532172, 0.007467249, -0.008235107, -0.030967949, 0.0151317455, 0.027361127, 0.011834885, 0.008707094, 0.008178751, -0.022458103, 0.02844599, 0.003605061, -0.02399382, 0.05212985, 0.06041427, -0.023317542, -0.013335379, -0.044099037, -0.040802173, -0.0047656544, -0.023909286, 0.017315563, 0.017428277, 0.00736158, -0.0016070436, -0.055454887, -0.038012523, -0.020626513, 0.018273626, 0.03260229, -0.016991513, 0.038463376, -0.022458103, -0.0109613575, -0.021810003, 0.04846667, -0.042521052, 0.008601425, -0.019259866, -0.0040048403, -0.03308132, -0.02499415, 0.026783472, -0.032884073, 0.021824092, 0.013145176, -0.009186125, -0.01769597, 0.03240504, -0.015343083, -0.012539342, 0.03578644, -0.012299826, 0.011898286, 0.035730083, 0.058441788, 0.032010544, 0.048720278, 0.012926794, -0.0015207474, -0.03313768, 0.014540002, 0.020189751, 0.00029058868, 0.011531969, -0.022514459, 0.019752987, -0.037956167, 0.005272864, -0.042295624, -0.08521117, -0.03494109, 0.053313337, 0.029981708, -0.008150573, -0.053200625, 0.059681635, -0.035476476, 0.034828376, 0.00087881065, 0.025712697, 0.018668123, 0.03212326, 0.008474623, 0.017836861, 0.004910068, 0.016174342, -0.059681635, 0.04004136, 0.00753065, 0.008150573, -0.038012523, 0.0051178834, 0.012525253, 0.022119964, 0.030235313, 0.008242152, -0.01835816, -0.003150686, 0.010714797, -0.0033162334, -0.028882755, -0.06836055, 0.056159347, 0.013624207, 0.0077349427, 0.0066183778, 0.018583586, -0.008883208, -0.046550546, 0.046945043, -0.07393985, -0.017343743, 0.029530855, -0.010982491, 0.008129438, 0.009700378, -0.024613742, -0.0030097943, 0.0078053884, -0.006438741, 0.04770586, -0.008221018, 0.01654066, 0.02498006, -0.015793936, -0.010827511, 0.02399382, 0.03192601, 0.022923045, -0.029192716, 0.006724046, -0.04601516, 0.038519733, 0.031531516, 0.019443026, 0.000109466084, 0.03525105, -0.027248414, -0.038125236, 0.011771483, -0.007467249, 0.010285079, 0.01670973, -0.007861745, 0.026868006, 0.052327096, 0.026374886, -0.03905512, 0.031193376, -0.053566944, 0.04257741, -0.004670553, 0.0168788, 0.035166513, -0.057878222, 0.07095295, -0.009749691, -0.0137510095, -0.02151413, -0.02431787, 0.010073741, -0.05176353, -0.02083785, 0.003959051, -0.02682574, 0.062104966, -0.011461522, 0.04170388, 0.0076363184, 0.026177637, 0.0144413775, 0.014821785, -0.00046890447, 0.0050544823, 0.00032228927, -0.038970586, -0.011355854, -0.056300238, -0.04302826, -0.003545182, 0.04021043, 0.0051108385, -0.048438493, 0.00252548, -0.07692675, -0.0012433673, 0.0054278444, 0.029305428, -0.016188432, -0.003263399, -0.046156053, 6.031917e-05, 0.060977835, -0.016611107, -0.010637308, -0.012602744, 0.016442036, -0.051509928, -0.016991513, 0.0019407802, 0.019161243, 0.045282524, 0.031869654, -0.036941748, -0.035814617, -0.017850952, -0.027192056, -0.049734693, -0.020964652, 0.0228526, -0.025050506, 0.023472521, 0.025740875, -0.017738238, -0.009813092, -0.030883415, -0.012405495, -0.03277136, -0.029502677, 0.016780175, -0.04421175, -0.0020816717, 0.010341435, 0.059230782, -0.041901127, -0.04119667, 0.025924034, 0.02334572, -0.0008435878, 0.020654691, -0.022753974, 0.010700708, -0.013856677, -0.0121941585, -0.011391076, 0.006590199, 0.0050227814, -0.007960369, 0.0008418266, -0.0198657, 0.10781016, -0.0384352, -0.019147152, 0.0057237167, -0.0038181592, -0.047424074, -0.009341106, 0.018499052, -0.016906979, 0.005642704, -0.01837225, -0.038125236, -0.024895526, -0.010285079, -0.055708494, 0.014173684, -0.019724809, 0.00024215724, 0.04500074, 0.048804812, 0.009777869, -0.006572588, 0.008277375, 0.012328005, -0.012609788, 0.026079014, -0.012990195, 0.017963665, -0.007312268, -0.0015682983, 0.05446865, -0.01258161, 0.00035376972, -0.011299497, -0.036321826, -0.0071854657, 0.012969062, 0.026558045, -0.051819887, -0.0029146927, -0.044606246, -0.010383703, -0.03919601, 0.013624207, 0.0030978515, 0.0121941585, -0.0022225631, 0.0512845, -0.0029780937, -0.025191398, -0.015751667, -0.021006921, 0.0039520063, 0.04418357, 0.020570157, 0.00083390146, 0.020541979, -0.004807922, -0.0114263, -0.036152754, 0.018428607, -0.032658648, -0.002035882, 0.013828499, 0.03144698, -0.0003275727, -0.029756282, 0.008488712, 0.0041879993, 0.027826069, 0.0007273523, -0.018949905, -0.0029023646, 0.007861745, 0.011398122, 0.0125322975, 0.014976765, 0.006318983, -0.0066536004, -0.042915545, 0.025867676, -0.015272637, 0.034602948, 0.050241902, 0.014582269, 0.005987888, 0.015244459, -0.050213724, -0.003212326, 0.01315222, -0.022866689, -0.004772699, 0.035673723, -0.024796901, -0.00699174, -0.002072866, 0.022077696, 0.021147812, 0.005093227, -0.039618686, -0.0049241576, 0.012264604, -0.062104966, -0.0022613083, -0.004339458, 0.065486364, -0.0033320836, 0.02944632, 0.017498722, 0.0033039053, -0.020260196, -0.0154980635, -0.05460954, -0.03626547, 0.0072629564, 0.0028900367, 1.2954037e-05, 0.01769597, -0.0045930627, 0.022260854, 0.0027192058, -0.0010566862, -0.0005212985, 0.012158935, -0.0017312041, -0.035110157, -0.0036032998, -0.02317665, -0.01639977, 0.010327346, 0.018259536, -0.011095204, -0.00061904197, -0.023134382, -0.011989865, 0.0025924034, 0.0056708823, 0.03110884, -0.013462181, -0.021105545, 0.010376658, -0.010017385, -0.025106862, 0.026093103, 0.018456785, -0.02134506, 0.0066993902, 0.011891241, -0.010017385, -0.012687278, -0.017132405, 0.04717047, 0.012475941, -0.018752657, -0.008657781, 0.005276386, -0.02582541, 0.02913636, -0.0193444, -0.01101067, 0.029305428, 0.011736261, 0.043140974, 0.02135915, 0.00089422066, 0.009827181, 0.013638296, 0.013884856, -0.014004614, 0.010285079, 0.008108305, -0.04035132, -0.02978446, 0.008481667, -0.022289034, 0.01621661, -0.0057941624, -0.019090796, -0.01852723, -0.022923045, 0.0077208537, -0.039985005, -0.017428277, -0.009460864, 0.018301804, 0.0014397348, 0.04815671, -0.012187114, 0.018879458, 0.021739556, 0.018414518, -0.013462181, -0.06368295, 0.0057096276, 0.013088819, 0.0061640027, 0.031193376, -0.008728228, -0.019245777, 0.010735931, 0.012454808, 0.0397314, -0.017597347, 0.012278693, 0.0130465515, -0.025473181, -0.03215144, -0.0053292206, -0.0027068777, 0.014068015, -0.028079674, 0.016498392, 0.015159924, -0.009207259, -0.02334572, -0.0013710503, 0.008488712, 0.0012231142, 0.0020464489, -0.025149131, -0.021063277, -0.014427288, -0.035222873, 0.051030897, 0.016103897, 0.0063401167, -0.03093977, -0.004684642, -0.0070199184, -0.008495756, -0.0038674714, -0.012222337, -0.022556728, 0.0036015387, -0.040943068, -0.011362898, 0.016794264, 0.017766416, -0.014194817, 0.0011755633, -0.039759576, 0.011384032, -0.0006318103, 0.008298509, 0.04449353, 0.0004838742, 0.016935157, -0.011341765, 0.016864711, -0.00027892113, 0.0009140335, -0.031306088, -0.049452912, -0.0068367594, -0.00011216283, -0.005079138, -0.014420244, 0.01803411, 0.03984411, -0.026276262, -0.0011077593, -0.00063313113, -0.006301372, -0.019992502, 0.0064316965, -0.024289692, 0.0120039545, 0.0068649375, -0.017724149, -0.015667133, -0.0036490895, -0.007953324, 0.024627833, 0.024402406, 0.021810003, -0.015977094, 0.010524594, -0.0060054995, 0.0414221, -0.048551206, 0.01472316, 0.015427617, 0.0029217373, 0.012666144, 0.0048995013, 0.007326357, -0.04187295, -0.0064176074, -0.00674518, 0.0047762212, -0.053059734, -0.09541172, 0.022063607, 0.029530855, 0.01556851, 0.011292453, -0.0038709936, -0.0055370354, -0.016005272, -0.0035170037, -0.0572583, 0.038632445, 0.007981502, -0.005434889, -0.023895197, 0.0021380284, -0.0015084195, 0.016117986, 0.005434889, -0.014694982, -0.007104453, 0.011595369, -0.055229463, 0.0036455672, 0.0027104, -0.010052607, -0.023697948, -0.016315235, -0.002757951, 0.039505973, 0.011095204, 0.0002681341, 0.058948997, -0.0074883825, 0.0050122146, 0.040604927, 0.012912705, -0.025078684, 0.040464036, -0.008925476, -0.00876345, -0.040633105, -0.009024099, 0.024796901, 0.03592733, 0.03626547, -0.029474499, -0.00055431994, 0.0010839839, 0.016737908, 0.013286067, -0.005441934, 0.0059420983, -0.0121941585, 0.015089478, -0.010186454, -0.03477202, -0.0076363184, -0.0087141385, 0.0018439173, 0.028065585, -0.022331301, 0.0029516767, -0.045789734, 0.0010672531, 0.018287715, -0.015948916, 0.04849485, 0.0057589393, 0.0066219, 0.002196146, -0.047255006, 0.012116668, 0.02085194, 0.025924034, -0.0036737456, -0.02877004, 0.016906979, -0.037336245, -0.016258877, 0.010883868, -0.003765325, -0.0049523357, -0.002613537, -0.03263047, 0.023204828, 0.0049946033, -0.007692675, -0.034236632, 0.034095738, 0.020133393, 0.019259866, -0.014103238, 0.024599653, 0.005889264, 0.02430378, 0.0111233825, -0.018780835, -0.00040550332, 0.020232018, 0.03806888, 0.009890582, 0.032376863, 0.031052483, 0.01871039, 0.03891423, -0.0009739124, 0.002759712, 0.017498722, -0.01158128, -0.0045578396, 0.02744566, 0.06497915, 0.024853257, 0.004709298, 0.016667463, -0.00066263025, -0.018132735, -0.013138131, -0.01124314, -0.0125182085, -0.0038111147, 0.03361671, -0.007270001, 0.0012011, -0.01771006, -0.00039999973, 0.024021998, 0.0027896515, 0.0024744067, 0.0013965869, -0.05939985, 0.0014150789, -0.0052517303, 0.052524347, 0.015779847, -0.03327857, 0.042633764, 0.0059420983, -0.023387987, 0.0039097387, -0.028023317, -0.011863063, 0.004378203, 0.02052789, -0.063626595, -0.014864052, 0.014293442, -0.00015938349, -0.007932191, -0.0010954313, 0.023528878, -0.007467249, 0.0059667546, 0.017132405, 0.005730761, -0.00020495309, -0.032038722, 0.0036631785, 0.042915545, -0.029925352, 0.015667133, 0.018935816, -0.0072065997, 0.01556851, -0.025473181, 0.017625526, -0.0026698937, -0.007446115, -0.008622559, -0.043422755, -0.020133393, -0.0039801844, 0.01489223, -0.021655021, 0.015357172, -0.03640636, -0.005663838, -0.028530527, 0.0022648307, -0.00043015933, 0.043591827, -0.015526242, 0.011870108, -0.02530411, -0.016315235, -0.00032316984, -0.030150779, -0.0052552526, 0.020372909, 0.0075024716, 0.0104330145, -0.00055608107, -0.026248084, -0.015202192, -0.03341946, 0.031559695, -0.0012046222, 0.07185466, -0.039590508, 0.022979401, 0.05810365, 0.014025748, -0.029756282, -0.022866689, 0.0073897582, 0.037618026, -0.004180955, -0.0051566283, 0.009728557, -0.03604004, 0.040633105, 0.0026963109, -0.0054172776, 0.034095738, -0.00595971, 0.040943068, -0.031390622, 0.055962097, 0.02117599, -0.012912705, -0.019626183, 0.055877563, 0.017343743, -0.0035416598, 0.013257889, -0.0186963, 0.01656884, -0.06396473, -0.0055405577, 0.020767406, -0.0046564634, 0.045085277, -0.009221348, 0.013645341, 0.008777539, 0.004730432, -0.018625854, -0.011067026, 0.021500042, -0.015047211, 0.004600107, -0.0014344514, -0.0023740216, -0.016188432, 0.006209792, 0.0011993388, 0.004180955, -0.017160583, 0.014497734, 0.015371261, 0.018259536, -0.028333278, -0.008390088, 0.041929305, 0.003923828, 0.02550136, -0.003300383, -0.008058993, -0.010418925, 0.058216363, 0.01885128, -0.02020384, 0.002858336, -0.009806047, -0.022274945, 0.0070445742, 0.026670758, 0.008213974, -0.035307407, -0.027713355, 0.042915545, -0.039675042, -0.0029217373, 0.012053267, -0.003853382, 0.01133472, -0.010073741, 0.005878697, 0.0070938864, -0.035673723, 0.024205158, 0.005896309, 0.030573452, 0.02416289, -0.0072911344, 0.01738601, 0.017005602, -0.02846008, 0.0030344503, 0.018794924, -0.0148076955, -0.0344057, 0.025430914, 0.033503994, -0.0050580045, 0.0077138087, 0.03243322, 0.01372283, -0.005441934, 0.0073404466, -0.0007832686, -0.04767768, 0.0070480965, 0.015145835, 0.026233995, -0.01670973, -0.019513471, -0.014849963, 0.007953324, -0.0032176094, 0.006572588, -0.0012477703, 0.004230267, 0.004476827, -0.021810003, -0.030009886, -0.019273955, -0.0030414949, -0.002918215, 0.060639694, 0.024641922, 0.010327346, 0.026558045, 0.018921727, -0.025867676, -0.016117986, 0.023881108, 0.025360467, 0.009770825, 0.03792799, -0.022429924, 0.033363104, -0.0018914682, 0.04040768, 0.018484963, 0.0070199184, -0.017583257, 0.016258877, 0.010954313, -0.008939565, -0.024148801, -0.02498006, -0.007889924, 0.02748793, 0.0307707, 0.029756282, 0.0051425393, 0.0045719286, -0.03046074, 0.013596028, 0.025684519, -0.0033197557, 0.006967084, 0.03677268, 0.0120039545, -0.0032792494, -0.0032211316, -0.02399382, -0.026924362, -0.013920079, -0.0042197, 0.025346378, -0.0027015943, -0.016991513, 0.0031594916, -0.007579962, 0.018978084, 0.017681882, 0.0126591, 0.028939111, 0.008833896, 0.10183637, 0.0059632324, -0.05196078, -0.023697948, 0.011045893, -0.008777539, -0.013807366, 0.019273955, -0.025346378, 0.0074742935, 0.009961028, -0.010813422, 0.018597675, 0.009636978, 0.014948587, 0.024064265, -0.008693005, -0.020570157, 0.014194817, -0.026219906, -0.02299349, 0.011067026, 0.032066904, 0.013391736, -0.05148175, -0.009489042, -0.03062981, -0.0012847543, 0.07286908, 0.026529867, -0.00025008238, 0.013638296, 0.016089808, 0.018654034, -0.0020394044, -0.024543297, 0.0147795165, -0.009601755, 0.0018791402, -0.040520392, -0.003360262, 0.02216223, 0.0137650985, 0.0059914105, 0.0048361, -0.0009844792, 0.016977424, -0.00934815, 0.024233336, -0.013088819, -0.017555078, -0.0050263037, 0.010595039, -0.027516108, 0.0071537653, -0.023247095, -0.0017655464, -0.015948916, 0.058160007, -0.025966302, 0.0121941585, -0.012384362, -0.0015612538, 0.009946939, 0.00628376, 0.011327676, 0.0109613575, 0.008601425, -0.018329982, 0.055680316, -0.012778858, -0.0100807855, -0.011067026, -0.0036490895, -0.01356785, 0.0073193125, -0.014272308, -0.027403394, -0.030742522, 0.02862915, 0.03062981, -0.014596358, 0.021697288, 0.0042408337, 0.027572464, 0.0019601528, -0.037138995, -0.031306088, 0.041929305, 0.017738238, 0.004857234, 0.008256241, 0.0118278405, 0.021753646, -0.00160176, -0.0018333505, -0.0047374764, 0.042239267, 0.0058329077, -0.026459422, 0.015075389, 0.021147812, -0.005212985, 0.01281408, 0.017738238, 0.008242152, -0.020372909, -0.011081115, -0.011017715, 0.007706764, 0.01834407, 0.01954165, 0.037477136, -0.010278034, 0.015808025, 0.00031590514, -0.017681882, -0.008967743, -0.020612424, -0.025416825, -0.0037970257, -0.029868996, 0.01720285, -0.0144554665, 0.026727116, 0.00414221, 0.0040154075, 0.05838543, 0.0005622451, -0.025219576, 0.004180955, -0.002932304, -0.0090663675, 0.011574236, 0.02450103, -0.012553431, -0.020612424, -0.032095082, 0.015526242, 0.008974788, 0.0053151315, -0.0003112821, -0.017935487, -0.0076222294, 0.03358853, 0.029474499, -0.011496745, -0.012835215, -0.020739228, -0.012482986, -0.037871633, 0.0052517303, -0.012926794, -0.0025237189, 0.0020323596, 0.045113456, -0.04835396, -0.027755624, -0.0079955915, 0.007896968, 0.0072559114, 0.015047211, -0.0014573464, -0.014032792, 0.021091456, -0.0046071517, -0.0065232757, -0.02582541, -0.035870973, -0.015343083, 0.03254593, -0.028431902, -0.003286294, 0.014328664, 0.008840941, 0.015948916, 0.012835215, 0.019400757, -0.012342094, -0.010693664, 0.004772699, -0.03254593, 0.010707753, -0.016822444, -0.0032827717, 0.021246437, -0.04485985, -0.04384543, -0.015906649, -0.009707424, 0.02299349, 0.019513471, -0.010151232, 0.018963994, -0.0057976847, 0.05739919, -0.019922055, -0.029108182, -0.0106232185, 0.021077367, 0.0036455672, -0.026614401, 0.04497256, -0.04446535, -0.0004556959, -0.004578973, 0.003962573, -0.004910068, 0.015089478, -0.0301226, 0.007664497, 0.008375999, 0.031982366, 0.006135824, 0.02152822, -0.015469885, -0.007210122, 0.034715664, -0.01233505, 0.0004490916, -0.0144413775, -0.003150686, -0.02003477, -0.027924692, -0.0015850292, -0.009376328, -0.0035997776, -0.03240504, -0.010912046, 0.0031999978, 0.022303123, -0.008988877, 0.00024633997, -0.0035698381, 0.0070974086, -0.002599448, -0.042267445, -0.016935157, -0.0002481011, -0.041393917, 0.014483645, 0.019006262, -0.02813603, 0.0072030774, -7.3032425e-05, 0.01802002, -0.017188761, 0.015991183, 0.020401087, 0.03542012, 0.04469078, 0.04071764, 0.011095204, -0.031390622, -0.03254593, 0.014187773, 0.016272966, -0.009721513, -0.026388975, -0.014849963, -0.005642704, -0.022556728, 0.0064457855, -0.043450933, 0.010834555, -0.015977094, 0.020880118, -0.02385293, -0.054806788, 0.03789981, 0.0013516777, -0.026431242, -0.015540331, 0.016695641, -0.037167173, -0.021190079, 0.023881108, -0.0045860177, 0.0064105624, -0.007763121, -0.013053596, 0.024472851, -0.0004962022, -0.00976378, 0.060019772, -0.0057624616, -0.04384543, 0.010313257, 0.0076715415, 0.0025888812, -0.03589915, 0.008791628, -0.012785902, 0.01042597, 0.015653044, 0.04767768, -0.009869449, 0.0064457855, -0.010947268, -0.0077349427, -0.032715004, -0.023867019, -0.011327676, -0.00046274049, -0.036998104, 0.013913034, 0.012250515, -0.009996251, 0.021204168, 0.020091126, -0.003740669, -0.0049769916, -0.0140891485, 0.024064265, 0.0038815604, 0.025684519, 0.041788414, -0.013553761, 0.006681779, -0.0050826604, -0.018175002, 0.008228063, -0.006230926, -0.018907638, 0.0154839745, -0.028713685, -0.015047211, -0.019682541, 0.02516322, 0.040802173, 0.007213644, 0.011743305, -0.015963005, -0.03818159, 0.01191942, -0.031728763, -0.011863063, 0.023881108, 0.0053116092, -0.020992832, -0.017991843, -0.00405063, -0.017780505, -0.0057659843, 0.02978446, 0.031165197, 0.0014221234, 0.021316882, 0.026008569, -0.0018544842, -0.032658648, 0.028474169, 0.013109953, 0.018076377, 0.0007991189, -0.0042373114, 0.028910933, -0.0029358263, 0.021866359, 0.024472851, -0.002576553, -0.033532172, 0.01920351, -0.0095665315, -0.03093977, 0.0034817809, 0.018654034, -0.0074038478, 0.021443684, 0.0038604268, -0.02745975, 0.031587873, 0.0061146906, 0.022711707, -0.019795254, -0.016991513, -0.04471896, -0.007875834, -0.0034941088, -0.043789074, 0.021091456, 0.024909616, -0.013194487, -0.0042690123, 0.027896514, -0.018414518, -0.023303451, -0.025797231, -0.009524264]}], object='list', usage=Usage(completion_tokens=0, prompt_tokens=3, total_tokens=3, completion_tokens_details=None, prompt_tokens_details=None))\n" + ] + } + ], + "source": [ + "import litellm\n", + "\n", + "\n", + "async def main():\n", + " response = await litellm.aembedding(\n", + " model=\"cometapi/text-embedding-3-small\", # The model name must include prefix \"cometapi/\" + the model name from CometAPI\n", + " api_key=api_key, # your CometAPI api-key\n", + " api_base=\"https://api.cometapi.com/v1\",\n", + " input=\"Your text string\",\n", + " )\n", + " print(response)\n", + "\n", + "\n", + "await main()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Async Image Generation" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ImageResponse(created=1760591151, background=None, data=[ImageObject(b64_json=None, revised_prompt=\"Generate an image of an adorable baby sea otter. It should be floating on its back in a calm, clear ocean, playfully grasping a colorful shell in its small paws. The sun is setting in the background, casting a peaceful orange and purple hue across the sky and reflecting upon the ocean waves. The otter's fur is a deep, rich brown and appears silky and wet, with glints of sunlight catching on it. Its eyes are bright, expressing joy and curiosity as it examines its newfound treasure.\", url='https://oaidalleapiprodscus.blob.core.windows.net/private/org-OKnsK88id12jfvnKByup1O0l/user-3GxuMyEg9YMU8LFCPHi31prf/img-7PUEF8Wb6thGDAuZWLJjSnfP.png?st=2025-10-16T04%3A05%3A51Z&se=2025-10-16T06%3A05%3A51Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=38e27a3b-6174-4d3e-90ac-d7d9ad49543f&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2025-10-16T02%3A51%3A01Z&ske=2025-10-17T02%3A51%3A01Z&sks=b&skv=2024-08-04&sig=IZKG2VE%2B6VdOe5Tq0Zk/5bVyGK/oK/yO8g%2BDX4krpug%3D')], output_format=None, quality=None, size=None, usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0, completion_tokens_details=None, prompt_tokens_details=None, input_tokens=0, input_tokens_details={'image_tokens': 0, 'text_tokens': 0}, output_tokens=0))\n" + ] + } + ], + "source": [ + "import asyncio\n", + "\n", + "import litellm\n", + "\n", + "\n", + "async def main():\n", + " response = await litellm.aimage_generation(\n", + " model=\"cometapi/dall-e-3\", # The model name must include prefix \"cometapi/\" + the model name from CometAPI\n", + " api_key=api_key, # your cometapi api-key\n", + " api_base=\"https://api.cometapi.com/v1\",\n", + " prompt=\"A cute baby sea otter\",\n", + " )\n", + " print(response)\n", + "\n", + "\n", + "await main()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "base", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.8" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/cookbook/litellm_proxy_server/secret_manager/custom_secret_manager_config.yaml b/cookbook/litellm_proxy_server/secret_manager/custom_secret_manager_config.yaml new file mode 100644 index 00000000000..3598a9b1b65 --- /dev/null +++ b/cookbook/litellm_proxy_server/secret_manager/custom_secret_manager_config.yaml @@ -0,0 +1,20 @@ +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + key_management_system: "custom" + key_management_settings: + custom_secret_manager: my_secret_manager.InMemorySecretManager + store_virtual_keys: true + prefix_for_stored_virtual_keys: "litellm/" + access_mode: "read_and_write" + +model_list: + - model_name: gpt-4 + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY # Read from custom secret manager + + - model_name: claude-3-5-sonnet + litellm_params: + model: anthropic/claude-3-5-sonnet-20241022 + api_key: os.environ/ANTHROPIC_API_KEY # Read from custom secret manager + diff --git a/cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py b/cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py new file mode 100644 index 00000000000..b3c1bf608e2 --- /dev/null +++ b/cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py @@ -0,0 +1,79 @@ +""" +Example custom secret manager for LiteLLM Proxy. + +This is a simple in-memory secret manager for testing purposes. +In production, replace this with your actual secret management system. +""" + +from typing import Optional, Union + +import httpx + +from litellm.integrations.custom_secret_manager import CustomSecretManager + + +class InMemorySecretManager(CustomSecretManager): + def __init__(self): + super().__init__(secret_manager_name="in_memory_secrets") + # Store your secrets in memory + print("INITIALIZING CUSTOM SECRET MANAGER IN MEMORY") + self.secrets = {} + print("CUSTOM SECRET MANAGER IN MEMORY INITIALIZED") + + async def async_read_secret( + self, + secret_name: str, + optional_params: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + ) -> Optional[str]: + """Read secret asynchronously""" + print("READING SECRET ASYNCHRONOUSLY") + print("SECRET NAME: %s", secret_name) + print("SECRET: %s", self.secrets.get(secret_name)) + return self.secrets.get(secret_name) + + def sync_read_secret( + self, + secret_name: str, + optional_params: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + ) -> Optional[str]: + """Read secret synchronously""" + from litellm._logging import verbose_proxy_logger + + verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: LOOKING FOR SECRET: {secret_name}") + value = self.secrets.get(secret_name) + verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: READ SECRET: {value}") + return value + + async def async_write_secret( + self, + secret_name: str, + secret_value: str, + description: Optional[str] = None, + optional_params: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + tags: Optional[Union[dict, list]] = None, + ) -> dict: + """Write a secret to the in-memory store""" + self.secrets[secret_name] = secret_value + print("ALL SECRETS=%s", self.secrets) + return { + "status": "success", + "secret_name": secret_name, + "description": description, + } + + async def async_delete_secret( + self, + secret_name: str, + recovery_window_in_days: Optional[int] = 7, + optional_params: Optional[dict] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, + ) -> dict: + """Delete a secret from the in-memory store""" + if secret_name in self.secrets: + del self.secrets[secret_name] + return {"status": "deleted", "secret_name": secret_name} + return {"status": "not_found", "secret_name": secret_name} + diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml index e361ee226b7..aa81e4efecc 100644 --- a/deploy/charts/litellm-helm/Chart.yaml +++ b/deploy/charts/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.6 +version: 0.4.7 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index 7a6893f28f1..243a4ba7d48 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -27,6 +27,10 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} serviceAccountName: {{ include "litellm.serviceAccountName" . }} containers: - name: prisma-migrations diff --git a/dist/litellm-1.79.1.tar.gz b/dist/litellm-1.79.1.tar.gz new file mode 100644 index 00000000000..5980922c1b5 Binary files /dev/null and b/dist/litellm-1.79.1.tar.gz differ diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 2e886915203..f95f540a7a5 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -57,6 +57,9 @@ USER root # Install only runtime dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ libssl3 \ + libatomic1 \ + nodejs \ + npm \ && rm -rf /var/lib/apt/lists/* WORKDIR /app diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 4178724e6e4..0cbdf761fe8 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -8,16 +8,36 @@ ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev FROM $LITELLM_BUILD_IMAGE AS builder WORKDIR /app -# Install build dependencies +# Install build dependencies including Node.js for UI build USER root -RUN apk add --no-cache build-base bash \ +RUN apk add --no-cache build-base bash nodejs npm \ && pip install --no-cache-dir --upgrade pip build # Copy project files COPY . . +# Set LITELLM_NON_ROOT flag for build time +ENV LITELLM_NON_ROOT=true + # Build Admin UI -RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh +RUN mkdir -p /tmp/litellm_ui && \ + cd ui/litellm-dashboard && \ + if [ -f "../../enterprise/enterprise_ui/enterprise_colors.json" ]; then \ + cp ../../enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \ + fi && \ + npm install && \ + npm run build && \ + cp -r ./out/* /tmp/litellm_ui/ && \ + cd /tmp/litellm_ui && \ + for html_file in *.html; do \ + if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \ + folder_name="${html_file%.html}" && \ + mkdir -p "$folder_name" && \ + mv "$html_file" "$folder_name/index.html"; \ + fi; \ + done && \ + cd /app/ui/litellm-dashboard && \ + rm -rf ./out # Build package and wheel dependencies RUN rm -rf dist/* && python -m build && \ @@ -42,12 +62,17 @@ COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf COPY --from=builder /app/schema.prisma /app/schema.prisma COPY --from=builder /app/dist/*.whl . COPY --from=builder /wheels/ /wheels/ +COPY --from=builder /tmp/litellm_ui /tmp/litellm_ui # Install package from wheel and dependencies RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \ && rm -f *.whl \ && rm -rf /wheels +# Remove test files and keys from dependencies +RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \ + find /usr/lib -type d -path "*/tornado/test" -delete + # Install semantic_router and aurelio-sdk using script RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh @@ -56,7 +81,6 @@ RUN pip uninstall jwt -y && \ pip uninstall PyJWT -y && \ pip install PyJWT==2.9.0 --no-cache-dir -# --- Prisma Handling for Non-Root User --- # Set Prisma cache directories ENV PRISMA_BINARY_CACHE_DIR=/nonexistent ENV NPM_CONFIG_CACHE=/.npm @@ -68,25 +92,20 @@ RUN pip install --no-cache-dir prisma && \ # Create directories and set permissions for non-root user RUN mkdir -p /nonexistent /.npm && \ - chown -R nobody:nogroup /app && \ - chown -R nobody:nogroup /nonexistent /.npm && \ + chown -R nobody:nogroup /app /tmp/litellm_ui /nonexistent /.npm && \ PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \ chown -R nobody:nogroup $PRISMA_PATH && \ LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \ [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH -# --- OpenShift Compatibility: Apply Red Hat recommended pattern --- -# Get paths for directories that need write access at runtime +# OpenShift compatibility RUN PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \ LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \ - # Set group ownership to 0 (root group) for OpenShift compatibility && \ - chgrp -R 0 $PRISMA_PATH && \ + chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui && \ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \ - # Mirror owner permissions to group (g=u) as recommended by Red Hat && \ - chmod -R g=u $PRISMA_PATH && \ + chmod -R g=u $PRISMA_PATH /tmp/litellm_ui && \ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \ - # Ensure directories are writable by group && \ - chmod -R g+w $PRISMA_PATH && \ + chmod -R g+w $PRISMA_PATH /tmp/litellm_ui && \ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true # Switch to non-root user @@ -94,14 +113,14 @@ USER nobody # Set HOME for prisma generate to have a writable directory ENV HOME=/app + +# Set LITELLM_NON_ROOT flag for runtime +ENV LITELLM_NON_ROOT=true + RUN prisma generate -# --- End of Prisma Handling --- EXPOSE 4000/tcp -# Set entrypoint and command ENTRYPOINT ["/app/docker/prod_entrypoint.sh"] -# Append "--detailed_debug" to the end of CMD to view detailed debug logs -# CMD ["--port", "4000", "--detailed_debug"] -CMD ["--port", "4000"] +CMD ["--port", "4000"] \ No newline at end of file diff --git a/docs/my-website/docs/adding_provider/adding_guardrail_support.md b/docs/my-website/docs/adding_provider/adding_guardrail_support.md new file mode 100644 index 00000000000..2646b626ab5 --- /dev/null +++ b/docs/my-website/docs/adding_provider/adding_guardrail_support.md @@ -0,0 +1,412 @@ +# Adding Guardrail Support to Endpoints + +This guide explains how to add guardrail translation support to new LiteLLM endpoints (e.g., Chat Completions, Responses API, etc.). + +## When to Add Guardrail Support + +Add guardrail support when: +- You're creating a new LiteLLM endpoint (e.g., a new API format) +- You want to enable guardrails for an existing endpoint that doesn't support them +- You need custom text extraction logic for a specific message format + +## Directory Structure + +Guardrail handlers follow this structure: + +``` +litellm/llms/{provider}/{endpoint}/guardrail_translation/ +├── __init__.py # Exports handler and registers call types +├── handler.py # Main handler implementation +└── README.md # Documentation (optional but recommended) +``` + +### Example Structures + +**OpenAI Chat Completions:** +``` +litellm/llms/openai/chat/guardrail_translation/ +├── __init__.py +├── handler.py +└── README.md +``` + +**OpenAI Responses API:** +``` +litellm/llms/openai/responses/guardrail_translation/ +├── __init__.py +├── handler.py +└── README.md +``` + +**Anthropic Messages:** +``` +litellm/llms/anthropic/chat/guardrail_translation/ +├── __init__.py +└── handler.py +``` + +## Step-by-Step Implementation + +### Step 1: Create the Handler Class + +Create `handler.py` that inherits from `BaseTranslation`: + +```python +""" +{Provider} {Endpoint} Handler for Unified Guardrails + +This module provides guardrail translation support for {Provider}'s {Endpoint} format. +""" + +import asyncio +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast + +from litellm._logging import verbose_proxy_logger +from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation + +if TYPE_CHECKING: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.utils import ModelResponse # Or appropriate response type + + +class MyEndpointHandler(BaseTranslation): + """ + Handler for processing {Endpoint} with guardrails. + + This class provides methods to: + 1. Process input (pre-call hook) + 2. Process output response (post-call hook) + """ + + async def process_input_messages( + self, + data: dict, + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process input by applying guardrails to text content. + + Args: + data: Request data dictionary + guardrail_to_apply: The guardrail instance to apply + + Returns: + Modified data with guardrails applied + """ + # Your implementation here + pass + + async def process_output_response( + self, + response: Any, # Use appropriate response type + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process output response by applying guardrails to text content. + + Args: + response: API response object + guardrail_to_apply: The guardrail instance to apply + + Returns: + Modified response with guardrails applied + """ + # Your implementation here + pass +``` + +### Step 2: Implement Core Methods + +#### A. Process Input Messages + +Extract text from input, apply guardrails, and map back: + +```python +async def process_input_messages( + self, + data: dict, + guardrail_to_apply: "CustomGuardrail", +) -> Any: + """Process input messages by applying guardrails to text content.""" + # 1. Get input data from request + messages = data.get("messages") # or appropriate field + if messages is None: + return data + + # 2. Extract text and create tasks + tasks = [] + task_mappings: List[Tuple[int, Optional[int]]] = [] + + for msg_idx, message in enumerate(messages): + await self._extract_input_text_and_create_tasks( + message=message, + msg_idx=msg_idx, + tasks=tasks, + task_mappings=task_mappings, + guardrail_to_apply=guardrail_to_apply, + ) + + # 3. Run all guardrail tasks in parallel + if tasks: + responses = await asyncio.gather(*tasks) + + # 4. Map responses back to original structure + await self._apply_guardrail_responses_to_input( + messages=messages, + responses=responses, + task_mappings=task_mappings, + ) + + return data +``` + +#### B. Process Output Response + +Extract text from response, apply guardrails, and update: + +```python +async def process_output_response( + self, + response: "ModelResponse", + guardrail_to_apply: "CustomGuardrail", +) -> Any: + """Process output response by applying guardrails to text content.""" + # 1. Check if response has text to process + if not self._has_text_content(response): + return response + + # 2. Extract text and create tasks + tasks = [] + task_mappings: List[Tuple[int, Optional[int]]] = [] + + for idx, item in enumerate(response.choices): # or appropriate field + await self._extract_output_text_and_create_tasks( + item=item, + idx=idx, + tasks=tasks, + task_mappings=task_mappings, + guardrail_to_apply=guardrail_to_apply, + ) + + # 3. Run all guardrail tasks in parallel + if tasks: + responses = await asyncio.gather(*tasks) + + # 4. Update response with guardrailed text + await self._apply_guardrail_responses_to_output( + response=response, + responses=responses, + task_mappings=task_mappings, + ) + + return response +``` + +### Step 3: Create Helper Methods + +Implement helper methods for text extraction and mapping: + +```python +async def _extract_input_text_and_create_tasks( + self, + message: Dict[str, Any], + msg_idx: int, + tasks: List, + task_mappings: List[Tuple[int, Optional[int]]], + guardrail_to_apply: "CustomGuardrail", +) -> None: + """Extract text content from a message and create guardrail tasks.""" + content = message.get("content") + if content is None: + return + + if isinstance(content, str): + # Simple string content + tasks.append(guardrail_to_apply.apply_guardrail(text=content)) + task_mappings.append((msg_idx, None)) + elif isinstance(content, list): + # List content (e.g., multimodal) + for content_idx, content_item in enumerate(content): + if isinstance(content_item, dict): + text_str = content_item.get("text") + if text_str: + tasks.append(guardrail_to_apply.apply_guardrail(text=text_str)) + task_mappings.append((msg_idx, int(content_idx))) + +async def _apply_guardrail_responses_to_input( + self, + messages: List[Dict[str, Any]], + responses: List[str], + task_mappings: List[Tuple[int, Optional[int]]], +) -> None: + """Apply guardrail responses back to input messages.""" + for task_idx, guardrail_response in enumerate(responses): + msg_idx, content_idx = task_mappings[task_idx] + + if content_idx is None: + # String content + messages[msg_idx]["content"] = guardrail_response + else: + # List content + messages[msg_idx]["content"][content_idx]["text"] = guardrail_response + +def _has_text_content(self, response: Any) -> bool: + """Check if response has any text content to process.""" + # Implement based on your response structure + return True # or appropriate logic +``` + +### Step 4: Register the Handler + +Create `__init__.py` to register the handler with call types: + +```python +"""My Endpoint handler for Unified Guardrails.""" + +from litellm.llms.{provider}/{endpoint}/guardrail_translation.handler import ( + MyEndpointHandler, +) +from litellm.types.utils import CallTypes + +guardrail_translation_mappings = { + CallTypes.my_endpoint: MyEndpointHandler, + CallTypes.amy_endpoint: MyEndpointHandler, # async version if applicable +} + +__all__ = ["guardrail_translation_mappings"] +``` + +**Important:** Make sure your `CallTypes` are defined in `litellm/types/utils.py`. + +### Step 5: Add Documentation + +Create `README.md` with usage examples and format details: + +```markdown +# {Provider} {Endpoint} Guardrail Translation Handler + +Handler for processing {Provider}'s {Endpoint} with guardrails. + +## Overview + +This handler processes {Endpoint} input/output by: +1. Extracting text from messages/responses +2. Applying guardrails to text content +3. Mapping guardrailed text back to original structure + +## Data Format + +### Input Format +```json +{ + "field": "value", + "messages": [...] +} +``` + +### Output Format +```json +{ + "field": "value", + "output": [...] +} +``` + +## Usage + +The handler is automatically discovered and applied when guardrails are used with this endpoint. + +```bash +curl -X POST 'http://localhost:4000/{my_endpoint}' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer your-api-key' \ +-d '{ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "guardrails": ["test"] +}' + +``` +## Extension + +Override these methods to customize behavior: +- `_extract_input_text_and_create_tasks()`: Custom text extraction +- `_apply_guardrail_responses_to_input()`: Custom response mapping +- `_has_text_content()`: Custom content detection +``` + +### Step 6: Add Unit Tests + +Create comprehensive tests in `tests/test_litellm/llms/{provider}/{endpoint}/`: + +```python +""" +Unit tests for {Provider} {Endpoint} Guardrail Translation Handler +""" + +import os +import sys +import pytest + +sys.path.insert(0, os.path.abspath("../../../../../..")) + +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.llms import get_guardrail_translation_mapping +from litellm.llms.{provider}.{endpoint}.guardrail_translation.handler import ( + MyEndpointHandler, +) +from litellm.types.utils import CallTypes + + +class MockGuardrail(CustomGuardrail): + """Mock guardrail for testing""" + + async def apply_guardrail(self, text: str) -> str: + return f"{text} [GUARDRAILED]" + + +class TestHandlerDiscovery: + """Test that the handler is properly discovered""" + + def test_handler_discovered(self): + handler_class = get_guardrail_translation_mapping(CallTypes.my_endpoint) + assert handler_class == MyEndpointHandler + + +class TestInputProcessing: + """Test input processing functionality""" + + @pytest.mark.asyncio + async def test_process_simple_input(self): + handler = MyEndpointHandler() + guardrail = MockGuardrail(guardrail_name="test") + + data = {"messages": [{"role": "user", "content": "Hello"}]} + result = await handler.process_input_messages(data, guardrail) + + assert result["messages"][0]["content"] == "Hello [GUARDRAILED]" + + +class TestOutputProcessing: + """Test output processing functionality""" + + @pytest.mark.asyncio + async def test_process_simple_output(self): + handler = MyEndpointHandler() + guardrail = MockGuardrail(guardrail_name="test") + + # Create mock response + response = create_mock_response() + result = await handler.process_output_response(response, guardrail) + + # Assert guardrail was applied + assert "GUARDRAILED" in get_response_text(result) +``` + +## Support + +For questions or issues: +- Check existing handler implementations for examples +- Review the base translation class documentation +- Create an issue on GitHub with the `guardrails` label + diff --git a/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md b/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md new file mode 100644 index 00000000000..2722a4a024c --- /dev/null +++ b/docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md @@ -0,0 +1,184 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Adding a New Guardrail Integration + +You're going to create a class that checks text before it goes to the LLM or after it comes back. If it violates your rules, you block it. + +## How It Works + +Request with guardrail: + +```bash +curl --location 'http://localhost:4000/chat/completions' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "gpt-4", + "messages": [{"role": "user", "content": "How do I hack a system?"}], + "guardrails": ["my-guardrail"] +}' +``` + +Your guardrail checks input, then output. If something's wrong, raise an exception. + +## Build Your Guardrail + +### Create Your Directory + +```bash +mkdir -p litellm/proxy/guardrails/guardrail_hooks/my_guardrail +cd litellm/proxy/guardrails/guardrail_hooks/my_guardrail +``` + +Two files: `my_guardrail.py` (main class) and `__init__.py` (initialization). + +### Write the Main Class + +`my_guardrail.py`: + +```python +import os +from typing import Optional, List +from fastapi import HTTPException + +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.types.guardrails import PiiEntityType +from litellm._logging import verbose_proxy_logger +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) + +class MyGuardrail(CustomGuardrail): + def __init__(self, api_key: Optional[str] = None, api_base: Optional[str] = None, **kwargs): + self.api_key = api_key or os.getenv("MY_GUARDRAIL_API_KEY") + self.api_base = api_base or os.getenv("MY_GUARDRAIL_API_BASE", "https://api.myguardrail.com") + super().__init__(default_on=True) + + async def apply_guardrail( + self, + text: str, + language: Optional[str] = None, + entities: Optional[List[PiiEntityType]] = None, + request_data: Optional[dict] = None, + ) -> str: + result = await self._check_with_api(text, request_data) + + if result.get("action") == "BLOCK": + raise Exception(f"Content blocked: {result.get('reason', 'Policy violation')}") + + return text + + async def _check_with_api(self, text: str, request_data: Optional[dict]) -> dict: + async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback) + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.api_key}", + } + + response = await async_client.post( + f"{self.api_base}/check", + headers=headers, + json={"text": text}, + timeout=5, + ) + + response.raise_for_status() + return response.json() +``` + +### Create the Init File + +`__init__.py`: + +```python +from typing import TYPE_CHECKING + +from litellm.types.guardrails import SupportedGuardrailIntegrations + +from .my_guardrail import MyGuardrail + +if TYPE_CHECKING: + from litellm.types.guardrails import Guardrail, LitellmParams + + +def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"): + import litellm + + _my_guardrail_callback = MyGuardrail( + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + guardrail_name=guardrail.get("guardrail_name", ""), + event_hook=litellm_params.mode, + default_on=litellm_params.default_on, + ) + + litellm.logging_callback_manager.add_litellm_callback(_my_guardrail_callback) + return _my_guardrail_callback + + +guardrail_initializer_registry = { + SupportedGuardrailIntegrations.MY_GUARDRAIL.value: initialize_guardrail, +} + +guardrail_class_registry = { + SupportedGuardrailIntegrations.MY_GUARDRAIL.value: MyGuardrail, +} +``` + +### Register Your Guardrail Type + +Add to `litellm/types/guardrails.py`: + +```python +class SupportedGuardrailIntegrations(str, Enum): + LAKERA = "lakera_prompt_injection" + APORIA = "aporia" + BEDROCK = "bedrock_guardrails" + PRESIDIO = "presidio" + ZSCALER_AI_GUARD = "zscaler_ai_guard" + MY_GUARDRAIL = "my_guardrail" +``` + +## Usage + +### Config File + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: gpt-4 + api_key: os.environ/OPENAI_API_KEY + +litellm_settings: + guardrails: + - guardrail_name: my_guardrail + litellm_params: + guardrail: my_guardrail + mode: during_call + api_key: os.environ/MY_GUARDRAIL_API_KEY + api_base: https://api.myguardrail.com +``` + +### Per-Request + +```bash +curl --location 'http://localhost:4000/chat/completions' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "gpt-4", + "messages": [{"role": "user", "content": "Test message"}], + "guardrails": ["my_guardrail"] +}' +``` + +## Testing + +Add unit tests inside `test_litellm/` folder. + + + diff --git a/docs/my-website/docs/anthropic_unified.md b/docs/my-website/docs/anthropic_unified.md index 03ba8a68847..9981547ce1f 100644 --- a/docs/my-website/docs/anthropic_unified.md +++ b/docs/my-website/docs/anthropic_unified.md @@ -10,13 +10,14 @@ Use LiteLLM to call all your LLM APIs in the Anthropic `v1/messages` format. | Feature | Supported | Notes | |-------|-------|-------| -| Cost Tracking | ✅ | | -| Logging | ✅ | works across all integrations | +| Cost Tracking | ✅ | Works with all supported models | +| Logging | ✅ | Works across all integrations | | End-user Tracking | ✅ | | | Streaming | ✅ | | -| Fallbacks | ✅ | between supported models | -| Loadbalancing | ✅ | between supported models | -| Support llm providers | **All LiteLLM supported providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai`, etc. | +| Fallbacks | ✅ | Works between supported models | +| Loadbalancing | ✅ | Works between supported models | +| Guardrails | ✅ | Applies to input and output text (non-streaming only) | +| Supported Providers | **All LiteLLM supported providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai`, etc. | ## Usage --- diff --git a/docs/my-website/docs/apply_guardrail.md b/docs/my-website/docs/apply_guardrail.md index 740eb232e13..18fe951c52a 100644 --- a/docs/my-website/docs/apply_guardrail.md +++ b/docs/my-website/docs/apply_guardrail.md @@ -3,13 +3,49 @@ import TabItem from '@theme/TabItem'; # /guardrails/apply_guardrail -Use this endpoint to directly call a guardrail configured on your LiteLLM instance. This is useful when you have services that need to directly call a guardrail. +Use this endpoint to directly call a guardrail configured on your LiteLLM instance. This is useful when you have services that need to directly call a guardrail. + +## Supported Guardrail Types + +This endpoint supports various guardrail types including: +- **Presidio** - PII detection and masking +- **Bedrock** - AWS Bedrock guardrails for content moderation +- **Lakera** - AI safety guardrails +- **Custom guardrails** - User-defined guardrails + +## Configuration + +### Bedrock Guardrail Configuration + +To use Bedrock guardrails with the apply_guardrail endpoint, configure your guardrail in your LiteLLM config.yaml: + +```yaml +guardrails: + - guardrail_name: "bedrock-content-guard" + litellm_params: + guardrail: bedrock + mode: "pre_call" + guardrailIdentifier: "your-guardrail-id" # Your actual Bedrock guardrail ID + guardrailVersion: "DRAFT" # or your version number + aws_region_name: "us-east-1" # Your AWS region + aws_role_name: "your-role-arn" # Your AWS role with Bedrock permissions + default_on: true +``` + +**Required AWS Setup:** +1. Create a Bedrock guardrail in AWS Console +2. Get the guardrail ID and version +3. Ensure your AWS credentials have Bedrock permissions +4. Configure the guardrail in your LiteLLM config ## Usage --- -In this example `mask_pii` is the guardrail name configured on LiteLLM. + + + +In this example `mask_pii` is a Presidio guardrail configured on LiteLLM. ```bash showLineNumbers title="Example calling the endpoint" curl -X POST 'http://localhost:4000/guardrails/apply_guardrail' \ @@ -23,6 +59,27 @@ curl -X POST 'http://localhost:4000/guardrails/apply_guardrail' \ }' ``` + + + +In this example `bedrock-content-guard` is a Bedrock guardrail configured on LiteLLM. + +```bash showLineNumbers title="Example calling the endpoint" +curl -X POST 'http://localhost:4000/guardrails/apply_guardrail' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer your-api-key' \ +-d '{ + "guardrail_name": "bedrock-content-guard", + "text": "This is potentially harmful content that should be blocked", + "language": "en" +}' +``` + +**Note**: For Bedrock guardrails, the `entities` parameter is not used as Bedrock handles content moderation based on its own policies. + + + + ## Request Format --- @@ -59,12 +116,39 @@ The response will contain the processed text after applying the guardrail. #### Example Response + + + ```json { "response_text": "My name is [REDACTED] and my email is [REDACTED]" } ``` + + + +```json +{ + "response_text": "This is potentially harmful content that should be blocked" +} +``` + +**Note**: If Bedrock guardrail blocks the content, the endpoint will return an error with the blocking reason. + + + + #### Response Fields - **response_text** (string): The text after applying the guardrail. + +#### Error Responses + +If a guardrail blocks content (e.g., Bedrock guardrail), the endpoint will return an error: + +```json +{ + "detail": "Content blocked by Bedrock guardrail: Content violates policy" +} +``` diff --git a/docs/my-website/docs/audio_transcription.md b/docs/my-website/docs/audio_transcription.md index 8cbc567180c..fd55cc66e92 100644 --- a/docs/my-website/docs/audio_transcription.md +++ b/docs/my-website/docs/audio_transcription.md @@ -7,12 +7,13 @@ import TabItem from '@theme/TabItem'; | Feature | Supported | Notes | |-------|-------|-------| -| Cost Tracking | ✅ | | -| Logging | ✅ | works across all integrations | +| Cost Tracking | ✅ | Works with all supported models | +| Logging | ✅ | Works across all integrations | | End-user Tracking | ✅ | | -| Fallbacks | ✅ | between supported models | -| Loadbalancing | ✅ | between supported models | -| Support llm providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai` | | +| Fallbacks | ✅ | Works between supported models | +| Loadbalancing | ✅ | Works between supported models | +| Guardrails | ✅ | Applies to output transcribed text (non-streaming only) | +| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai` | | ## Quick Start diff --git a/docs/my-website/docs/bedrock_converse.md b/docs/my-website/docs/bedrock_converse.md new file mode 100644 index 00000000000..cf66b1a50a6 --- /dev/null +++ b/docs/my-website/docs/bedrock_converse.md @@ -0,0 +1,151 @@ +# /converse + +Call Bedrock's `/converse` endpoint through LiteLLM Proxy. + +| Feature | Supported | +|---------|-----------| +| Cost Tracking | ✅ | +| Logging | ✅ | +| Streaming | ✅ via `/converse-stream` | +| Load Balancing | ✅ | + +## Quick Start + +### 1. Setup config.yaml + +```yaml showLineNumbers +model_list: + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # reads from environment + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + custom_llm_provider: bedrock +``` + +Set AWS credentials in your environment: + +```bash showLineNumbers +export AWS_ACCESS_KEY_ID="your-access-key" +export AWS_SECRET_ACCESS_KEY="your-secret-key" +``` + +### 2. Start Proxy + +```bash showLineNumbers +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +### 3. Call /converse endpoint + +```bash showLineNumbers +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/converse' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "messages": [ + { + "role": "user", + "content": [{"text": "Hello, how are you?"}] + } + ], + "inferenceConfig": { + "temperature": 0.5, + "maxTokens": 100 + } +}' +``` + +## Streaming + +For streaming responses, use `/converse-stream`: + +```bash showLineNumbers +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/converse-stream' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "messages": [ + { + "role": "user", + "content": [{"text": "Tell me a short story"}] + } + ], + "inferenceConfig": { + "temperature": 0.7, + "maxTokens": 200 + } +}' +``` + +## Load Balancing + +Define multiple deployments with the same `model_name` for automatic load balancing: + +```yaml showLineNumbers +model_list: + # Deployment 1 - us-west-2 + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + custom_llm_provider: bedrock + + # Deployment 2 - us-east-1 + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-east-1 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + custom_llm_provider: bedrock +``` + +The proxy automatically distributes requests across both regions. + +## Using boto3 SDK + +```python showLineNumbers +import boto3 +import json +import os + +# Set dummy AWS credentials (required by boto3, but not used by LiteLLM proxy) +os.environ['AWS_ACCESS_KEY_ID'] = 'dummy' +os.environ['AWS_SECRET_ACCESS_KEY'] = 'dummy' +os.environ['AWS_BEARER_TOKEN_BEDROCK'] = "sk-1234" # your litellm proxy api key + +# Point boto3 to the LiteLLM proxy +bedrock_runtime = boto3.client( + service_name='bedrock-runtime', + region_name='us-west-2', + endpoint_url='http://0.0.0.0:4000/bedrock' +) + +response = bedrock_runtime.converse( + modelId='my-bedrock-model', # Your model_name from config.yaml + messages=[ + { + "role": "user", + "content": [{"text": "Hello, how are you?"}] + } + ], + inferenceConfig={ + "temperature": 0.5, + "maxTokens": 100 + } +) + +print(response['output']['message']['content'][0]['text']) +``` + +## More Info + +For complete documentation including Guardrails, Knowledge Bases, and Agents, see: +- [Full Bedrock Passthrough Docs](./pass_through/bedrock) + diff --git a/docs/my-website/docs/bedrock_invoke.md b/docs/my-website/docs/bedrock_invoke.md new file mode 100644 index 00000000000..6f29f1d51c3 --- /dev/null +++ b/docs/my-website/docs/bedrock_invoke.md @@ -0,0 +1,145 @@ +# /invoke + +Call Bedrock's `/invoke` endpoint through LiteLLM Proxy. + +| Feature | Supported | +|---------|-----------| +| Cost Tracking | ✅ | +| Logging | ✅ | +| Streaming | ✅ via `/invoke-with-response-stream` | +| Load Balancing | ✅ | + +## Quick Start + +### 1. Setup config.yaml + +```yaml showLineNumbers +model_list: + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # reads from environment + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + custom_llm_provider: bedrock +``` + +Set AWS credentials in your environment: + +```bash showLineNumbers +export AWS_ACCESS_KEY_ID="your-access-key" +export AWS_SECRET_ACCESS_KEY="your-secret-key" +``` + +### 2. Start Proxy + +```bash showLineNumbers +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +### 3. Call /invoke endpoint + +```bash showLineNumbers +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/invoke' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ], + "anthropic_version": "bedrock-2023-05-31" +}' +``` + +## Streaming + +For streaming responses, use `/invoke-with-response-stream`: + +```bash showLineNumbers +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/invoke-with-response-stream' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Tell me a short story" + } + ], + "anthropic_version": "bedrock-2023-05-31" +}' +``` + +## Load Balancing + +Define multiple deployments with the same `model_name` for automatic load balancing: + +```yaml showLineNumbers +model_list: + # Deployment 1 - us-west-2 + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + custom_llm_provider: bedrock + + # Deployment 2 - us-east-1 + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-east-1 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + custom_llm_provider: bedrock +``` + +The proxy automatically distributes requests across both regions. + +## Using boto3 SDK + +```python showLineNumbers +import boto3 +import json +import os + +# Set dummy AWS credentials (required by boto3, but not used by LiteLLM proxy) +os.environ['AWS_ACCESS_KEY_ID'] = 'dummy' +os.environ['AWS_SECRET_ACCESS_KEY'] = 'dummy' +os.environ['AWS_BEARER_TOKEN_BEDROCK'] = "sk-1234" # your litellm proxy api key + +# Point boto3 to the LiteLLM proxy +bedrock_runtime = boto3.client( + service_name='bedrock-runtime', + region_name='us-west-2', + endpoint_url='http://0.0.0.0:4000/bedrock' +) + +response = bedrock_runtime.invoke_model( + modelId='my-bedrock-model', # Your model_name from config.yaml + contentType='application/json', + accept='application/json', + body=json.dumps({ + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hello"}], + "anthropic_version": "bedrock-2023-05-31" + }) +) + +response_body = json.loads(response['body'].read()) +print(response_body['content'][0]['text']) +``` + +## More Info + +For complete documentation including Guardrails, Knowledge Bases, and Agents, see: +- [Full Bedrock Passthrough Docs](./pass_through/bedrock) + diff --git a/docs/my-website/docs/benchmarks.md b/docs/my-website/docs/benchmarks.md index 47697355dbf..f00732450d1 100644 --- a/docs/my-website/docs/benchmarks.md +++ b/docs/my-website/docs/benchmarks.md @@ -55,6 +55,10 @@ Each machine deploying LiteLLM had the following specs: - 4 CPU - 8GB RAM +## Configuration + +- Database: PostgreSQL +- Redis: Not used ## Locust Settings @@ -118,6 +122,48 @@ class MyUser(HttpUser): ``` +## LiteLLM vs Portkey Performance Comparison + +**Test Configuration**: 4 CPUs, 8 GB RAM per instance | Load: 1k concurrent users, 500 ramp-up + +### Multi-Instance (4×) Performance + +| Metric | Portkey (no DB) | LiteLLM (with DB) | +| ------------------- | --------------- | ----------------- | +| **Total Requests** | 293,796 | 312,405 | +| **Failed Requests** | 0 | 0 | +| **Median Latency** | 100 ms | 100 ms | +| **p95 Latency** | 230 ms | 150 ms | +| **p99 Latency** | 500 ms | 240 ms | +| **Average Latency** | 123 ms | 111 ms | +| **Current RPS** | 1,170.9 | 1,170 | + +### Technical Insights + +**Portkey** + +**Pros** + +* Low memory footprint +* Stable latency with minimal spikes + +**Cons** + +* CPU utilization capped around ~40%, indicating underutilization of available compute resources +* Experienced three I/O timeout outages + +**LiteLLM** + +**Pros** + +* Fully utilizes available CPU capacity +* Strong connection handling and low latency after initial warm-up spikes + +**Cons** + +* High memory usage during initialization and per request + + ## Logging Callbacks diff --git a/docs/my-website/docs/completion/image_generation_chat.md b/docs/my-website/docs/completion/image_generation_chat.md index 58ae70e2fff..5538b7f8ff3 100644 --- a/docs/my-website/docs/completion/image_generation_chat.md +++ b/docs/my-website/docs/completion/image_generation_chat.md @@ -15,16 +15,22 @@ Supported Providers: - Google AI Studio (`gemini`) - Vertex AI (`vertex_ai/`) -LiteLLM will standardize the `image` response in the assistant message for models that support image generation during chat completions. +LiteLLM will standardize the `images` response in the assistant message for models that support image generation during chat completions. ```python title="Example response from litellm" "message": { ... "content": "Here's the image you requested:", - "image": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...", - "detail": "auto" - } + "images": [ + { + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...", + "detail": "auto" + }, + "index": 0, + "type": "image_url" + } + ] } ``` @@ -47,7 +53,7 @@ response = completion( ) print(response.choices[0].message.content) # Text response -print(response.choices[0].message.image) # Image data +print(response.choices[0].message.images) # List of image objects ``` @@ -103,10 +109,16 @@ curl http://0.0.0.0:4000/v1/chat/completions \ "message": { "content": "Here's the image you requested:", "role": "assistant", - "image": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...", - "detail": "auto" - } + "images": [ + { + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...", + "detail": "auto" + }, + "index": 0, + "type": "image_url" + } + ] } } ], @@ -141,8 +153,8 @@ response = completion( ) for chunk in response: - if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None: - print("Generated image:", chunk.choices[0].delta.image["url"]) + if hasattr(chunk.choices[0].delta, "images") and chunk.choices[0].delta.images is not None: + print("Generated image:", chunk.choices[0].delta.images[0]["image_url"]["url"]) break ``` @@ -175,7 +187,7 @@ data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084 data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"content":"Here's the image you requested:"},"finish_reason":null}]} -data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"image":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...","detail":"auto"}},"finish_reason":null}]} +data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"images":[{"image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...","detail":"auto"},"index":0,"type":"image_url"}]},"finish_reason":null}]} data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]} @@ -200,8 +212,8 @@ async def generate_image(): ) print(response.choices[0].message.content) # Text response - print(response.choices[0].message.image) # Image data - + print(response.choices[0].message.images) # List of image objects + return response # Run the async function @@ -212,21 +224,31 @@ asyncio.run(generate_image()) | Provider | Model | |----------|--------| -| Google AI Studio | `gemini/gemini-2.5-flash-image-preview` | -| Vertex AI | `vertex_ai/gemini-2.5-flash-image-preview` | +| Google AI Studio | `gemini/gemini-2.0-flash-preview-image-generation`, `gemini/gemini-2.5-flash-image-preview` | +| Vertex AI | `vertex_ai/gemini-2.0-flash-preview-image-generation`, `vertex_ai/gemini-2.5-flash-image-preview` | -## Spec +## Spec -The `image` field in the response follows this structure: +The `images` field in the response follows this structure: ```python -"image": { - "url": "data:image/png;base64,", - "detail": "auto" -} +"images": [ + { + "image_url": { + "url": "data:image/png;base64,", + "detail": "auto" + }, + "index": 0, + "type": "image_url" + } +] ``` -- `url` - str: Base64 encoded image data in data URI format -- `detail` - str: Image detail level (always "auto" for generated images) +- `images` - List[ImageURLListItem]: Array of generated images + - `image_url` - ImageURLObject: Container for image data + - `url` - str: Base64 encoded image data in data URI format + - `detail` - str: Image detail level (always "auto" for generated images) + - `index` - int: Index of the image in the response + - `type` - str: Type identifier (always "image_url") -The image is returned as a base64-encoded data URI that can be directly used in HTML `` tags or saved to a file. +The images are returned as base64-encoded data URIs that can be directly used in HTML `` tags or saved to files. diff --git a/docs/my-website/docs/completion/json_mode.md b/docs/my-website/docs/completion/json_mode.md index ec140ce5827..c86a1e59893 100644 --- a/docs/my-website/docs/completion/json_mode.md +++ b/docs/my-website/docs/completion/json_mode.md @@ -309,33 +309,30 @@ curl http://0.0.0.0:4000/v1/chat/completions \ {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."}, ], "response_format": { - "type": "json_object", - "response_schema": { - "type": "json_schema", - "json_schema": { - "name": "math_reasoning", - "schema": { - "type": "object", - "properties": { - "steps": { - "type": "array", - "items": { - "type": "object", - "properties": { - "explanation": { "type": "string" }, - "output": { "type": "string" } - }, - "required": ["explanation", "output"], - "additionalProperties": false - } + "type": "json_schema", + "json_schema": { + "name": "math_reasoning", + "schema": { + "type": "object", + "properties": { + "steps": { + "type": "array", + "items": { + "type": "object", + "properties": { + "explanation": { "type": "string" }, + "output": { "type": "string" } }, - "final_answer": { "type": "string" } - }, - "required": ["steps", "final_answer"], - "additionalProperties": false + "required": ["explanation", "output"], + "additionalProperties": false + } }, - "strict": true + "final_answer": { "type": "string" } }, + "required": ["steps", "final_answer"], + "additionalProperties": false + }, + "strict": true } }, }' diff --git a/docs/my-website/docs/completion/knowledgebase.md b/docs/my-website/docs/completion/knowledgebase.md index ee0e3086785..3040f7f1cc0 100644 --- a/docs/my-website/docs/completion/knowledgebase.md +++ b/docs/my-website/docs/completion/knowledgebase.md @@ -18,7 +18,7 @@ LiteLLM integrates with vector stores, allowing your models to access your organ ## Supported Vector Stores - [Bedrock Knowledge Bases](https://aws.amazon.com/bedrock/knowledge-bases/) - [OpenAI Vector Stores](https://platform.openai.com/docs/api-reference/vector-stores/search) -- [Azure Vector Stores](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/file-search?tabs=python#vector-stores) +- [Azure Vector Stores](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/file-search?tabs=python#vector-stores) (Cannot be directly queried. Only available for calling in Assistants messages. We will be adding Azure AI Search Vector Store API support soon.) - [Vertex AI RAG API](https://cloud.google.com/vertex-ai/generative-ai/docs/rag-overview) ## Quick Start @@ -412,6 +412,219 @@ This is sent to: `https://bedrock-agent-runtime.{aws_region}.amazonaws.com/knowl This process happens automatically whenever you include the `vector_store_ids` parameter in your request. +## Accessing Search Results (Citations) + +When using vector stores, LiteLLM automatically returns search results in `provider_specific_fields`. This allows you to show users citations for the AI's response. + +### Key Concept + +Search results are always in: `response.choices[0].message.provider_specific_fields["search_results"]` + +For streaming: Results appear in the **final chunk** when `finish_reason == "stop"` + +### Non-Streaming Example + + +**Non-Streaming Response with search results:** + +```json +{ + "id": "chatcmpl-abc123", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "LiteLLM is a platform...", + "provider_specific_fields": { + "search_results": [{ + "search_query": "What is litellm?", + "data": [{ + "score": 0.95, + "content": [{"text": "...", "type": "text"}], + "filename": "litellm-docs.md", + "file_id": "doc-123" + }] + }] + } + }, + "finish_reason": "stop" + }] +} +``` + + + + +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-litellm-api-key" +) + +response = client.chat.completions.create( + model="claude-3-5-sonnet", + messages=[{"role": "user", "content": "What is litellm?"}], + tools=[{"type": "file_search", "vector_store_ids": ["T37J8R4WTM"]}] +) + +# Get AI response +print(response.choices[0].message.content) + +# Get search results (citations) +search_results = response.choices[0].message.provider_specific_fields.get("search_results", []) + +for result_page in search_results: + for idx, item in enumerate(result_page['data'], 1): + print(f"[{idx}] {item.get('filename', 'Unknown')} (score: {item['score']:.2f})") +``` + + + + + +```typescript +import OpenAI from 'openai'; + +const client = new OpenAI({ + baseURL: 'http://localhost:4000', + apiKey: process.env.LITELLM_API_KEY +}); + +const response = await client.chat.completions.create({ + model: 'claude-3-5-sonnet', + messages: [{ role: 'user', content: 'What is litellm?' }], + tools: [{ type: 'file_search', vector_store_ids: ['T37J8R4WTM'] }] +}); + +// Get AI response +console.log(response.choices[0].message.content); + +// Get search results (citations) +const message = response.choices[0].message as any; +const searchResults = message.provider_specific_fields?.search_results || []; + +searchResults.forEach((page: any) => { + page.data.forEach((item: any, idx: number) => { + console.log(`[${idx + 1}] ${item.filename || 'Unknown'} (${item.score.toFixed(2)})`); + }); +}); +``` + + + + +### Streaming Example + +**Streaming Response with search results (final chunk):** + +```json +{ + "id": "chatcmpl-abc123", + "choices": [{ + "index": 0, + "delta": { + "provider_specific_fields": { + "search_results": [{ + "search_query": "What is litellm?", + "data": [{ + "score": 0.95, + "content": [{"text": "...", "type": "text"}], + "filename": "litellm-docs.md", + "file_id": "doc-123" + }] + }] + } + }, + "finish_reason": "stop" + }] +} +``` + + + + +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-litellm-api-key" +) + +stream = client.chat.completions.create( + model="claude-3-5-sonnet", + messages=[{"role": "user", "content": "What is litellm?"}], + tools=[{"type": "file_search", "vector_store_ids": ["T37J8R4WTM"]}], + stream=True +) + +for chunk in stream: + # Stream content + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="", flush=True) + + # Get citations in final chunk + if chunk.choices[0].finish_reason == "stop": + search_results = getattr(chunk.choices[0].delta, 'provider_specific_fields', {}).get('search_results', []) + if search_results: + print("\n\nSources:") + for page in search_results: + for idx, item in enumerate(page['data'], 1): + print(f" [{idx}] {item.get('filename', 'Unknown')} ({item['score']:.2f})") +``` + + + + + +```typescript +import OpenAI from 'openai'; + +const stream = await client.chat.completions.create({ + model: 'claude-3-5-sonnet', + messages: [{ role: 'user', content: 'What is litellm?' }], + tools: [{ type: 'file_search', vector_store_ids: ['T37J8R4WTM'] }], + stream: true +}); + +for await (const chunk of stream) { + // Stream content + if (chunk.choices[0]?.delta?.content) { + process.stdout.write(chunk.choices[0].delta.content); + } + + // Get citations in final chunk + if (chunk.choices[0]?.finish_reason === 'stop') { + const searchResults = (chunk.choices[0].delta as any).provider_specific_fields?.search_results || []; + if (searchResults.length > 0) { + console.log('\n\nSources:'); + searchResults.forEach((page: any) => { + page.data.forEach((item: any, idx: number) => { + console.log(` [${idx + 1}] ${item.filename || 'Unknown'} (${item.score.toFixed(2)})`); + }); + }); + } + } +} +``` + + + + +### Search Result Fields + +| Field | Type | Description | +|-------|------|-------------| +| `search_query` | string | The query used to search the vector store | +| `data` | array | Array of search results | +| `data[].score` | float | Relevance score (0-1, higher is more relevant) | +| `data[].content` | array | Content chunks with `text` and `type` | +| `data[].filename` | string | Name of the source file (optional) | +| `data[].file_id` | string | Identifier for the source file (optional) | +| `data[].attributes` | object | Provider-specific metadata (optional) | + ## API Reference ### LiteLLM Completion Knowledge Base Parameters diff --git a/docs/my-website/docs/completion/prompt_caching.md b/docs/my-website/docs/completion/prompt_caching.md index 9447a11d527..630c9e58d24 100644 --- a/docs/my-website/docs/completion/prompt_caching.md +++ b/docs/my-website/docs/completion/prompt_caching.md @@ -27,7 +27,7 @@ For the supported providers, LiteLLM follows the OpenAI prompt caching usage obj } ``` -- `prompt_tokens`: These are the non-cached prompt tokens (same as Anthropic, equivalent to Deepseek `prompt_cache_miss_tokens`). +- `prompt_tokens`: These are all prompt tokens including cache-miss and cache-hit input tokens. - `completion_tokens`: These are the output tokens generated by the model. - `total_tokens`: Sum of prompt_tokens + completion_tokens. - `prompt_tokens_details`: Object containing cached_tokens. @@ -506,3 +506,11 @@ curl -L -X GET 'http://0.0.0.0:4000/v1/model/info' \ This checks our maintained [model info/cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json) + +## Read More + +:::tip Auto-Inject Prompt Caching +Want LiteLLM to automatically add `cache_control` directives without modifying your code? + +See [**Auto-Inject Prompt Caching Tutorial**](../tutorials/prompt_caching.md) to learn how to use `cache_control_injection_points` to automatically cache system messages, specific messages by index, or custom injection patterns. +::: diff --git a/docs/my-website/docs/contact.md b/docs/my-website/docs/contact.md index 947ec86991c..b0aa9c6ce6a 100644 --- a/docs/my-website/docs/contact.md +++ b/docs/my-website/docs/contact.md @@ -2,6 +2,6 @@ [![](https://dcbadge.vercel.app/api/server/wuPM9dRgDw)](https://discord.gg/wuPM9dRgDw) -* [Community Slack 💭](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3) +* [Community Slack 💭](https://www.litellm.ai/support) * [Meet with us 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) * Contact us at ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/containers.md b/docs/my-website/docs/containers.md new file mode 100644 index 00000000000..597e0e2e4c6 --- /dev/null +++ b/docs/my-website/docs/containers.md @@ -0,0 +1,465 @@ +# /containers + +Manage OpenAI code interpreter containers (sessions) for executing code in isolated environments. + +| Feature | Supported | +|---------|-----------| +| Cost Tracking | ✅ | +| Logging | ✅ (Full request/response logging) | +| Load Balancing | ✅ | +| Proxy Server Support | ✅ Full proxy integration with virtual keys | +| Spend Management | ✅ Budget tracking and rate limiting | +| Supported Providers | `openai`| + +:::tip + +Containers provide isolated execution environments for code interpreter sessions. You can create, list, retrieve, and delete containers. + +::: + +## **LiteLLM Python SDK Usage** + +### Quick Start + +**Create a Container** + +```python +import litellm +import os + +# setup env +os.environ["OPENAI_API_KEY"] = "sk-.." + +container = litellm.create_container( + name="My Code Interpreter Container", + custom_llm_provider="openai", + expires_after={ + "anchor": "last_active_at", + "minutes": 20 + } +) + +print(f"Container ID: {container.id}") +print(f"Container Name: {container.name}") +``` + +### Async Usage + +```python +from litellm import acreate_container +import os + +os.environ["OPENAI_API_KEY"] = "sk-.." + +container = await acreate_container( + name="My Code Interpreter Container", + custom_llm_provider="openai", + expires_after={ + "anchor": "last_active_at", + "minutes": 20 + } +) + +print(f"Container ID: {container.id}") +print(f"Container Name: {container.name}") +``` + +### List Containers + +```python +from litellm import list_containers +import os + +os.environ["OPENAI_API_KEY"] = "sk-.." + +containers = list_containers( + custom_llm_provider="openai", + limit=20, + order="desc" +) + +print(f"Found {len(containers.data)} containers") +for container in containers.data: + print(f" - {container.id}: {container.name}") +``` + +**Async Usage:** + +```python +from litellm import alist_containers + +containers = await alist_containers( + custom_llm_provider="openai", + limit=20, + order="desc" +) + +print(f"Found {len(containers.data)} containers") +for container in containers.data: + print(f" - {container.id}: {container.name}") +``` + +### Retrieve a Container + +```python +from litellm import retrieve_container +import os + +os.environ["OPENAI_API_KEY"] = "sk-.." + +container = retrieve_container( + container_id="cntr_123...", + custom_llm_provider="openai" +) + +print(f"Container: {container.name}") +print(f"Status: {container.status}") +print(f"Created: {container.created_at}") +``` + +**Async Usage:** + +```python +from litellm import aretrieve_container + +container = await aretrieve_container( + container_id="cntr_123...", + custom_llm_provider="openai" +) + +print(f"Container: {container.name}") +print(f"Status: {container.status}") +print(f"Created: {container.created_at}") +``` + +### Delete a Container + +```python +from litellm import delete_container +import os + +os.environ["OPENAI_API_KEY"] = "sk-.." + +result = delete_container( + container_id="cntr_123...", + custom_llm_provider="openai" +) + +print(f"Deleted: {result.deleted}") +print(f"Container ID: {result.id}") +``` + +**Async Usage:** + +```python +from litellm import adelete_container + +result = await adelete_container( + container_id="cntr_123...", + custom_llm_provider="openai" +) + +print(f"Deleted: {result.deleted}") +print(f"Container ID: {result.id}") +``` + +## **LiteLLM Proxy Usage** + +LiteLLM provides OpenAI API compatible container endpoints for managing code interpreter sessions: + +- `/v1/containers` - Create and list containers +- `/v1/containers/{container_id}` - Retrieve and delete containers + +**Setup** + +```bash +$ export OPENAI_API_KEY="sk-..." + +$ litellm + +# RUNNING on http://0.0.0.0:4000 +``` + +**Custom Provider Specification** + +You can specify the custom LLM provider in multiple ways (priority order): +1. Header: `-H "custom-llm-provider: openai"` +2. Query param: `?custom_llm_provider=openai` +3. Request body: `{"custom_llm_provider": "openai", ...}` +4. Defaults to "openai" if not specified + +**Create a Container** + +```bash +# Default provider (openai) +curl -X POST "http://localhost:4000/v1/containers" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "My Container", + "expires_after": { + "anchor": "last_active_at", + "minutes": 20 + } + }' +``` + +```bash +# Via header +curl -X POST "http://localhost:4000/v1/containers" \ + -H "Authorization: Bearer sk-1234" \ + -H "custom-llm-provider: openai" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "My Container" + }' +``` + +```bash +# Via query parameter +curl -X POST "http://localhost:4000/v1/containers?custom_llm_provider=openai" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "My Container" + }' +``` + +**List Containers** + +```bash +curl "http://localhost:4000/v1/containers?limit=20&order=desc" \ + -H "Authorization: Bearer sk-1234" +``` + +**Retrieve a Container** + +```bash +curl "http://localhost:4000/v1/containers/cntr_123..." \ + -H "Authorization: Bearer sk-1234" +``` + +**Delete a Container** + +```bash +curl -X DELETE "http://localhost:4000/v1/containers/cntr_123..." \ + -H "Authorization: Bearer sk-1234" +``` + +## **Using OpenAI Client with LiteLLM Proxy** + +You can use the standard OpenAI Python client to interact with LiteLLM's container endpoints. This provides a familiar interface while leveraging LiteLLM's proxy features. + +### Setup + +First, configure your OpenAI client to point to your LiteLLM proxy: + +```python +from openai import OpenAI + +client = OpenAI( + api_key="sk-1234", # Your LiteLLM proxy key + base_url="http://localhost:4000" # LiteLLM proxy URL +) +``` + +### Create a Container + +```python +container = client.containers.create( + name="test-container", + expires_after={ + "anchor": "last_active_at", + "minutes": 20 + }, + extra_body={"custom_llm_provider": "openai"} +) + +print(f"Container ID: {container.id}") +print(f"Container Name: {container.name}") +print(f"Created at: {container.created_at}") +``` + +### List Containers + +```python +containers = client.containers.list( + limit=20, + extra_body={"custom_llm_provider": "openai"} +) + +print(f"Found {len(containers.data)} containers") +for container in containers.data: + print(f" - {container.id}: {container.name}") +``` + +### Retrieve a Container + +```python +container = client.containers.retrieve( + container_id="cntr_6901d28b3c8881908b702815828a5bde0380b3408aeae8c7", + extra_body={"custom_llm_provider": "openai"} +) + +print(f"Container: {container.name}") +print(f"Status: {container.status}") +print(f"Last active: {container.last_active_at}") +``` + +### Delete a Container + +```python +result = client.containers.delete( + container_id="cntr_6901d28b3c8881908b702815828a5bde0380b3408aeae8c7", + extra_body={"custom_llm_provider": "openai"} +) + +print(f"Deleted: {result.deleted}") +print(f"Container ID: {result.id}") +``` + +### Complete Workflow Example + +Here's a complete example showing the full container management workflow: + +```python +from openai import OpenAI + +# Initialize client +client = OpenAI( + api_key="sk-1234", + base_url="http://localhost:4000" +) + +# 1. Create a container +print("Creating container...") +container = client.containers.create( + name="My Code Interpreter Session", + expires_after={ + "anchor": "last_active_at", + "minutes": 20 + }, + extra_body={"custom_llm_provider": "openai"} +) + +container_id = container.id +print(f"Container created. ID: {container_id}") + +# 2. List all containers +print("\nListing containers...") +containers = client.containers.list( + extra_body={"custom_llm_provider": "openai"} +) + +for c in containers.data: + print(f" - {c.id}: {c.name} (Status: {c.status})") + +# 3. Retrieve specific container +print(f"\nRetrieving container {container_id}...") +retrieved = client.containers.retrieve( + container_id=container_id, + extra_body={"custom_llm_provider": "openai"} +) + +print(f"Container: {retrieved.name}") +print(f"Status: {retrieved.status}") +print(f"Last active: {retrieved.last_active_at}") + +# 4. Delete container +print(f"\nDeleting container {container_id}...") +result = client.containers.delete( + container_id=container_id, + extra_body={"custom_llm_provider": "openai"} +) + +print(f"Deleted: {result.deleted}") +``` + +## Container Parameters + +### Create Container Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `name` | string | Yes | Name of the container | +| `expires_after` | object | No | Container expiration settings | +| `expires_after.anchor` | string | No | Anchor point for expiration (e.g., "last_active_at") | +| `expires_after.minutes` | integer | No | Minutes until expiration from anchor | +| `file_ids` | array | No | List of file IDs to include in the container | +| `custom_llm_provider` | string | No | LLM provider to use (default: "openai") | + +### List Container Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `after` | string | No | Cursor for pagination | +| `limit` | integer | No | Number of items to return (1-100, default: 20) | +| `order` | string | No | Sort order: "asc" or "desc" (default: "desc") | +| `custom_llm_provider` | string | No | LLM provider to use (default: "openai") | + +### Retrieve/Delete Container Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `container_id` | string | Yes | ID of the container to retrieve/delete | +| `custom_llm_provider` | string | No | LLM provider to use (default: "openai") | + +## Response Objects + +### ContainerObject + +```json +{ + "id": "cntr_123...", + "object": "container", + "created_at": 1234567890, + "name": "My Container", + "status": "active", + "last_active_at": 1234567890, + "expires_at": 1234569090, + "file_ids": [] +} +``` + +### ContainerListResponse + +```json +{ + "object": "list", + "data": [ + { + "id": "cntr_123...", + "object": "container", + "created_at": 1234567890, + "name": "My Container", + "status": "active" + } + ], + "first_id": "cntr_123...", + "last_id": "cntr_456...", + "has_more": false +} +``` + +### DeleteContainerResult + +```json +{ + "id": "cntr_123...", + "object": "container.deleted", + "deleted": true +} +``` + +## **Supported Providers** + +| Provider | Support Status | Notes | +|-------------|----------------|-------| +| OpenAI | ✅ Supported | Full support for all container operations | + +:::info + +Currently, only OpenAI supports container management for code interpreter sessions. Support for additional providers may be added in the future. + +::: + diff --git a/docs/my-website/docs/exception_mapping.md b/docs/my-website/docs/exception_mapping.md index 2342f444e17..efdada2a1eb 100644 --- a/docs/my-website/docs/exception_mapping.md +++ b/docs/my-website/docs/exception_mapping.md @@ -112,6 +112,85 @@ except openai.APITimeoutError as e: print(f"should_retry: {should_retry}") ``` +## Advanced + +### Accessing Provider-Specific Error Details + +LiteLLM exceptions include a `provider_specific_fields` attribute that contains additional error information specific to each provider. This is particularly useful for Azure OpenAI, which provides detailed content filtering information. + +#### Azure OpenAI - Content Policy Violation Inner Error Access + +When Azure OpenAI returns content policy violations, you can access the detailed content filtering results through the `innererror` field: + +```python +import litellm +from litellm.exceptions import ContentPolicyViolationError + +try: + response = litellm.completion( + model="azure/gpt-4", + messages=[ + { + "role": "user", + "content": "Some content that might violate policies" + } + ] + ) +except ContentPolicyViolationError as e: + # Access Azure-specific error details + if e.provider_specific_fields and "innererror" in e.provider_specific_fields: + innererror = e.provider_specific_fields["innererror"] + + # Access content filter results + content_filter_result = innererror.get("content_filter_result", {}) + + print(f"Content filter code: {innererror.get('code')}") + print(f"Hate filtered: {content_filter_result.get('hate', {}).get('filtered')}") + print(f"Violence severity: {content_filter_result.get('violence', {}).get('severity')}") + print(f"Sexual content filtered: {content_filter_result.get('sexual', {}).get('filtered')}") +``` + +**Example Response Structure:** + +When calling the LiteLLM proxy, content policy violations will return detailed filtering information: + +```json +{ + "error": { + "message": "litellm.ContentPolicyViolationError: AzureException - The response was filtered due to the prompt triggering Azure OpenAI's content management policy...", + "type": null, + "param": null, + "code": "400", + "provider_specific_fields": { + "innererror": { + "code": "ResponsibleAIPolicyViolation", + "content_filter_result": { + "hate": { + "filtered": true, + "severity": "high" + }, + "jailbreak": { + "filtered": false, + "detected": false + }, + "self_harm": { + "filtered": false, + "severity": "safe" + }, + "sexual": { + "filtered": false, + "severity": "safe" + }, + "violence": { + "filtered": true, + "severity": "medium" + } + } + } + } + } +} + ## Details To see how it's implemented - [check out the code](https://github.com/BerriAI/litellm/blob/a42c197e5a6de56ea576c73715e6c7c6b19fa249/litellm/utils.py#L1217) diff --git a/docs/my-website/docs/extras/creating_adapters.md b/docs/my-website/docs/extras/creating_adapters.md new file mode 100644 index 00000000000..42e48f6ab3f --- /dev/null +++ b/docs/my-website/docs/extras/creating_adapters.md @@ -0,0 +1,206 @@ +# Call any LiteLLM model in your custom format + +Use this to call any LiteLLM supported `.completion()` model, in your custom format. Useful if you have a custom API and want to support any LiteLLM supported model. + +## How it works + +Your request → Adapter translates to OpenAI format → LiteLLM processes it → Adapter translates response back → Your response + +## Create an Adapter + +Inherit from `CustomLogger` and implement 3 methods: + +```python +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.llms.openai import ChatCompletionRequest +from litellm.types.utils import ModelResponse + +class MyAdapter(CustomLogger): + def translate_completion_input_params(self, kwargs) -> ChatCompletionRequest: + """Convert your format → OpenAI format""" + # Example: Anthropic to OpenAI + return { + "model": kwargs["model"], + "messages": self._convert_messages(kwargs["messages"]), + "max_tokens": kwargs.get("max_tokens"), + } + + def translate_completion_output_params(self, response: ModelResponse): + """Convert OpenAI format → your format""" + # Return your provider's response format + return MyProviderResponse( + id=response.id, + content=response.choices[0].message.content, + usage=response.usage, + ) + + def translate_completion_output_params_streaming(self, completion_stream): + """Handle streaming responses""" + return MyStreamWrapper(completion_stream) +``` + +## Register it + +```python +import litellm + +my_adapter = MyAdapter() +litellm.adapters = [{"id": "my_provider", "adapter": my_adapter}] +``` + +## Use it + +```python +from litellm import adapter_completion + +# Now you can use your provider's format with any LiteLLM model +response = adapter_completion( + adapter_id="my_provider", + model="gpt-4", # or any LiteLLM model + messages=[{"role": "user", "content": "hello"}], + max_tokens=100 +) +``` + +### Streaming + +```python +stream = adapter_completion( + adapter_id="my_provider", + model="gpt-4", + messages=[{"role": "user", "content": "hello"}], + stream=True +) + +for chunk in stream: + print(chunk) +``` + +### Async + +```python +from litellm import aadapter_completion + +response = await aadapter_completion( + adapter_id="my_provider", + model="gpt-4", + messages=[{"role": "user", "content": "hello"}] +) +``` + +## Example: Anthropic Adapter + +Here's how we translate Anthropic's format: + +### Input Translation + +```python +def translate_completion_input_params(self, kwargs): + model = kwargs.pop("model") + messages = kwargs.pop("messages") + + # Convert Anthropic messages to OpenAI format + openai_messages = [] + for msg in messages: + if msg["role"] == "user": + openai_messages.append({ + "role": "user", + "content": msg["content"] + }) + + # Handle system message + if "system" in kwargs: + openai_messages.insert(0, { + "role": "system", + "content": kwargs.pop("system") + }) + + return { + "model": model, + "messages": openai_messages, + **kwargs # pass through other params + } +``` + +### Output Translation + +```python +def translate_completion_output_params(self, response): + return AnthropicResponse( + id=response.id, + type="message", + role="assistant", + content=[{ + "type": "text", + "text": response.choices[0].message.content + }], + usage={ + "input_tokens": response.usage.prompt_tokens, + "output_tokens": response.usage.completion_tokens + } + ) +``` + +### Streaming + +```python +from litellm.types.utils import AdapterCompletionStreamWrapper + +class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): + def __init__(self, completion_stream, model): + super().__init__(completion_stream) + self.model = model + self.first_chunk = True + + async def __anext__(self): + # First chunk + if self.first_chunk: + self.first_chunk = False + return {"type": "message_start", "message": {...}} + + # Stream chunks + async for chunk in self.completion_stream: + return { + "type": "content_block_delta", + "delta": {"text": chunk.choices[0].delta.content} + } + + # Last chunk + return {"type": "message_stop"} + +def translate_completion_output_params_streaming(self, stream, model): + return AnthropicStreamWrapper(stream, model) +``` + +## Use with Proxy + +Add to your proxy config: + +```yaml +general_settings: + pass_through_endpoints: + - path: "/v1/messages" + target: "my_module.MyAdapter" +``` + +Then call it: + +```bash +curl http://localhost:4000/v1/messages \ + -H "Authorization: Bearer sk-1234" \ + -d '{"model": "gpt-4", "messages": [...]}' +``` + +## Real Example + +Check out the full Anthropic adapter: +- [transformation.py](https://github.com/BerriAI/litellm/blob/main/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py) +- [handler.py](https://github.com/BerriAI/litellm/blob/main/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py) +- [streaming_iterator.py](https://github.com/BerriAI/litellm/blob/main/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py) + +## That's it + +1. Create a class that inherits `CustomLogger` +2. Implement the 3 translation methods +3. Register with `litellm.adapters = [{"id": "...", "adapter": ...}]` +4. Call with `adapter_completion(adapter_id="...")` diff --git a/docs/my-website/docs/files_endpoints.md b/docs/my-website/docs/files_endpoints.md index 31a02d41a3f..88493fe0bbd 100644 --- a/docs/my-website/docs/files_endpoints.md +++ b/docs/my-website/docs/files_endpoints.md @@ -57,7 +57,7 @@ client = OpenAI( client.files.create( file=wav_data, purpose="user_data", - extra_body={"custom_llm_provider": "openai"} + extra_headers={"custom-llm-provider": "openai"} ) ``` @@ -71,7 +71,7 @@ client = OpenAI( base_url="http://0.0.0.0:4000/v1" ) -files = client.files.list(extra_body={"custom_llm_provider": "openai"}) +files = client.files.list(extra_headers={"custom-llm-provider": "openai"}) print("files=", files) ``` @@ -85,7 +85,7 @@ client = OpenAI( base_url="http://0.0.0.0:4000/v1" ) -file = client.files.retrieve(file_id="file-abc123", extra_body={"custom_llm_provider": "openai"}) +file = client.files.retrieve(file_id="file-abc123", extra_headers={"custom-llm-provider": "openai"}) print("file=", file) ``` @@ -99,7 +99,7 @@ client = OpenAI( base_url="http://0.0.0.0:4000/v1" ) -response = client.files.delete(file_id="file-abc123", extra_body={"custom_llm_provider": "openai"}) +response = client.files.delete(file_id="file-abc123", extra_headers={"custom-llm-provider": "openai"}) print("delete response=", response) ``` @@ -113,7 +113,7 @@ client = OpenAI( base_url="http://0.0.0.0:4000/v1" ) -content = client.files.content(file_id="file-abc123", extra_body={"custom_llm_provider": "openai"}) +content = client.files.content(file_id="file-abc123", extra_headers={"custom-llm-provider": "openai"}) print("content=", content) ``` diff --git a/docs/my-website/docs/fine_tuning.md b/docs/my-website/docs/fine_tuning.md index f3f955cb01d..2779a478f8f 100644 --- a/docs/my-website/docs/fine_tuning.md +++ b/docs/my-website/docs/fine_tuning.md @@ -62,7 +62,7 @@ client = AsyncOpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") # base_u file_name = "openai_batch_completions.jsonl" response = await client.files.create( - extra_body={"custom_llm_provider": "azure"}, # tell litellm proxy which provider to use + extra_headers={"custom-llm-provider": "azure"}, # tell litellm proxy which provider to use file=open(file_name, "rb"), purpose="fine-tune", ) @@ -73,8 +73,8 @@ response = await client.files.create( ```shell curl http://localhost:4000/v1/files \ -H "Authorization: Bearer sk-1234" \ + -H "custom-llm-provider: azure" \ -F purpose="batch" \ - -F custom_llm_provider="azure"\ -F file="@mydata.jsonl" ``` @@ -92,7 +92,7 @@ curl http://localhost:4000/v1/files \ ft_job = await client.fine_tuning.jobs.create( model="gpt-35-turbo-1106", # Azure OpenAI model you want to fine-tune training_file="file-abc123", # file_id from create file response - extra_body={"custom_llm_provider": "azure"}, # tell litellm proxy which provider to use + extra_headers={"custom-llm-provider": "azure"}, # tell litellm proxy which provider to use ) ``` @@ -103,8 +103,8 @@ ft_job = await client.fine_tuning.jobs.create( curl http://localhost:4000/v1/fine_tuning/jobs \ -H "Content-Type: application/json" \ -H "Authorization: Bearer sk-1234" \ + -H "custom-llm-provider: azure" \ -d '{ - "custom_llm_provider": "azure", "model": "gpt-35-turbo-1106", "training_file": "file-abc123" }' @@ -215,7 +215,7 @@ curl http://localhost:4000/v1/fine_tuning/jobs \ # cancel specific fine tuning job cancel_ft_job = await client.fine_tuning.jobs.cancel( fine_tuning_job_id="123", # fine tuning job id - extra_body={"custom_llm_provider": "azure"}, # tell litellm proxy which provider to use + extra_headers={"custom-llm-provider": "azure"}, # tell litellm proxy which provider to use ) print("response from cancel ft job={}".format(cancel_ft_job)) @@ -228,7 +228,7 @@ print("response from cancel ft job={}".format(cancel_ft_job)) curl -X POST http://localhost:4000/v1/fine_tuning/jobs/ftjob-abc123/cancel \ -H "Authorization: Bearer sk-1234" \ -H "Content-Type: application/json" \ - -d '{"custom_llm_provider": "azure"}' + -H "custom-llm-provider: azure" ``` @@ -242,7 +242,7 @@ curl -X POST http://localhost:4000/v1/fine_tuning/jobs/ftjob-abc123/cancel \ ```python list_ft_jobs = await client.fine_tuning.jobs.list( - extra_query={"custom_llm_provider": "azure"} # tell litellm proxy which provider to use + extra_headers={"custom-llm-provider": "azure"} # tell litellm proxy which provider to use ) print("list of ft jobs={}".format(list_ft_jobs)) @@ -252,9 +252,10 @@ print("list of ft jobs={}".format(list_ft_jobs)) ```shell -curl -X GET 'http://localhost:4000/v1/fine_tuning/jobs?custom_llm_provider=azure' \ +curl -X GET 'http://localhost:4000/v1/fine_tuning/jobs' \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer sk-1234" + -H "Authorization: Bearer sk-1234" \ + -H "custom-llm-provider: azure" ``` diff --git a/docs/my-website/docs/generateContent.md b/docs/my-website/docs/generateContent.md index e6823ebf05d..4453e5ce06d 100644 --- a/docs/my-website/docs/generateContent.md +++ b/docs/my-website/docs/generateContent.md @@ -1,7 +1,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Google AI generateContent +# /generateContent Use LiteLLM to call Google AI's generateContent endpoints for text generation, multimodal interactions, and streaming responses. diff --git a/docs/my-website/docs/guides/security_settings.md b/docs/my-website/docs/guides/security_settings.md index 7995f6c3c9c..d6397a7c197 100644 --- a/docs/my-website/docs/guides/security_settings.md +++ b/docs/my-website/docs/guides/security_settings.md @@ -117,10 +117,52 @@ litellm_settings: ```bash export SSL_CERTIFICATE="/path/to/certificate.pem" ``` + -## 5. Use HTTP_PROXY environment variable +## 5. Configure ECDH Curve for SSL/TLS Performance + +The `ssl_ecdh_curve` setting allows you to configure the Elliptic Curve Diffie-Hellman (ECDH) curve used for SSL/TLS key exchange. This is particularly useful for disabling Post-Quantum Cryptography (PQC) to improve performance in environments where PQC is not required. + +**Use Case:** Some OpenSSL 3.x systems enable PQC by default, which can slow down TLS handshakes. Setting the ECDH curve to `X25519` disables PQC and can significantly improve connection performance. + + + + +```python +import litellm +litellm.ssl_ecdh_curve = "X25519" # Disables PQC for better performance +``` + + + + +```yaml +litellm_settings: + ssl_ecdh_curve: "X25519" +``` + + + + +```bash +export SSL_ECDH_CURVE="X25519" +``` + + + + +**Common Valid Curves:** + +- `X25519` - Modern, fast curve (recommended for disabling PQC) +- `prime256v1` - NIST P-256 curve +- `secp384r1` - NIST P-384 curve +- `secp521r1` - NIST P-521 curve + +**Note:** If an invalid curve name is provided or if your Python/OpenSSL version doesn't support this feature, LiteLLM will log a warning and continue with default curves. + +## 6. Use HTTP_PROXY environment variable Both httpx and aiohttp libraries use `urllib.request.getproxies` from environment variables. Before client initialization, you may set proxy (and optional SSL_CERT_FILE) by setting the environment variables: diff --git a/docs/my-website/docs/image_edits.md b/docs/my-website/docs/image_edits.md index 84dddd5e4ad..9a53da510f7 100644 --- a/docs/my-website/docs/image_edits.md +++ b/docs/my-website/docs/image_edits.md @@ -14,9 +14,9 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit | Fallbacks | ✅ | Works between supported models | | Loadbalancing | ✅ | Works between supported models | | Supported operations | Create image edits | Single and multiple images supported | -| Supported LiteLLM SDK Versions | 1.63.8+ | | -| Supported LiteLLM Proxy Versions | 1.71.1+ | | -| Supported LLM providers | **OpenAI** | Currently only `openai` is supported | +| Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ | +| Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ | +| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)** | Gemini supports the new `gemini-2.5-flash-image` family | #### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/) @@ -149,6 +149,54 @@ for i, image_data in enumerate(response.data): print(f"Image {i+1}: {image_data.url}") ``` +``` + + + + + +#### Basic Image Edit +```python showLineNumbers title="Gemini Image Edit" +import base64 +import os +from litellm import image_edit + +os.environ["GEMINI_API_KEY"] = "your-api-key" + +response = image_edit( + model="gemini/gemini-2.5-flash-image", + image=open("original_image.png", "rb"), + prompt="Add aurora borealis to the night sky", + size="1792x1024", # mapped to aspectRatio=16:9 for Gemini +) + +edited_image_bytes = base64.b64decode(response.data[0].b64_json) +with open("edited_image.png", "wb") as f: + f.write(edited_image_bytes) +``` + +#### Multiple Images Edit +```python showLineNumbers title="Gemini Multiple Images Edit" +import base64 +import os +from litellm import image_edit + +os.environ["GEMINI_API_KEY"] = "your-api-key" + +response = image_edit( + model="gemini/gemini-2.5-flash-image", + image=[ + open("scene.png", "rb"), + open("style_reference.png", "rb"), + ], + prompt="Blend the reference style into the scene while keeping the subject sharp.", +) + +for idx, image_obj in enumerate(response.data): + with open(f"gemini_edit_{idx}.png", "wb") as f: + f.write(base64.b64decode(image_obj.b64_json)) +``` + @@ -224,6 +272,36 @@ curl -X POST "http://localhost:4000/v1/images/edits" \ -F "response_format=url" ``` +``` + + + + + +1. Add the Gemini image edit model to your `config.yaml`: +```yaml showLineNumbers title="Gemini Proxy Configuration" +model_list: + - model_name: gemini-image-edit + litellm_params: + model: gemini/gemini-2.5-flash-image + api_key: os.environ/GEMINI_API_KEY +``` + +2. Start the LiteLLM proxy server: +```bash showLineNumbers title="Start LiteLLM Proxy Server" +litellm --config /path/to/config.yaml +``` + +3. Make an image edit request (Gemini responses are base64-only): +```bash showLineNumbers title="Gemini Proxy Image Edit" +curl -X POST "http://0.0.0.0:4000/v1/images/edits" \ + -H "Authorization: Bearer " \ + -F "model=gemini-image-edit" \ + -F "image=@original_image.png" \ + -F "prompt=Add a warm golden-hour glow to the scene" \ + -F "size=1024x1024" +``` + diff --git a/docs/my-website/docs/image_generation.md b/docs/my-website/docs/image_generation.md index 8cd5803aa6c..b4eaef36521 100644 --- a/docs/my-website/docs/image_generation.md +++ b/docs/my-website/docs/image_generation.md @@ -5,6 +5,18 @@ import TabItem from '@theme/TabItem'; # Image Generations +## Overview + +| Feature | Supported | Notes | +|---------|-----------|-------| +| Cost Tracking | ✅ | Works with all supported models | +| Logging | ✅ | Works across all integrations | +| End-user Tracking | ✅ | | +| Fallbacks | ✅ | Works between supported models | +| Loadbalancing | ✅ | Works between supported models | +| Guardrails | ✅ | Applies to input prompts (non-streaming only) | +| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Recraft, Xinference, Nscale | | + ## Quick Start ### LiteLLM Python SDK diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md index 6b0ed067c55..c735b8ecdd9 100644 --- a/docs/my-website/docs/mcp.md +++ b/docs/my-website/docs/mcp.md @@ -107,6 +107,26 @@ For stdio MCP servers, select "Standard Input/Output (stdio)" as the transport t style={{width: '80%', display: 'block', margin: '0'}} /> +
+
+ +### Static Headers + +Sometimes your MCP server needs specific headers on every request. Maybe it's an API key, maybe it's a custom header the server expects. Instead of configuring auth, you can just set them directly. + + + +These headers get sent with every request to the server. That's it. + + +**When to use this:** +- Your server needs custom headers that don't fit the standard auth patterns +- You want full control over exactly what headers are sent +- You're debugging and need to quickly add headers without changing auth configuration + @@ -175,6 +195,7 @@ mcp_servers: | `authorization` | `Authorization: ` | - **Extra Headers**: Optional list of additional header names that should be forwarded from client to the MCP server +- **Static Headers**: Optional map of header key/value pairs to include every request to the MCP server. - **Spec Version**: Optional MCP specification version (defaults to `2025-06-18`) Examples for each auth type: @@ -217,8 +238,14 @@ mcp_servers: auth_type: "bearer_token" auth_value: "ghp_example_token" extra_headers: ["custom_key", "x-custom-header"] # These headers will be forwarded from client -``` + # Example with static headers + my_mcp_server: + url: "https://my-mcp-server.com/mcp" + static_headers: # These headers will be requested to the MCP server + X-API-Key: "abc123" + X-Custom-Header: "some-value" +``` ### MCP Aliases @@ -1204,6 +1231,10 @@ mcp_servers: scopes: ["public_repo", "user:email"] ``` +**Note** +In the future, users will only need to specify the `url` of the MCP server. +LiteLLM will automatically resolve the corresponding `authorization_url`, `token_url`, and `registration_url` based on the MCP server metadata (e.g., `.well-known/oauth-authorization-server` or `oauth-protected-resource`). + [**See Claude Code Tutorial**](./tutorials/claude_responses_api#connecting-mcp-servers) ## Using your MCP with client side credentials diff --git a/docs/my-website/docs/moderation.md b/docs/my-website/docs/moderation.md index f9c2810bc8a..1f67b0a7543 100644 --- a/docs/my-website/docs/moderation.md +++ b/docs/my-website/docs/moderation.md @@ -22,10 +22,19 @@ response = moderation( For `/moderations` endpoint, there is **no need to specify `model` in the request or on the litellm config.yaml** -Start litellm proxy server + +1. Setup config.yaml +```yaml +model_list: + - model_name: text-moderation-stable + litellm_params: + model: openai/omni-moderation-latest +``` + +2. Start litellm proxy server ``` -litellm +litellm --config /path/to/config.yaml ``` @@ -41,7 +50,7 @@ client = OpenAI(api_key="", base_url="http://0.0.0.0:4000") response = client.moderations.create( input="hello from litellm", - model="text-moderation-stable" # optional, defaults to `omni-moderation-latest` + model="text-moderation-stable" ) print(response) diff --git a/docs/my-website/docs/observability/braintrust.md b/docs/my-website/docs/observability/braintrust.md index e6b4fe769bc..645ce074ca5 100644 --- a/docs/my-website/docs/observability/braintrust.md +++ b/docs/my-website/docs/observability/braintrust.md @@ -75,6 +75,12 @@ It is recommended that you include the `project_id` or `project_name` to ensure You can customize the span name in Braintrust logging by passing `span_name` in the metadata. By default, the span name is set to "Chat Completion". +### Custom Span Attributes + +You can customize the span id, root span name and span parents in Braintrust logging by passing `span_id`, `root_span_id` and `span_parents` in the metadata. +`span_parents` should be a string containing a list of span ids, joined by , + + diff --git a/docs/my-website/docs/observability/datadog.md b/docs/my-website/docs/observability/datadog.md index 08ebf8b28ce..5cb5ab3af2d 100644 --- a/docs/my-website/docs/observability/datadog.md +++ b/docs/my-website/docs/observability/datadog.md @@ -56,12 +56,32 @@ litellm_settings: **Step 2**: Set Required env variables for datadog +#### Direct API + +Send logs directly to Datadog API: + ```shell DD_API_KEY="5f2d0f310***********" # your datadog API Key DD_SITE="us5.datadoghq.com" # your datadog base url DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source. use to differentiate dev vs. prod deployments ``` +#### Via DataDog Agent + +Send logs through a local DataDog agent (useful for containerized environments): + +```shell +DD_AGENT_HOST="localhost" # hostname or IP of DataDog agent +DD_AGENT_PORT="10518" # [OPTIONAL] port of DataDog agent (default: 10518) +DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth) +DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source +``` + +When `DD_AGENT_HOST` is set, logs are sent to the agent instead of directly to DataDog API. This is useful for: +- Centralized log shipping in containerized environments +- Reducing direct API calls from multiple services +- Leveraging agent-side processing and filtering + **Step 3**: Start the proxy, make a test request Start proxy @@ -169,8 +189,10 @@ LiteLLM supports customizing the following Datadog environment variables | Environment Variable | Description | Default Value | Required | |---------------------|-------------|---------------|----------| -| `DD_API_KEY` | Your Datadog API key for authentication | None | ✅ Yes | -| `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") | None | ✅ Yes | +| `DD_API_KEY` | Your Datadog API key for authentication (required for direct API, optional for agent) | None | Conditional* | +| `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") (required for direct API) | None | Conditional* | +| `DD_AGENT_HOST` | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API | None | ❌ No | +| `DD_AGENT_PORT` | Port of DataDog agent for log intake | "10518" | ❌ No | | `DD_ENV` | Environment tag for your logs (e.g., "production", "staging") | "unknown" | ❌ No | | `DD_SERVICE` | Service name for your logs | "litellm-server" | ❌ No | | `DD_SOURCE` | Source name for your logs | "litellm" | ❌ No | @@ -178,3 +200,6 @@ LiteLLM supports customizing the following Datadog environment variables | `HOSTNAME` | Hostname tag for your logs | "" | ❌ No | | `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No | +\* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required +\* **Optional when using DataDog Agent**: Set `DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required + diff --git a/docs/my-website/docs/observability/opik_integration.md b/docs/my-website/docs/observability/opik_integration.md index 1ba1c2de210..d28c46f0b4b 100644 --- a/docs/my-website/docs/observability/opik_integration.md +++ b/docs/my-website/docs/observability/opik_integration.md @@ -220,7 +220,41 @@ curl --location --request POST 'http://0.0.0.0:4000/chat/completions' \ }' ``` +## Automatic Metadata from API Keys +In some cases, the requester may be unable or unaware of how to add Opik metadata to their requests. To ensure all Opik-related actions are properly tracked, LiteLLM Proxy can automatically associate metadata from a user-specific API key when none is provided in the request. + +### How It Works + +When you create an API key in LiteLLM Proxy, you can attach Opik-specific metadata to the key itself. This metadata will be automatically applied to all requests made with that key, unless the request explicitly provides its own Opik metadata (which takes precedence). + + +### Usage + +**Step 1: Save Opik Metadata to the corresponding Api Key** +Go to 'Virtual Keys', click on your choosen api key and edit 'Settings'. +Now save the opik metadata as user api key metdata. + + + +**Step 2: Use the key - Opik metadata is automatically applied** + +```bash +curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-key-from-step-1' \ +-d '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "What's the weather like in Boston today?" + } + ] +}' +``` + +All requests made with this key will automatically be tracked in the "TestProject" Opik project with the specified tags, without requiring the user to pass metadata in each request. ## Support & Talk to Founders diff --git a/docs/my-website/docs/observability/sentry.md b/docs/my-website/docs/observability/sentry.md index b7992e35c54..46b19331b24 100644 --- a/docs/my-website/docs/observability/sentry.md +++ b/docs/my-website/docs/observability/sentry.md @@ -61,6 +61,12 @@ print(response) These options are useful for high-volume applications where sampling a subset of errors and transactions provides sufficient visibility while managing costs. +#### Sentry Environment +- **SENTRY_ENVIRONMENT**: Specifies the environment name for your Sentry events (e.g., "production", "staging", "development") + - Helps organize and filter errors by deployment environment in Sentry dashboard + - Example: `os.environ["SENTRY_ENVIRONMENT"] = "staging"` + - If not set, Sentry will use 'production' as the default environment + ## Redacting Messages, Response Content from Sentry Logging Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to sentry, but request metadata will still be logged. diff --git a/docs/my-website/docs/ocr.md b/docs/my-website/docs/ocr.md new file mode 100644 index 00000000000..93cb74ee69f --- /dev/null +++ b/docs/my-website/docs/ocr.md @@ -0,0 +1,266 @@ +# /ocr + +| Feature | Supported | +|---------|-----------| +| Cost Tracking | ✅ | +| Logging | ✅ (Basic Logging not supported) | +| Load Balancing | ✅ | +| Supported Providers | `mistral`, `azure_ai`, `vertex_ai` | + +:::tip + +LiteLLM follows the [Mistral API request/response for the OCR API](https://docs.mistral.ai/capabilities/vision/#optical-character-recognition-ocr) + +::: + +## **LiteLLM Python SDK Usage** +### Quick Start + +```python +from litellm import ocr +import os + +os.environ["MISTRAL_API_KEY"] = "sk-.." + +response = ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "document_url", + "document_url": "https://arxiv.org/pdf/2201.04234" + } +) + +# Access extracted text +for page in response.pages: + print(f"Page {page.index}:") + print(page.markdown) +``` + +### Async Usage + +```python +from litellm import aocr +import os, asyncio + +os.environ["MISTRAL_API_KEY"] = "sk-.." + +async def test_async_ocr(): + response = await aocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "document_url", + "document_url": "https://arxiv.org/pdf/2201.04234" + } + ) + + # Access extracted text + for page in response.pages: + print(f"Page {page.index}:") + print(page.markdown) + +asyncio.run(test_async_ocr()) +``` + +### Using Base64 Encoded Documents + +```python +import base64 +from litellm import ocr + +# Encode PDF to base64 +with open("document.pdf", "rb") as f: + base64_pdf = base64.b64encode(f.read()).decode('utf-8') + +response = ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "document_url", + "document_url": f"data:application/pdf;base64,{base64_pdf}" + } +) +``` + +### Optional Parameters + +```python +response = ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "document_url", + "document_url": "https://example.com/doc.pdf" + }, + # Optional Mistral parameters + pages=[0, 1, 2], # Only process specific pages + include_image_base64=True, # Include extracted images + image_limit=10, # Max images to return + image_min_size=100 # Min image size to include +) +``` + +## **LiteLLM Proxy Usage** + +LiteLLM provides a Mistral API compatible `/ocr` endpoint for OCR calls. + +**Setup** + +Add this to your litellm proxy config.yaml + +```yaml +model_list: + - model_name: mistral-ocr + litellm_params: + model: mistral/mistral-ocr-latest + api_key: os.environ/MISTRAL_API_KEY +``` + +Start litellm + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +Test request + +```bash +curl http://0.0.0.0:4000/v1/ocr \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "mistral-ocr", + "document": { + "type": "document_url", + "document_url": "https://arxiv.org/pdf/2201.04234" + } + }' +``` + + +## **Request/Response Format** + +:::info + +LiteLLM follows the **Mistral OCR API specification**. + +See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilities/vision/#optical-character-recognition-ocr) for complete details. + +::: + +### Example Request + +```python +{ + "model": "mistral/mistral-ocr-latest", + "document": { + "type": "document_url", + "document_url": "https://arxiv.org/pdf/2201.04234" + }, + "pages": [0, 1, 2], # Optional: specific pages to process + "include_image_base64": True, # Optional: include extracted images + "image_limit": 10, # Optional: max images to return + "image_min_size": 100 # Optional: min image size in pixels +} +``` + +### Request Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `model` | string | Yes | The OCR model to use (e.g., `"mistral/mistral-ocr-latest"`) | +| `document` | object | Yes | Document to process. Must contain `type` and URL field | +| `document.type` | string | Yes | Either `"document_url"` for PDFs/docs or `"image_url"` for images | +| `document.document_url` | string | Conditional | URL to the document (required if `type` is `"document_url"`) | +| `document.image_url` | string | Conditional | URL to the image (required if `type` is `"image_url"`) | +| `pages` | array | No | List of specific page indices to process (0-indexed) | +| `include_image_base64` | boolean | No | Whether to include extracted images as base64 strings | +| `image_limit` | integer | No | Maximum number of images to return | +| `image_min_size` | integer | No | Minimum size (in pixels) for images to include | + +#### Document Format Examples + +**For PDFs and documents:** +```json +{ + "type": "document_url", + "document_url": "https://example.com/document.pdf" +} +``` + +**For images:** +```json +{ + "type": "image_url", + "image_url": "https://example.com/image.png" +} +``` + +**For base64-encoded content:** +```json +{ + "type": "document_url", + "document_url": "data:application/pdf;base64,JVBERi0xLjQKJ..." +} +``` + +### Response Format + +The response follows Mistral's OCR format with the following structure: + +```json +{ + "pages": [ + { + "index": 0, + "markdown": "# Document Title\n\nExtracted text content...", + "dimensions": { + "dpi": 200, + "height": 2200, + "width": 1700 + }, + "images": [ + { + "image_base64": "base64string...", + "bbox": { + "x": 100, + "y": 200, + "width": 300, + "height": 400 + } + } + ] + } + ], + "model": "mistral-ocr-2505-completion", + "usage_info": { + "pages_processed": 29, + "doc_size_bytes": 3002783 + }, + "document_annotation": null, + "object": "ocr" +} +``` + +#### Response Fields + +| Field | Type | Description | +|-------|------|-------------| +| `pages` | array | List of processed pages with extracted content | +| `pages[].index` | integer | Page number (0-indexed) | +| `pages[].markdown` | string | Extracted text in Markdown format | +| `pages[].dimensions` | object | Page dimensions (dpi, height, width in pixels) | +| `pages[].images` | array | Extracted images from the page (if `include_image_base64=true`) | +| `model` | string | The model used for OCR processing | +| `usage_info` | object | Processing statistics (pages processed, document size) | +| `document_annotation` | object | Optional document-level annotations | +| `object` | string | Always `"ocr"` for OCR responses | + + +## **Supported Providers** + +| Provider | Link to Usage | +|-------------|--------------------| +| Mistral AI | [Usage](#quick-start) | +| Azure AI | [Usage](../docs/providers/azure_ocr) | +| Vertex AI | [Usage](../docs/providers/vertex_ocr) | + diff --git a/docs/my-website/docs/pass_through/anthropic_completion.md b/docs/my-website/docs/pass_through/anthropic_completion.md index e644b7d348f..e0c7c7c5496 100644 --- a/docs/my-website/docs/pass_through/anthropic_completion.md +++ b/docs/my-website/docs/pass_through/anthropic_completion.md @@ -1,7 +1,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Anthropic SDK +# Anthropic Passthrough Pass-through endpoints for Anthropic - call provider-specific endpoint, in native format (no translation). diff --git a/docs/my-website/docs/pass_through/bedrock.md b/docs/my-website/docs/pass_through/bedrock.md index 48502864d78..b8d20d77da0 100644 --- a/docs/my-website/docs/pass_through/bedrock.md +++ b/docs/my-website/docs/pass_through/bedrock.md @@ -5,24 +5,55 @@ Pass-through endpoints for Bedrock - call provider-specific endpoint, in native | Feature | Supported | Notes | |-------|-------|-------| | Cost Tracking | ✅ | For `/invoke` and `/converse` endpoints | -| Logging | ✅ | works across all integrations | +| Load Balancing | ✅ | You can load balance `/invoke`, `/converse` routes across multiple deployments| Logging | ✅ | works across all integrations | | End-user Tracking | ❌ | [Tell us if you need this](https://github.com/BerriAI/litellm/issues/new) | | Streaming | ✅ | | Just replace `https://bedrock-runtime.{aws_region_name}.amazonaws.com` with `LITELLM_PROXY_BASE_URL/bedrock` 🚀 -#### **Example Usage** -```bash -curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' \ --H 'Authorization: Bearer anything' \ +## Overview + +LiteLLM supports two ways to call Bedrock endpoints: + +### 1. **Using config.yaml** (Recommended for model endpoints) + +Define your Bedrock models in `config.yaml` and reference them by name. The proxy handles authentication and routing. + +**Use for**: `/converse`, `/converse-stream`, `/invoke`, `/invoke-with-response-stream` + +```yaml showLineNumbers +model_list: + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + custom_llm_provider: bedrock +``` + +```bash showLineNumbers +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/converse' \ +-H 'Authorization: Bearer sk-1234' \ -H 'Content-Type: application/json' \ --d '{ - "messages": [ - {"role": "user", - "content": [{"text": "Hello"}] - } - ] -}' +-d '{"messages": [{"role": "user", "content": [{"text": "Hello"}]}]}' +``` + +### 2. **Direct passthrough** (For non-model endpoints) + +Set AWS credentials via environment variables and call Bedrock endpoints directly. + +**Use for**: Guardrails, Knowledge Bases, Agents, and other non-model endpoints + +```bash showLineNumbers +export AWS_ACCESS_KEY_ID="" +export AWS_SECRET_ACCESS_KEY="" +export AWS_REGION_NAME="us-west-2" +``` + +```bash showLineNumbers +curl "http://0.0.0.0:4000/bedrock/guardrail/my-guardrail-id/version/1/apply" \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{"contents": [{"text": {"text": "Hello"}}], "source": "INPUT"}' ``` Supports **ALL** Bedrock Endpoints (including streaming). @@ -33,39 +64,235 @@ Supports **ALL** Bedrock Endpoints (including streaming). Let's call the Bedrock [`/converse` endpoint](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html) -1. Add AWS Keys to your environment +1. Create a `config.yaml` file with your Bedrock model -```bash +```yaml showLineNumbers +model_list: + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + custom_llm_provider: bedrock +``` + +Set your AWS credentials: + +```bash showLineNumbers export AWS_ACCESS_KEY_ID="" # Access key export AWS_SECRET_ACCESS_KEY="" # Secret access key -export AWS_REGION_NAME="" # us-east-1, us-east-2, us-west-1, us-west-2 ``` 2. Start LiteLLM Proxy -```bash -litellm +```bash showLineNumbers +litellm --config config.yaml # RUNNING on http://0.0.0.0:4000 ``` 3. Test it! -Let's call the Bedrock converse endpoint +Let's call the Bedrock converse endpoint using the model name from config: -```bash -curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' \ --H 'Authorization: Bearer anything' \ +```bash showLineNumbers +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/converse' \ +-H 'Authorization: Bearer sk-1234' \ -H 'Content-Type: application/json' \ -d '{ "messages": [ - {"role": "user", - "content": [{"text": "Hello"}] + { + "role": "user", + "content": [{"text": "Hello, how are you?"}] + } + ], + "inferenceConfig": { + "maxTokens": 100 } - ] }' ``` +## Setup with config.yaml + +Use config.yaml to define Bedrock models and use them via passthrough endpoints. + +### 1. Define models in config.yaml + +```yaml showLineNumbers +model_list: + - model_name: my-claude-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + custom_llm_provider: bedrock + + - model_name: my-cohere-model + litellm_params: + model: bedrock/cohere.command-r-v1:0 + aws_region_name: us-east-1 + custom_llm_provider: bedrock +``` + +### 2. Start proxy with config + +```bash showLineNumbers +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +### 3. Call Bedrock Converse endpoint + +Use the `model_name` from config in the URL path: + +```bash showLineNumbers +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-claude-model/converse' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "messages": [ + { + "role": "user", + "content": [{"text": "Hello, how are you?"}] + } + ], + "inferenceConfig": { + "temperature": 0.5, + "maxTokens": 100 + } +}' +``` + +### 4. Call Bedrock Converse Stream endpoint + +For streaming responses, use the `/converse-stream` endpoint: + +```bash showLineNumbers +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-claude-model/converse-stream' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "messages": [ + { + "role": "user", + "content": [{"text": "Tell me a short story"}] + } + ], + "inferenceConfig": { + "temperature": 0.7, + "maxTokens": 200 + } +}' +``` + +### Supported Bedrock Endpoints with config.yaml + +When using models from config.yaml, you can call any Bedrock endpoint: + +| Endpoint | Description | Example | +|----------|-------------|---------| +| `/model/{model_name}/converse` | Converse API | `http://0.0.0.0:4000/bedrock/model/my-claude-model/converse` | +| `/model/{model_name}/converse-stream` | Streaming Converse | `http://0.0.0.0:4000/bedrock/model/my-claude-model/converse-stream` | +| `/model/{model_name}/invoke` | Legacy Invoke API | `http://0.0.0.0:4000/bedrock/model/my-claude-model/invoke` | +| `/model/{model_name}/invoke-with-response-stream` | Legacy Streaming | `http://0.0.0.0:4000/bedrock/model/my-claude-model/invoke-with-response-stream` | + +The proxy automatically resolves the `model_name` to the actual Bedrock model ID and region configured in your `config.yaml`. + +### Load Balancing Across Multiple Deployments + +Define multiple Bedrock deployments with the same `model_name` to enable automatic load balancing. + +#### 1. Define multiple deployments in config.yaml + +```yaml showLineNumbers +model_list: + # First deployment - us-west-2 + - model_name: my-claude-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + custom_llm_provider: bedrock + + # Second deployment - us-east-1 (load balanced) + - model_name: my-claude-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-east-1 + custom_llm_provider: bedrock +``` + +#### 2. Start proxy with config + +```bash showLineNumbers +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +#### 3. Call the endpoint - requests are automatically load balanced + +```bash showLineNumbers +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-claude-model/invoke' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ], + "anthropic_version": "bedrock-2023-05-31" +}' +``` + +The proxy will automatically distribute requests across both `us-west-2` and `us-east-1` deployments. This works for all Bedrock endpoints: `/invoke`, `/invoke-with-response-stream`, `/converse`, and `/converse-stream`. + +#### Using boto3 SDK with load balancing + +You can also call the load-balanced endpoint using the boto3 SDK: + +```python showLineNumbers +import boto3 +import json +import os + +# Set dummy AWS credentials (required by boto3, but not used by LiteLLM proxy) +os.environ['AWS_ACCESS_KEY_ID'] = 'dummy' +os.environ['AWS_SECRET_ACCESS_KEY'] = 'dummy' +os.environ['AWS_BEARER_TOKEN_BEDROCK'] = "sk-1234" # your litellm proxy api key + +# Point boto3 to the LiteLLM proxy +bedrock_runtime = boto3.client( + service_name='bedrock-runtime', + region_name='us-west-2', + endpoint_url='http://0.0.0.0:4000/bedrock' +) + +# Call the load-balanced model +response = bedrock_runtime.invoke_model( + modelId='my-claude-model', # Your model_name from config.yaml + contentType='application/json', + accept='application/json', + body=json.dumps({ + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ], + "anthropic_version": "bedrock-2023-05-31" + }) +) + +# Parse response +response_body = json.loads(response['body'].read()) +print(response_body['content'][0]['text']) +``` + +The proxy will automatically load balance your boto3 requests across all configured deployments. + ## Examples @@ -84,7 +311,7 @@ Key Changes: #### LiteLLM Proxy Call -```bash +```bash showLineNumbers curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' \ -H 'Authorization: Bearer sk-anything' \ -H 'Content-Type: application/json' \ @@ -99,7 +326,7 @@ curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' #### Direct Bedrock API Call -```bash +```bash showLineNumbers curl -X POST 'https://bedrock-runtime.us-west-2.amazonaws.com/model/cohere.command-r-v1:0/converse' \ -H 'Authorization: AWS4-HMAC-SHA256..' \ -H 'Content-Type: application/json' \ @@ -114,9 +341,25 @@ curl -X POST 'https://bedrock-runtime.us-west-2.amazonaws.com/model/cohere.comma ### **Example 2: Apply Guardrail** +**Setup**: Set AWS credentials for direct passthrough + +```bash showLineNumbers +export AWS_ACCESS_KEY_ID="your-access-key" +export AWS_SECRET_ACCESS_KEY="your-secret-key" +export AWS_REGION_NAME="us-west-2" +``` + +Start proxy: + +```bash showLineNumbers +litellm + +# RUNNING on http://0.0.0.0:4000 +``` + #### LiteLLM Proxy Call -```bash +```bash showLineNumbers curl "http://0.0.0.0:4000/bedrock/guardrail/guardrailIdentifier/version/guardrailVersion/apply" \ -H 'Authorization: Bearer sk-anything' \ -H 'Content-Type: application/json' \ @@ -129,7 +372,7 @@ curl "http://0.0.0.0:4000/bedrock/guardrail/guardrailIdentifier/version/guardrai #### Direct Bedrock API Call -```bash +```bash showLineNumbers curl "https://bedrock-runtime.us-west-2.amazonaws.com/guardrail/guardrailIdentifier/version/guardrailVersion/apply" \ -H 'Authorization: AWS4-HMAC-SHA256..' \ -H 'Content-Type: application/json' \ @@ -142,7 +385,25 @@ curl "https://bedrock-runtime.us-west-2.amazonaws.com/guardrail/guardrailIdentif ### **Example 3: Query Knowledge Base** -```bash +**Setup**: Set AWS credentials for direct passthrough + +```bash showLineNumbers +export AWS_ACCESS_KEY_ID="your-access-key" +export AWS_SECRET_ACCESS_KEY="your-secret-key" +export AWS_REGION_NAME="us-west-2" +``` + +Start proxy: + +```bash showLineNumbers +litellm + +# RUNNING on http://0.0.0.0:4000 +``` + +#### LiteLLM Proxy Call + +```bash showLineNumbers curl -X POST "http://0.0.0.0:4000/bedrock/knowledgebases/{knowledgeBaseId}/retrieve" \ -H 'Authorization: Bearer sk-anything' \ -H 'Content-Type: application/json' \ @@ -163,7 +424,7 @@ curl -X POST "http://0.0.0.0:4000/bedrock/knowledgebases/{knowledgeBaseId}/retri #### Direct Bedrock API Call -```bash +```bash showLineNumbers curl -X POST "https://bedrock-agent-runtime.us-west-2.amazonaws.com/knowledgebases/{knowledgeBaseId}/retrieve" \ -H 'Authorization: AWS4-HMAC-SHA256..' \ -H 'Content-Type: application/json' \ @@ -194,7 +455,7 @@ Use this, to avoid giving developers the raw AWS Keys, but still letting them us 1. Setup environment -```bash +```bash showLineNumbers export DATABASE_URL="" export LITELLM_MASTER_KEY="" export AWS_ACCESS_KEY_ID="" # Access key @@ -202,7 +463,7 @@ export AWS_SECRET_ACCESS_KEY="" # Secret access key export AWS_REGION_NAME="" # us-east-1, us-east-2, us-west-1, us-west-2 ``` -```bash +```bash showLineNumbers litellm # RUNNING on http://0.0.0.0:4000 @@ -210,7 +471,7 @@ litellm 2. Generate virtual key -```bash +```bash showLineNumbers curl -X POST 'http://0.0.0.0:4000/key/generate' \ -H 'Authorization: Bearer sk-1234' \ -H 'Content-Type: application/json' \ @@ -219,7 +480,7 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \ Expected Response -```bash +```bash showLineNumbers { ... "key": "sk-1234ewknldferwedojwojw" @@ -229,7 +490,7 @@ Expected Response 3. Test it! -```bash +```bash showLineNumbers curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' \ -H 'Authorization: Bearer sk-1234ewknldferwedojwojw' \ -H 'Content-Type: application/json' \ @@ -246,46 +507,46 @@ curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' Call Bedrock Agents via LiteLLM proxy -```python +**Setup**: Set AWS credentials on your LiteLLM proxy server + +```bash showLineNumbers +export AWS_ACCESS_KEY_ID="your-access-key" +export AWS_SECRET_ACCESS_KEY="your-secret-key" +export AWS_REGION_NAME="us-west-2" +``` + +Start proxy: + +```bash showLineNumbers +litellm + +# RUNNING on http://0.0.0.0:4000 +``` + +**Usage from Python**: + +```python showLineNumbers import os -import boto3 -from botocore.config import Config - -# # Define your proxy endpoint -proxy_endpoint = "http://0.0.0.0:4000/bedrock" # 👈 your proxy base url - -# # Create a Config object with the proxy -# Custom headers -custom_headers = { - 'litellm_user_api_key': 'Bearer sk-1234', # 👈 your proxy api key -} - - -os.environ["AWS_ACCESS_KEY_ID"] = "my-fake-key-id" -os.environ["AWS_SECRET_ACCESS_KEY"] = "my-fake-access-key" +import boto3 +# Set dummy AWS credentials (required by boto3, but not used by LiteLLM proxy) +os.environ["AWS_ACCESS_KEY_ID"] = "dummy" +os.environ["AWS_SECRET_ACCESS_KEY"] = "dummy" +os.environ["AWS_BEARER_TOKEN_BEDROCK"] = "sk-1234" # your litellm proxy api key # Create the client runtime_client = boto3.client( service_name="bedrock-agent-runtime", region_name="us-west-2", - endpoint_url=proxy_endpoint + endpoint_url="http://0.0.0.0:4000/bedrock" ) -# Custom header injection -def inject_custom_headers(request, **kwargs): - request.headers.update(custom_headers) - -# Attach the event to inject custom headers before the request is sent -runtime_client.meta.events.register('before-send.*.*', inject_custom_headers) - - response = runtime_client.invoke_agent( - agentId="L1RT58GYRW", - agentAliasId="MFPSBCXYTW", - sessionId="12345", - inputText="Who do you know?" - ) + agentId="L1RT58GYRW", + agentAliasId="MFPSBCXYTW", + sessionId="12345", + inputText="Who do you know?" +) completion = "" @@ -294,5 +555,4 @@ for event in response.get("completion"): completion += chunk["bytes"].decode() print(completion) - ``` diff --git a/docs/my-website/docs/pass_through/openai_passthrough.md b/docs/my-website/docs/pass_through/openai_passthrough.md index 27123695751..d7c98eba7b3 100644 --- a/docs/my-website/docs/pass_through/openai_passthrough.md +++ b/docs/my-website/docs/pass_through/openai_passthrough.md @@ -19,6 +19,9 @@ Simply replace `https://api.openai.com` with `LITELLM_PROXY_BASE_URL/openai` ## Usage Examples +Requirements: +Set `OPENAI_API_KEY` in your environment variables. + ### Assistants API #### Create OpenAI Client diff --git a/docs/my-website/docs/pass_through/vertex_ai.md b/docs/my-website/docs/pass_through/vertex_ai.md index 77095667113..2efef60070d 100644 --- a/docs/my-website/docs/pass_through/vertex_ai.md +++ b/docs/my-website/docs/pass_through/vertex_ai.md @@ -18,8 +18,8 @@ Pass-through endpoints for Vertex AI - call provider-specific endpoint, in nativ LiteLLM supports 3 vertex ai passthrough routes: 1. `/vertex_ai` → routes to `https://{vertex_location}-aiplatform.googleapis.com/` -2. `/vertex_ai/discovery` → routes to [`https://discoveryengine.googleapis.com`](https://discoveryengine.googleapis.com/) -3. `/vertex_ai/live` → upgrades to the Vertex AI Live API WebSocket (`google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent`) +2. `/vertex_ai/discovery` → routes to [`https://discoveryengine.googleapis.com`](https://discoveryengine.googleapis.com/) - [See Search Datastores Guide](./vertex_ai_search_datastores.md) +3. `/vertex_ai/live` → upgrades to the Vertex AI Live API WebSocket (`google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent`) - [See Live WebSocket Guide](./vertex_ai_live_websocket.md) ## How to use diff --git a/docs/my-website/docs/pass_through/vertex_ai_search_datastores.md b/docs/my-website/docs/pass_through/vertex_ai_search_datastores.md new file mode 100644 index 00000000000..20501d71f97 --- /dev/null +++ b/docs/my-website/docs/pass_through/vertex_ai_search_datastores.md @@ -0,0 +1,139 @@ +# Vertex AI Search Datastores + +Call Vertex AI Discovery Engine Search API through LiteLLM. + +Provider Doc: https://cloud.google.com/generative-ai-app-builder/docs/reference/rest/v1/projects.locations.dataStores.servingConfigs/search + +## What you get + +- Reference datastores by ID. LiteLLM finds the credentials. +- No project/location in every request. +- Configure credentials once, use everywhere. +- Cost tracking works automatically. + +## Quick Start + +**Step 1. Set credentials** + +```bash +export DEFAULT_VERTEXAI_PROJECT="your-project-id" +export DEFAULT_VERTEXAI_LOCATION="us-central1" +export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="/path/to/credentials.json" +``` + +**Step 2. Start proxy** + +```bash +litellm +``` + +**Step 3. Search your datastore** + +```bash +curl -X POST \ + "http://localhost:4000/vertex_ai/discovery/v1/projects/my-project/locations/global/collections/default_collection/dataStores/my-datastore/servingConfigs/default_config:search" \ + -H "Content-Type: application/json" \ + -H "x-litellm-api-key: Bearer sk-1234" \ + -d '{ + "query": "How do I authenticate?", + "pageSize": 10 + }' +``` + +## Managed Vector Stores (Recommended) + +Register your datastore once. Reference it by ID. + +**In config.yaml:** + +```yaml +vector_store_registry: + - vector_store_name: "vertex-ai-litellm-website-knowledgebase" + litellm_params: + vector_store_id: "my-datastore" + custom_llm_provider: "vertex_ai/search_api" + vertex_app_id: "test-litellm-app_1761094730750" + vertex_project: "test-vector-store-db" + vertex_location: "global" + vector_store_description: "Vertex AI vector store for the Litellm website knowledgebase" + vector_store_metadata: + source: "https://www.litellm.com/docs" +``` + +**How it works:** + +LiteLLM sees `dataStores/my-datastore` in your URL. It looks up the vector store. Uses the right project and credentials automatically. + +## Endpoint + +`{PROXY_BASE_URL}/vertex_ai/discovery/{endpoint:path}` + +Routes to `https://discoveryengine.googleapis.com` + +## Examples + +### Basic Search + +```bash +curl -X POST \ + "http://localhost:4000/vertex_ai/discovery/v1/projects/my-project/locations/global/collections/default_collection/dataStores/my-datastore/servingConfigs/default_config:search" \ + -H "Content-Type: application/json" \ + -H "x-litellm-api-key: Bearer sk-1234" \ + -d '{ + "query": "pricing", + "pageSize": 10 + }' +``` + +### Search with Filters + +```bash +curl -X POST \ + "http://localhost:4000/vertex_ai/discovery/v1/projects/my-project/locations/global/collections/default_collection/dataStores/my-datastore/servingConfigs/default_config:search" \ + -H "Content-Type: application/json" \ + -H "x-litellm-api-key: Bearer sk-1234" \ + -d '{ + "query": "tutorials", + "pageSize": 20, + "filter": "category = \"beginner\"", + "spellCorrectionSpec": {"mode": "AUTO"} + }' +``` + +### Python + +```python +import requests + +url = "http://localhost:4000/vertex_ai/discovery/v1/projects/my-project/locations/global/collections/default_collection/dataStores/my-datastore/servingConfigs/default_config:search" + +response = requests.post(url, + headers={ + "Content-Type": "application/json", + "x-litellm-api-key": "Bearer sk-1234" + }, + json={"query": "pricing", "pageSize": 10} +) + +for result in response.json().get("results", []): + data = result["document"]["derivedStructData"] + print(f"{data['title']}: {data['link']}") +``` + +### Use with Chat Completion + +```bash +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -d '{ + "model": "claude-3-5-sonnet", + "messages": [{"role": "user", "content": "What is litellm?"}], + "tools": [ + { + "type": "file_search", + "vector_store_ids": ["my-datastore"] + } + ] + }' +``` \ No newline at end of file diff --git a/docs/my-website/docs/projects/Softgen.md b/docs/my-website/docs/projects/Softgen.md new file mode 100644 index 00000000000..2e5024a0770 --- /dev/null +++ b/docs/my-website/docs/projects/Softgen.md @@ -0,0 +1,7 @@ +# Softgen + +`Softgen` is an AI-powered platform that builds full-stack web apps from your plain instructions. +LiteLLM helps `Softgen` users to choose and use different LLMs. + +- [Softgen](https://softgen.ai) +- [Academy](hhttps://academy.softgen.ai) diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index 1663d32ddfc..0ea042e5d98 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -953,7 +953,7 @@ except Exception as e: s/o @[Shekhar Patnaik](https://www.linkedin.com/in/patnaikshekhar) for requesting this! -### Anthropic Hosted Tools (Computer, Text Editor, Web Search) +### Anthropic Hosted Tools (Computer, Text Editor, Web Search, Memory) @@ -1183,6 +1183,72 @@ curl http://0.0.0.0:4000/v1/chat/completions \ + + + +:::info +The Anthropic Memory tool is currently in beta. +::: + + + + +```python +from litellm import completion + +tools = [{ + "type": "memory_20250818", + "name": "memory" +}] + +model = "claude-sonnet-4-5-20250929" +messages = [{"role": "user", "content": "Please remember that my favorite color is blue."}] + +response = completion( + model=model, + messages=messages, + tools=tools, +) + +print(response) +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-memory-model + litellm_params: + model: anthropic/claude-sonnet-4-5-20250929 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d '{ + "model": "claude-memory-model", + "messages": [{"role": "user", "content": "Please remember that my favorite color is blue."}], + "tools": [{"type": "memory_20250818", "name": "memory"}] + }' +``` + + + + + diff --git a/docs/my-website/docs/providers/azure/azure.md b/docs/my-website/docs/providers/azure/azure.md index 1feec52b3ec..2f845357328 100644 --- a/docs/my-website/docs/providers/azure/azure.md +++ b/docs/my-website/docs/providers/azure/azure.md @@ -11,7 +11,7 @@ import TabItem from '@theme/TabItem'; |-------|-------| | Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series | | Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models) | -| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](#azure-text-to-speech-tts), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) | +| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](azure_speech), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) | | Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview) ## API Keys, Params @@ -538,39 +538,6 @@ response = litellm.completion( print(response) ``` -## Azure Text to Speech (tts) - -**LiteLLM PROXY** - -```yaml - - model_name: azure/tts-1 - litellm_params: - model: azure/tts-1 - api_base: "os.environ/AZURE_API_BASE_TTS" - api_key: "os.environ/AZURE_API_KEY_TTS" - api_version: "os.environ/AZURE_API_VERSION" -``` - -**LiteLLM SDK** - -```python -from litellm import completion - -## set ENV variables -os.environ["AZURE_API_KEY"] = "" -os.environ["AZURE_API_BASE"] = "" -os.environ["AZURE_API_VERSION"] = "" - -# azure call -speech_file_path = Path(__file__).parent / "speech.mp3" -response = speech( - model="azure/ ```python -client.batches.list(extra_query={"custom_llm_provider": "azure"}) +client.batches.list(extra_headers={"custom-llm-provider": "azure"}) ``` diff --git a/docs/my-website/docs/providers/azure/azure_speech.md b/docs/my-website/docs/providers/azure/azure_speech.md new file mode 100644 index 00000000000..3bcc3ab931f --- /dev/null +++ b/docs/my-website/docs/providers/azure/azure_speech.md @@ -0,0 +1,75 @@ +# Azure Text to Speech (tts) + +## Overview + +| Property | Details | +|-------|-------| +| Description | Convert text to natural-sounding speech using Azure OpenAI's Text to Speech models | +| Provider Route on LiteLLM | `azure/` | +| Supported Operations | `/audio/speech` | +| Link to Provider Doc | [Azure OpenAI TTS ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/text-to-speech-quickstart) + +## Quick Start + +### **LiteLLM SDK** + +```python showLineNumbers title="SDK Usage" +from litellm import speech +from pathlib import Path +import os + +## set ENV variables +os.environ["AZURE_API_KEY"] = "" +os.environ["AZURE_API_BASE"] = "" +os.environ["AZURE_API_VERSION"] = "" + +# azure call +speech_file_path = Path(__file__).parent / "speech.mp3" +response = speech( + model="azure/", + voice="alloy", + input="the quick brown fox jumped over the lazy dogs", + ) +response.stream_to_file(speech_file_path) +``` + +### **LiteLLM PROXY** + +```yaml showLineNumbers title="proxy_config.yaml" +model_list: + - model_name: azure/tts-1 + litellm_params: + model: azure/tts-1 + api_base: "os.environ/AZURE_API_BASE_TTS" + api_key: "os.environ/AZURE_API_KEY_TTS" + api_version: "os.environ/AZURE_API_VERSION" +``` + +## Available Voices + +Azure OpenAI supports the following voices: +- `alloy` - Neutral and balanced +- `echo` - Warm and upbeat +- `fable` - Expressive and dramatic +- `onyx` - Deep and authoritative +- `nova` - Friendly and conversational +- `shimmer` - Bright and cheerful + +## Supported Parameters + +```python showLineNumbers title="All Parameters" +response = speech( + model="azure/", + voice="alloy", # Required: Voice selection + input="text to convert", # Required: Input text + speed=1.0, # Optional: 0.25 to 4.0 (default: 1.0) + response_format="mp3" # Optional: mp3, opus, aac, flac, wav, pcm +) +``` + +## Supported Models + +- `tts-1` - Standard quality, optimized for speed +- `tts-1-hd` - High definition, optimized for quality + +Use your Azure deployment name: `azure/` \ No newline at end of file diff --git a/docs/my-website/docs/providers/azure/videos.md b/docs/my-website/docs/providers/azure/videos.md new file mode 100644 index 00000000000..62f8d0df182 --- /dev/null +++ b/docs/my-website/docs/providers/azure/videos.md @@ -0,0 +1,282 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Azure Video Generation + +LiteLLM supports Azure OpenAI's video generation models including Sora with full end-to-end integration. + +| Property | Details | +|-------|-------| +| Description | Azure OpenAI's video generation models including Sora-2 | +| Provider Route on LiteLLM | `azure/` | +| Supported Models | `sora-2` | +| Cost Tracking | ✅ Duration-based pricing ($0.10/second) | +| Logging Support | ✅ Full request/response logging | +| Guardrails Support | ✅ Content moderation and safety checks | +| Proxy Server Support | ✅ Full proxy integration with virtual keys | +| Spend Management | ✅ Budget tracking and rate limiting | +| Link to Provider Doc | [Azure OpenAI Video Generation ↗](https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/video-generation) | + +## Quick Start + +### Required API Keys + +```python +import os +os.environ["AZURE_OPENAI_API_KEY"] = "your-azure-api-key" +os.environ["AZURE_OPENAI_API_BASE"] = "https://your-resource.openai.azure.com/" +``` + +### Basic Usage + +```python +from litellm import video_generation, video_status, video_content +import os +import time + +os.environ["AZURE_OPENAI_API_KEY"] = "your-azure-api-key" +os.environ["AZURE_OPENAI_API_BASE"] = "https://your-resource.openai.azure.com/" + +# Generate video +response = video_generation( + model="azure/sora-2", + prompt="A cat playing with a ball of yarn in a sunny garden", + seconds="8", + size="720x1280" +) + +print(f"Video ID: {response.id}") +print(f"Initial Status: {response.status}") + +# Check status until video is ready +while True: + status_response = video_status( + video_id=response.id + ) + + print(f"Current Status: {status_response.status}") + + if status_response.status == "completed": + break + elif status_response.status == "failed": + print("Video generation failed") + break + + time.sleep(10) # Wait 10 seconds before checking again + +# Download video content when ready +video_bytes = video_content( + video_id=response.id +) + +# Save to file +with open("generated_video.mp4", "wb") as f: + f.write(video_bytes) +``` + +## Usage - LiteLLM Proxy Server + +Here's how to call Azure video generation models with the LiteLLM Proxy Server + +### 1. Save key in your environment + +```bash +export AZURE_OPENAI_API_KEY="your-azure-api-key" +export AZURE_OPENAI_API_BASE="https://your-resource.openai.azure.com/" +``` + +### 2. Start the proxy + + + + +```yaml +model_list: + - model_name: azure-sora-2 + litellm_params: + model: azure/sora-2 + api_key: os.environ/AZURE_OPENAI_API_KEY + api_base: os.environ/AZURE_OPENAI_API_BASE +``` + + + + +```bash +$ litellm --model azure/sora-2 + +# Server running on http://0.0.0.0:4000 +``` + + + + + +### 3. Test it + + + + +```shell +curl --location 'http://0.0.0.0:4000/videos/generations' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer sk-1234' \ +--data '{ + "model": "azure-sora-2", + "prompt": "A cat playing with a ball of yarn in a sunny garden", + "seconds": "8", + "size": "720x1280" +}' +``` + + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.videos.create( + model="azure-sora-2", + prompt="A cat playing with a ball of yarn in a sunny garden", + seconds=8, + size="720x1280" +) + +print(response) +``` + + + + +## Supported Models + +| Model Name | +|------------| +| sora-2 | +|sora-2-pro | +|sora-2-pro-high-res| + + +## Logging & Observability + +### Request/Response Logging + +All video generation requests are automatically logged with: + +- **Request details**: prompt, model, duration, size +- **Response details**: video ID, status, creation time +- **Cost tracking**: duration-based pricing calculation +- **Performance metrics**: request latency, processing time + +### Logging Providers + +Video generation works with all LiteLLM logging providers: + +- **Datadog**: Real-time monitoring and alerting +- **Helicone**: Request tracing and debugging +- **LangSmith**: LangChain integration and tracing +- **Custom webhooks**: Send logs to your own endpoints + +**Example: Enable Datadog logging** + +```yaml +general_settings: + alerting: ["datadog"] + datadog_api_key: os.environ/DATADOG_API_KEY +``` + + +## Video Generation Parameters + +- `prompt` (required): Text description of the desired video +- `model` (optional): Model to use, defaults to "azure/sora-2" +- `seconds` (optional): Video duration in seconds (e.g., "8", "16") +- `size` (optional): Video dimensions (e.g., "720x1280", "1280x720") +- `input_reference` (optional): Reference image for video editing +- `user` (optional): User identifier for tracking + +## Video Content Retrieval + +```python +# Download video content +video_bytes = video_content( + video_id="video_1234567890" +) + +# Save to file +with open("video.mp4", "wb") as f: + f.write(video_bytes) +``` + +## Complete Workflow + +```python +import litellm +import time + +def generate_and_download_video(prompt): + # Step 1: Generate video + response = litellm.video_generation( + prompt=prompt, + model="azure/sora-2", + seconds="8", + size="720x1280" + ) + + video_id = response.id + print(f"Video ID: {video_id}") + + # Step 2: Wait for processing (in practice, poll status) + time.sleep(30) + + # Step 3: Download video + video_bytes = litellm.video_content( + video_id=video_id + ) + + # Step 4: Save to file + with open(f"video_{video_id}.mp4", "wb") as f: + f.write(video_bytes) + + return f"video_{video_id}.mp4" + +# Usage +video_file = generate_and_download_video( + "A cat playing with a ball of yarn in a sunny garden" +) +``` + +## Video Remix (Video Editing) + +```python +# Video editing with reference image +response = litellm.video_remix( + video_id="video_456", + prompt="Make the cat jump higher", + input_reference=open("path/to/image.jpg", "rb"), # Reference image as file object + seconds="8" +) + +print(f"Video ID: {response.id}") +``` + +## Error Handling + +```python +from litellm.exceptions import BadRequestError, AuthenticationError + +try: + response = video_generation( + prompt="A cat playing with a ball of yarn", + model="azure/sora-2" + ) +except AuthenticationError as e: + print(f"Authentication failed: {e}") +except BadRequestError as e: + print(f"Bad request: {e}") +``` diff --git a/docs/my-website/docs/providers/azure_ai/azure_ai_vector_stores_passthrough.md b/docs/my-website/docs/providers/azure_ai/azure_ai_vector_stores_passthrough.md new file mode 100644 index 00000000000..a528b1ccfcf --- /dev/null +++ b/docs/my-website/docs/providers/azure_ai/azure_ai_vector_stores_passthrough.md @@ -0,0 +1,391 @@ +# Azure AI Search - Vector Store (Passthrough API) + +Use this to allow developers to **create** and **search** vector stores using the Azure AI Search API in the **native** Azure AI Search API format, without giving them the Azure AI credentials. + +This is for the proxy only. + +## Admin Flow + +### 1. Add the vector store to LiteLLM + +```yaml +model_list: + - model_name: embedding-model + litellm_params: + model: openai/text-embedding-3-large + + +vector_store_registry: + - vector_store_name: "azure-ai-search" + litellm_params: + vector_store_id: "can-be-anything" # vector store id can be anything for the purpose of passthrough api + custom_llm_provider: "azure_ai" + api_key: os.environ/AZURE_SEARCH_API_KEY + api_base: https://azure-kb-search.search.windows.net + litellm_embedding_model: "azure/text-embedding-3-large" + litellm_embedding_config: + api_base: https://krris-mh44uf7y-eastus2.cognitiveservices.azure.com/ + api_key: os.environ/AZURE_API_KEY + api_version: "2025-09-01" + +general_settings: + database_url: "postgresql://user:password@host:port/database" + master_key: "sk-1234" +``` + +Add your vector store credentials to LiteLLM. + +### 2. Start the proxy. + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +### 3. Create a virtual index. + +```bash +curl -L -X POST 'http://0.0.0.0:4000/v1/indexes' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "index_name": "dall-e-4", + "litellm_params": { + "vector_store_index": "real-index-name-2", + "vector_store_name": "azure-ai-search" + } + +}' +``` + +This is a virtual index, which the developer can use to create and search vector stores. + +### 4. Create a key with the vector store permissions. + +```bash +curl -L -X POST 'http://0.0.0.0:4000/key/generate' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "allowed_vector_store_indexes": [{"index_name": "dall-e-4", "index_permissions": ["write", "read"]}], + "models": ["embedding-model"] +}' +``` + +Give the key access to the virtual index and the embedding model. + +**Expected response** + +```json +{ + "key": "sk-my-virtual-key" +} +``` + +## Developer Flow + +### 1. Create a vector store with some documents. + +Note: Use the '/azure_ai' endpoint for the passthrough api that uses the `azure_ai` provider in your `_new_secret_config.yaml` file. + +```python +import requests +import json + +# ---------------------------- +# 🔐 CONFIGURATION +# ---------------------------- +# Azure OpenAI (for embeddings) +AZURE_OPENAI_ENDPOINT = "http://0.0.0.0:4000" +AZURE_OPENAI_KEY = "sk-my-virtual-key" +EMBEDDING_DEPLOYMENT_NAME = "embedding-model" + +# Azure AI Search +AZURE_AI_SEARCH_ENDPOINT = "http://0.0.0.0:4000/azure_ai" # IMPORTANT: Use the '/azure_ai' endpoint for the passthrough api to Azure +SEARCH_API_KEY = "sk-my-virtual-key" +INDEX_NAME = "dall-e-4" + + + +# Vector dimensions (text-embedding-3-large uses 3072 dimensions) +VECTOR_DIMENSIONS = 3072 + +# Example docs (replace with your own) +documents = [ + {"id": "1", "content": "Refunds must be requested within 30 days."}, + {"id": "2", "content": "We offer 24/7 support for all enterprise customers."}, +] + + +# ---------------------------- +# 📋 STEP 0 — Create Index Schema +# ---------------------------- +def delete_index_if_exists(): + """Delete the index if it exists""" + index_url = f"{AZURE_AI_SEARCH_ENDPOINT}/indexes/{INDEX_NAME}?api-version=2024-07-01" + headers = {"api-key": SEARCH_API_KEY} + + response = requests.delete(index_url, headers=headers) + + if response.status_code == 204: + print(f"🗑️ Deleted existing index '{INDEX_NAME}'") + return True + elif response.status_code == 404: + print(f"ℹ️ Index '{INDEX_NAME}' does not exist yet") + return False + else: + print(f"⚠️ Delete response: {response.status_code}") + print(f" Message: {response.text}") + return False + + +def create_index(): + """Create the Azure AI Search index with proper schema""" + index_url = f"{AZURE_AI_SEARCH_ENDPOINT}/indexes/{INDEX_NAME}?api-version=2024-07-01" + headers = {"Content-Type": "application/json", "api-key": SEARCH_API_KEY} + + index_schema = { + "name": INDEX_NAME, + "fields": [ + {"name": "id", "type": "Edm.String", "key": True, "filterable": True}, + { + "name": "content", + "type": "Edm.String", + "searchable": True, + "filterable": False, + }, + { + "name": "contentVector", + "type": "Collection(Edm.Single)", + "searchable": True, + "dimensions": VECTOR_DIMENSIONS, + "vectorSearchProfile": "my-vector-profile", + }, + ], + "vectorSearch": { + "algorithms": [ + { + "name": "my-hnsw-algorithm", + "kind": "hnsw", + "hnswParameters": { + "metric": "cosine", + "m": 4, + "efConstruction": 400, + "efSearch": 500, + }, + } + ], + "profiles": [ + {"name": "my-vector-profile", "algorithm": "my-hnsw-algorithm"} + ], + }, + } + + # Create the index + response = requests.put(index_url, headers=headers, json=index_schema) + + if response.status_code == 201: + print(f"✅ Index '{INDEX_NAME}' created successfully.") + return True + elif response.status_code == 204: + print(f"✅ Index '{INDEX_NAME}' updated successfully.") + return True + else: + print(f"❌ Failed to create index: {response.status_code}") + print(f" Message: {response.text}") + return False + + +# Delete and recreate the index with correct schema +print("🔄 Setting up Azure AI Search index...") +delete_index_if_exists() +if not create_index(): + print("❌ Could not create index. Exiting.") + exit(1) + + +# ---------------------------- +# 🧠 STEP 1 — Generate Embeddings +# ---------------------------- +def get_embedding(text: str): + url = f"{AZURE_OPENAI_ENDPOINT}/openai/deployments/{EMBEDDING_DEPLOYMENT_NAME}/embeddings?api-version=2024-10-21" + headers = {"Content-Type": "application/json", "api-key": AZURE_OPENAI_KEY} + payload = {"input": text} + response = requests.post(url, headers=headers, json=payload) + + if response.status_code != 200: + raise Exception(f"Embedding failed: {response.status_code}\n{response.text}") + return response.json()["data"][0]["embedding"] + + +# Generate embeddings for each document +for doc in documents: + doc["contentVector"] = get_embedding(doc["content"]) + print(f"✅ Embedded doc {doc['id']} (vector length: {len(doc['contentVector'])})") + +# ---------------------------- +# 📤 STEP 2 — Upload to Azure AI Search +# ---------------------------- +upload_url = f"{AZURE_AI_SEARCH_ENDPOINT}/indexes/{INDEX_NAME}/docs/index?api-version=2024-07-01" +headers = {"Content-Type": "application/json", "api-key": SEARCH_API_KEY} + +payload = { + "value": [ + { + "@search.action": "upload", + "id": doc["id"], + "content": doc["content"], + "contentVector": doc["contentVector"], + } + for doc in documents + ] +} + +response = requests.post(upload_url, headers=headers, data=json.dumps(payload)) + +# ---------------------------- +# 🧾 RESULT +# ---------------------------- +if response.status_code == 200: + print("✅ Documents uploaded successfully.") +else: + print(f"❌ Upload failed: {response.status_code}") + print(response.text) + +``` + + +### 2. Search the vector store. + + +```python +import requests +import json + +# ---------------------------- +# 🔐 CONFIGURATION +# ---------------------------- +# Azure OpenAI (for embeddings) +AZURE_OPENAI_ENDPOINT = "http://0.0.0.0:4000" +AZURE_OPENAI_KEY = "sk-my-virtual-key" +EMBEDDING_DEPLOYMENT_NAME = "embedding-model" + +# Azure AI Search +AZURE_AI_SEARCH_ENDPOINT = "http://0.0.0.0:4000/azure_ai" +SEARCH_API_KEY = "sk-my-virtual-key" +INDEX_NAME = "dall-e-4" + + +# ---------------------------- +# 🧠 Generate Query Embedding +# ---------------------------- +def get_embedding(text: str): + """Generate embedding for the query text""" + url = f"{AZURE_OPENAI_ENDPOINT}/openai/deployments/{EMBEDDING_DEPLOYMENT_NAME}/embeddings?api-version=2024-10-21" + headers = {"Content-Type": "application/json", "api-key": AZURE_OPENAI_KEY} + payload = {"input": text} + response = requests.post(url, headers=headers, json=payload) + + if response.status_code != 200: + raise Exception(f"Embedding failed: {response.status_code}\n{response.text}") + return response.json()["data"][0]["embedding"] + + +# ---------------------------- +# 🔍 Vector Search Function +# ---------------------------- +def search_knowledge_base(query: str, top_k: int = 3): + """ + Search the knowledge base using vector similarity + + Args: + query: The search query string + top_k: Number of top results to return (default: 3) + + Returns: + List of search results with content and scores + """ + print(f"🔍 Searching for: '{query}'") + + # Step 1: Generate embedding for the query + print(" Generating query embedding...") + query_vector = get_embedding(query) + + # Step 2: Perform vector search + search_url = f"{AZURE_AI_SEARCH_ENDPOINT}/indexes/{INDEX_NAME}/docs/search?api-version=2024-07-01" + headers = {"Content-Type": "application/json", "api-key": SEARCH_API_KEY} + + # Build the search request with vector search + search_payload = { + "search": "*", # Get all documents + "vectorQueries": [ + { + "vector": query_vector, + "fields": "contentVector", + "kind": "vector", + "k": top_k, # Number of nearest neighbors to return + } + ], + "select": "id,content", # Fields to return + "top": top_k, + } + + # Execute the search + response = requests.post(search_url, headers=headers, json=search_payload) + + if response.status_code != 200: + raise Exception(f"Search failed: {response.status_code}\n{response.text}") + + # Parse and return results + results = response.json() + return results.get("value", []) + + +# ---------------------------- +# 📊 Display Results +# ---------------------------- +def display_results(results): + """Pretty print the search results""" + if not results: + print("\n❌ No results found.") + return + + print(f"\n✅ Found {len(results)} results:\n") + print("=" * 80) + + for i, result in enumerate(results, 1): + print(f"\n📄 Result #{i}") + print(f" ID: {result.get('id', 'N/A')}") + print(f" Score: {result.get('@search.score', 'N/A')}") + print(f" Content: {result.get('content', 'N/A')}") + print("-" * 80) + + +# ---------------------------- +# 🎯 MAIN - Example Queries +# ---------------------------- +if __name__ == "__main__": + # Example 1: Search for refund policy + print("\n" + "=" * 80) + print("EXAMPLE 1: Refund Policy Query") + print("=" * 80) + results = search_knowledge_base("How do I get a refund?", top_k=2) + display_results(results) + + # Example 2: Search for customer support + print("\n\n" + "=" * 80) + print("EXAMPLE 2: Customer Support Query") + print("=" * 80) + results = search_knowledge_base("When can I contact support?", top_k=2) + display_results(results) + + # Example 3: Custom query - uncomment to use + # print("\n\n" + "=" * 80) + # print("CUSTOM QUERY") + # print("=" * 80) + # custom_query = input("Enter your query: ") + # results = search_knowledge_base(custom_query, top_k=3) + # display_results(results) + +``` \ No newline at end of file diff --git a/docs/my-website/docs/providers/azure_ai_speech.md b/docs/my-website/docs/providers/azure_ai_speech.md new file mode 100644 index 00000000000..434a796a2fb --- /dev/null +++ b/docs/my-website/docs/providers/azure_ai_speech.md @@ -0,0 +1,374 @@ +# Azure AI Speech (Cognitive Services) + +Azure AI Speech is Azure's Cognitive Services text-to-speech API, separate from Azure OpenAI. It provides high-quality neural voices with broader language support and advanced speech customization. + +**When to use this vs Azure OpenAI TTS:** +- **Azure AI Speech** - More languages, neural voices, SSML support, speech customization +- **Azure OpenAI TTS** - OpenAI models, integrated with Azure OpenAI services + + +## Overview + +| Property | Details | +|-------|-------| +| Description | Azure AI Speech is Azure's Cognitive Services text-to-speech API, separate from Azure OpenAI. It provides high-quality neural voices with broader language support and advanced speech customization. | +| Provider Route on LiteLLM | `azure/speech/` | + +## Quick Start + +**LiteLLM SDK** + +```python showLineNumbers title="SDK Usage" +from litellm import speech +from pathlib import Path +import os + +os.environ["AZURE_TTS_API_KEY"] = "your-cognitive-services-key" + +speech_file_path = Path(__file__).parent / "speech.mp3" +response = speech( + model="azure/speech/azure-tts", + voice="alloy", + input="Hello, this is Azure AI Speech", + api_base="https://eastus.tts.speech.microsoft.com", + api_key=os.environ["AZURE_TTS_API_KEY"], +) +response.stream_to_file(speech_file_path) +``` + +**LiteLLM Proxy** + +```yaml showLineNumbers title="proxy_config.yaml" +model_list: + - model_name: azure-speech + litellm_params: + model: azure/speech/azure-tts + api_base: https://eastus.tts.speech.microsoft.com + api_key: os.environ/AZURE_TTS_API_KEY +``` + +## Setup + +1. Create an Azure Cognitive Services resource in the [Azure Portal](https://portal.azure.com) +2. Get your API key from the resource +3. Note your region (e.g., `eastus`, `westus`, `westeurope`) +4. Use the regional endpoint: `https://{region}.tts.speech.microsoft.com` + +## Cost Tracking (Pricing) + +LiteLLM automatically tracks costs for Azure AI Speech based on the number of characters processed. + +### Available Models + +| Model | Voice Type | Cost per 1M Characters | +|-------|-----------|----------------------| +| `azure/speech/azure-tts` | Neural | $15 | +| `azure/speech/azure-tts-hd` | Neural HD | $30 | + +### How Costs are Calculated + +Azure AI Speech charges based on the number of characters in your input text. LiteLLM automatically: +- Counts the number of characters in your `input` parameter +- Calculates the cost based on the model pricing +- Returns the cost in the response object + +```python showLineNumbers title="View Request Cost" +from litellm import speech + +response = speech( + model="azure/speech/azure-tts", + voice="alloy", + input="Hello, this is a test message", + api_base="https://eastus.tts.speech.microsoft.com", + api_key=os.environ["AZURE_TTS_API_KEY"], +) + +# Access the calculated cost +cost = response._hidden_params.get("response_cost") +print(f"Request cost: ${cost}") +``` + +### Verify Azure Pricing + +To check the latest Azure AI Speech pricing: + +1. Visit the [Azure Pricing Calculator](https://azure.microsoft.com/en-us/pricing/calculator/) +2. Set **Service** to "AI Services" +3. Set **API** to "Azure AI Speech" +4. Select **Text to Speech** and your region +5. View the current pricing per million characters + +**Note:** Pricing may vary by region and Azure subscription type. + +## Voice Mapping + +LiteLLM automatically maps OpenAI voice names to Azure Neural voices: + +| OpenAI Voice | Azure Neural Voice | Description | +|-------------|-------------------|-------------| +| `alloy` | en-US-JennyNeural | Neutral and balanced | +| `echo` | en-US-GuyNeural | Warm and upbeat | +| `fable` | en-GB-RyanNeural | Expressive and dramatic | +| `onyx` | en-US-DavisNeural | Deep and authoritative | +| `nova` | en-US-AmberNeural | Friendly and conversational | +| `shimmer` | en-US-AriaNeural | Bright and cheerful | + +## Supported Parameters + +```python showLineNumbers title="All Parameters" +response = speech( + model="azure/speech/azure-tts", + voice="alloy", # Required: Voice selection + input="text to convert", # Required: Input text + speed=1.0, # Optional: 0.25 to 4.0 (default: 1.0) + response_format="mp3", # Optional: mp3, opus, wav, pcm + api_base="https://eastus.tts.speech.microsoft.com", + api_key="your-key", +) +``` + +### Response Formats + +| Format | Azure Output Format | Sample Rate | +|--------|-------------------|-------------| +| `mp3` | audio-24khz-48kbitrate-mono-mp3 | 24kHz | +| `opus` | ogg-48khz-16bit-mono-opus | 48kHz | +| `wav` | riff-24khz-16bit-mono-pcm | 24kHz | +| `pcm` | raw-24khz-16bit-mono-pcm | 24kHz | + +## Sending Azure-Specific Params + +Azure AI Speech supports advanced SSML features through optional parameters: + +- `style`: Speaking style (e.g., "cheerful", "sad", "angry", "whispering") +- `styledegree`: Style intensity (0.01 to 2) +- `role`: Voice role (e.g., "Girl", "Boy", "SeniorFemale", "SeniorMale") +- `lang`: Language code for multilingual voices (e.g., "es-ES", "fr-FR", "hi-IN") + +### **LiteLLM SDK** + +#### Custom Azure Voice + +```python showLineNumbers title="Custom Azure Voice" +from litellm import speech + +response = speech( + model="azure/speech/azure-tts", + voice="en-US-AndrewNeural", # Use Azure voice directly + input="Hello, this is a test", + api_base="https://eastus.tts.speech.microsoft.com", + api_key=os.environ["AZURE_TTS_API_KEY"], + response_format="mp3" +) +response.stream_to_file("speech.mp3") +``` + +#### Speaking Style + +```python showLineNumbers title="Speaking Style" +from litellm import speech + +response = speech( + model="azure/speech/azure-tts", + voice="en-US-JennyNeural", # Must be a voice that supports styles + input="Who are you? What is chicken dinner?", + api_base="https://eastus.tts.speech.microsoft.com", + api_key=os.environ["AZURE_TTS_API_KEY"], + style="whispering", # Azure-specific: cheerful, sad, angry, whispering, etc. +) +response.stream_to_file("speech.mp3") +``` + +#### Style with Degree and Role + +```python showLineNumbers title="Style with Degree and Role" +from litellm import speech + +response = speech( + model="azure/speech/azure-tts", + voice="en-US-AriaNeural", + input="Good morning! How are you today?", + api_base="https://eastus.tts.speech.microsoft.com", + api_key=os.environ["AZURE_TTS_API_KEY"], + style="cheerful", # Azure-specific: Speaking style + styledegree="2", # Azure-specific: 0.01 to 2 (intensity) + role="SeniorFemale", # Azure-specific: Girl, Boy, SeniorFemale, etc. +) +response.stream_to_file("speech.mp3") +``` + +#### Language Override for Multilingual Voices + +```python showLineNumbers title="Language Override" +from litellm import speech + +response = speech( + model="azure/speech/azure-tts", + voice="en-US-AvaMultilingualNeural", # Multilingual voice + input="आप कौन हैं? चिकन डिनर क्या है?", # Hindi text + api_base="https://eastus.tts.speech.microsoft.com", + api_key=os.environ["AZURE_TTS_API_KEY"], + lang="hi-IN", # Azure-specific: Override language +) +response.stream_to_file("speech.mp3") +``` + +### **LiteLLM AI Gateway (CURL)** + +First, ensure you have set up your proxy config as shown in the [LiteLLM Proxy setup](#quick-start) above. + +**Using the model name from your config:** + +```yaml +model_list: + - model_name: azure-speech # This is what you'll use in your API calls + litellm_params: + model: azure/speech/azure-tts + api_base: https://eastus.tts.speech.microsoft.com + api_key: os.environ/AZURE_TTS_API_KEY +``` + +#### Custom Azure Voice + +```bash +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "azure-speech", + "voice": "en-US-AndrewNeural", + "input": "Hello, this is a test" + }' \ + --output speech.mp3 +``` + +#### Speaking Style + +```bash +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "azure-speech", + "input": "Who are you? What is chicken dinner?", + "voice": "en-US-JennyNeural", + "style": "whispering" + }' \ + --output speech.mp3 +``` + +#### Style with Degree and Role + +```bash +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "azure-speech", + "voice": "en-US-AriaNeural", + "input": "Good morning! How are you today?", + "style": "cheerful", + "styledegree": "2", + "role": "SeniorFemale" + }' \ + --output speech.mp3 +``` + +#### Language Override + +```bash +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "azure-speech", + "input": "आप कौन हैं? चिकन डिनर क्या है?", + "voice": "en-US-AvaMultilingualNeural", + "lang": "hi-IN" + }' \ + --output speech.mp3 +``` + +### Azure-Specific Parameters Reference + +| Parameter | Description | Example Values | Notes | +|-----------|-------------|----------------|-------| +| `style` | Speaking style | `cheerful`, `sad`, `angry`, `excited`, `friendly`, `hopeful`, `shouting`, `terrified`, `unfriendly`, `whispering` | Only supported by certain voices. See [Azure voice styles documentation](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup-voice#use-speaking-styles-and-roles) | +| `styledegree` | Style intensity | `0.01` to `2` | Higher values = more intense. Default is `1` | +| `role` | Voice role | `Girl`, `Boy`, `YoungAdultFemale`, `YoungAdultMale`, `OlderAdultFemale`, `OlderAdultMale`, `SeniorFemale`, `SeniorMale` | Only supported by certain voices | +| `lang` | Language code | `es-ES`, `fr-FR`, `de-DE`, `hi-IN`, etc. | For multilingual voices. Overrides the default language | + +## Async Support + +```python showLineNumbers title="Async Usage" +import asyncio +from litellm import aspeech +from pathlib import Path + +async def generate_speech(): + response = await aspeech( + model="azure/speech/azure-tts", + voice="alloy", + input="Hello from async", + api_base="https://eastus.tts.speech.microsoft.com", + api_key=os.environ["AZURE_TTS_API_KEY"], + ) + + speech_file_path = Path(__file__).parent / "speech.mp3" + response.stream_to_file(speech_file_path) + +asyncio.run(generate_speech()) +``` + +## Regional Endpoints + +Replace `{region}` with your Azure resource region: + +- US East: `https://eastus.tts.speech.microsoft.com` +- US West: `https://westus.tts.speech.microsoft.com` +- Europe West: `https://westeurope.tts.speech.microsoft.com` +- Asia Southeast: `https://southeastasia.tts.speech.microsoft.com` + +[Full list of regions](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/regions) + +## Advanced Features + +### Custom Neural Voices + +You can use any Azure Neural voice by passing the full voice name: + +```python showLineNumbers title="Custom Voice" +response = speech( + model="azure/speech/azure-tts", + voice="en-US-AriaNeural", # Direct Azure voice name + input="Using a specific neural voice", + api_base="https://eastus.tts.speech.microsoft.com", + api_key=os.environ["AZURE_TTS_API_KEY"], +) +``` + +Browse available voices in the [Azure Speech Gallery](https://speech.microsoft.com/portal/voicegallery). + +## Error Handling + +```python showLineNumbers title="Error Handling" +from litellm import speech +from litellm.exceptions import APIError + +try: + response = speech( + model="azure/speech/azure-tts", + voice="alloy", + input="Test message", + api_base="https://eastus.tts.speech.microsoft.com", + api_key=os.environ["AZURE_TTS_API_KEY"], + ) +except APIError as e: + print(f"Azure Speech error: {e}") +``` + +## Reference + +- [Azure Speech Service Documentation](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/) +- [Text-to-Speech REST API](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/rest-text-to-speech) + diff --git a/docs/my-website/docs/providers/azure_ai_vector_stores.md b/docs/my-website/docs/providers/azure_ai_vector_stores.md new file mode 100644 index 00000000000..b9dfa3bdc9c --- /dev/null +++ b/docs/my-website/docs/providers/azure_ai_vector_stores.md @@ -0,0 +1,245 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Azure AI Search - Vector Store (Unified API) + +Use this to **search** Azure AI Search Vector Stores, with LiteLLM's unified `/chat/completions` API. + +## Quick Start + +You need three things: +1. An Azure AI Search service +2. An embedding model (to convert your queries to vectors) +3. A search index with vector fields + +## Usage + + + + +### Basic Search + +```python +from litellm import vector_stores +import os + +# Set your credentials +os.environ["AZURE_SEARCH_API_KEY"] = "your-search-api-key" +os.environ["AZURE_AI_SEARCH_EMBEDDING_API_BASE"] = "your-embedding-endpoint" +os.environ["AZURE_AI_SEARCH_EMBEDDING_API_KEY"] = "your-embedding-api-key" + +# Search the vector store +response = vector_stores.search( + vector_store_id="my-vector-index", # Your Azure AI Search index name + query="What is the capital of France?", + custom_llm_provider="azure_ai", + azure_search_service_name="your-search-service", + litellm_embedding_model="azure/text-embedding-3-large", + litellm_embedding_config={ + "api_base": os.getenv("AZURE_AI_SEARCH_EMBEDDING_API_BASE"), + "api_key": os.getenv("AZURE_AI_SEARCH_EMBEDDING_API_KEY"), + }, + api_key=os.getenv("AZURE_SEARCH_API_KEY"), +) + +print(response) +``` + +### Async Search + +```python +from litellm import vector_stores + +response = await vector_stores.asearch( + vector_store_id="my-vector-index", + query="What is the capital of France?", + custom_llm_provider="azure_ai", + azure_search_service_name="your-search-service", + litellm_embedding_model="azure/text-embedding-3-large", + litellm_embedding_config={ + "api_base": os.getenv("AZURE_AI_SEARCH_EMBEDDING_API_BASE"), + "api_key": os.getenv("AZURE_AI_SEARCH_EMBEDDING_API_KEY"), + }, + api_key=os.getenv("AZURE_SEARCH_API_KEY"), +) + +print(response) +``` + +### Advanced Options + +```python +from litellm import vector_stores + +response = vector_stores.search( + vector_store_id="my-vector-index", + query="What is the capital of France?", + custom_llm_provider="azure_ai", + azure_search_service_name="your-search-service", + litellm_embedding_model="azure/text-embedding-3-large", + litellm_embedding_config={ + "api_base": os.getenv("AZURE_AI_SEARCH_EMBEDDING_API_BASE"), + "api_key": os.getenv("AZURE_AI_SEARCH_EMBEDDING_API_KEY"), + }, + api_key=os.getenv("AZURE_SEARCH_API_KEY"), + top_k=10, # Number of results to return + azure_search_vector_field="contentVector", # Custom vector field name +) + +print(response) +``` + + + + + +### Setup Config + +Add this to your config.yaml: + +```yaml +vector_store_registry: + - vector_store_name: "azure-ai-search-litellm-website-knowledgebase" + litellm_params: + vector_store_id: "test-litellm-app_1761094730750" + custom_llm_provider: "azure_ai" + api_key: os.environ/AZURE_SEARCH_API_KEY + litellm_embedding_model: "azure/text-embedding-3-large" + litellm_embedding_config: + api_base: https://krris-mh44uf7y-eastus2.cognitiveservices.azure.com/ + api_key: os.environ/AZURE_API_KEY + api_version: "2025-09-01" +``` + +### Start Proxy + +```bash +litellm --config /path/to/config.yaml +``` + +### Search via API + +```bash +curl -X POST 'http://0.0.0.0:4000/v1/vector_stores/my-vector-index/search' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "query": "What is the capital of France?", +}' +``` + + + + +## Required Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `vector_store_id` | string | Your Azure AI Search index name | +| `custom_llm_provider` | string | Set to `"azure_ai"` | +| `azure_search_service_name` | string | Name of your Azure AI Search service | +| `litellm_embedding_model` | string | Model to generate query embeddings (e.g., `"azure/text-embedding-3-large"`) | +| `litellm_embedding_config` | dict | Config for the embedding model (api_base, api_key, api_version) | +| `api_key` | string | Your Azure AI Search API key | + +## Supported Features + +| Feature | Status | Notes | +|---------|--------|-------| +| Logging | ✅ Supported | Full logging support available | +| Guardrails | ❌ Not Yet Supported | Guardrails are not currently supported for vector stores | +| Cost Tracking | ✅ Supported | Cost is $0 according to Azure | +| Unified API | ✅ Supported | Call via OpenAI compatible `/v1/vector_stores/search` endpoint | +| Passthrough | ❌ Not yet supported | | + +## Response Format + +The response follows the standard LiteLLM vector store format: + +```json +{ + "object": "vector_store.search_results.page", + "search_query": "What is the capital of France?", + "data": [ + { + "score": 0.95, + "content": [ + { + "text": "Paris is the capital of France...", + "type": "text" + } + ], + "file_id": "doc_123", + "filename": "Document doc_123", + "attributes": { + "document_id": "doc_123" + } + } + ] +} +``` + +## How It Works + +When you search: + +1. LiteLLM converts your query to a vector using the embedding model you specified +2. It sends the vector to Azure AI Search +3. Azure AI Search finds the most similar documents in your index +4. Results come back with similarity scores + +The embedding model can be any model supported by LiteLLM - Azure OpenAI, OpenAI, Bedrock, etc. + +## Setting Up Your Azure AI Search Index + +Your index needs a vector field. Here's what that looks like: + +```json +{ + "name": "my-vector-index", + "fields": [ + { + "name": "id", + "type": "Edm.String", + "key": true + }, + { + "name": "content", + "type": "Edm.String" + }, + { + "name": "contentVector", + "type": "Collection(Edm.Single)", + "searchable": true, + "dimensions": 1536, + "vectorSearchProfile": "myVectorProfile" + } + ] +} +``` + +The vector dimensions must match your embedding model. For example: +- `text-embedding-3-large`: 1536 dimensions +- `text-embedding-3-small`: 1536 dimensions +- `text-embedding-ada-002`: 1536 dimensions + + +## Common Issues + +**"Failed to generate embedding for query"** + +Your embedding model config is wrong. Check: +- `litellm_embedding_config` has the right api_base and api_key +- The embedding model name is correct +- Your credentials work + +**"Index not found"** + +The `vector_store_id` doesn't match any index in your search service. Check: +- The index name is correct +- You're using the right search service name + +**"Field 'contentVector' not found"** + +Your index uses a different vector field name. Pass it via `azure_search_vector_field`. + diff --git a/docs/my-website/docs/providers/azure_document_intelligence.md b/docs/my-website/docs/providers/azure_document_intelligence.md new file mode 100644 index 00000000000..edc3c616fa7 --- /dev/null +++ b/docs/my-website/docs/providers/azure_document_intelligence.md @@ -0,0 +1,408 @@ +# Azure Document Intelligence OCR + +## Overview + +| Property | Details | +|-------|-------| +| Description | Azure Document Intelligence (formerly Form Recognizer) provides advanced document analysis capabilities including text extraction, layout analysis, and structure recognition | +| Provider Route on LiteLLM | `azure_ai/doc-intelligence/` | +| Supported Operations | `/ocr` | +| Link to Provider Doc | [Azure Document Intelligence ↗](https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/) + +Extract text and analyze document structure using Azure Document Intelligence's powerful prebuilt models. + +## Quick Start + +### **LiteLLM SDK** + +```python showLineNumbers title="SDK Usage" +import litellm +import os + +# Set environment variables +os.environ["AZURE_DOCUMENT_INTELLIGENCE_API_KEY"] = "your-api-key" +os.environ["AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT"] = "https://your-resource.cognitiveservices.azure.com" + +# OCR with PDF URL +response = litellm.ocr( + model="azure_ai/doc-intelligence/prebuilt-layout", + document={ + "type": "document_url", + "document_url": "https://example.com/document.pdf" + } +) + +# Access extracted text +for page in response.pages: + print(f"Page {page.index}:") + print(page.markdown) +``` + +### **LiteLLM PROXY** + +```yaml showLineNumbers title="proxy_config.yaml" +model_list: + - model_name: azure-doc-intel + litellm_params: + model: azure_ai/doc-intelligence/prebuilt-layout + api_key: os.environ/AZURE_DOCUMENT_INTELLIGENCE_API_KEY + api_base: os.environ/AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT + model_info: + mode: ocr +``` + +**Start Proxy** +```bash +litellm --config proxy_config.yaml +``` + +**Call OCR via Proxy** +```bash showLineNumbers title="cURL Request" +curl -X POST http://localhost:4000/ocr \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-api-key" \ + -d '{ + "model": "azure-doc-intel", + "document": { + "type": "document_url", + "document_url": "https://arxiv.org/pdf/2201.04234" + } + }' +``` + +## How It Works + +Azure Document Intelligence uses an asynchronous API pattern. LiteLLM AI Gateway handles the request/response transformation and polling automatically. + +### Complete Flow Diagram + +```mermaid +sequenceDiagram + participant Client + box rgb(200, 220, 255) LiteLLM AI Gateway + participant LiteLLM + end + participant Azure as Azure Document Intelligence + + Client->>LiteLLM: POST /ocr (Mistral format) + Note over LiteLLM: Transform to Azure format + + LiteLLM->>Azure: POST :analyze + Azure-->>LiteLLM: 202 Accepted + polling URL + + Note over LiteLLM: Automatic Polling + loop Every 2-10 seconds + LiteLLM->>Azure: GET polling URL + Azure-->>LiteLLM: Status: running + end + + LiteLLM->>Azure: GET polling URL + Azure-->>LiteLLM: Status: succeeded + results + + Note over LiteLLM: Transform to Mistral format + LiteLLM-->>Client: OCR Response (Mistral format) +``` + +### What LiteLLM Does For You + +When you call `litellm.ocr()` via SDK or `/ocr` via Proxy: + +1. **Request Transformation**: Converts Mistral OCR format → Azure Document Intelligence format +2. **Submits Document**: Sends transformed request to Azure DI API +3. **Handles 202 Response**: Captures the `Operation-Location` URL from response headers +4. **Automatic Polling**: + - Polls the operation URL at intervals specified by `retry-after` header (default: 2 seconds) + - Continues until status is `succeeded` or `failed` + - Respects Azure's rate limiting via `retry-after` headers +5. **Response Transformation**: Converts Azure DI format → Mistral OCR format +6. **Returns Result**: Sends unified Mistral format response to client + +**Polling Configuration:** +- Default timeout: 120 seconds +- Configurable via `AZURE_OPERATION_POLLING_TIMEOUT` environment variable +- Uses sync (`time.sleep()`) or async (`await asyncio.sleep()`) based on call type + +:::info +**Typical processing time**: 2-10 seconds depending on document size and complexity +::: + +## Supported Models + +Azure Document Intelligence offers several prebuilt models optimized for different use cases: + +### prebuilt-layout (Recommended) + +Best for general document OCR with structure preservation. + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + + + +```python showLineNumbers title="Layout Model - SDK" +import litellm +import os + +os.environ["AZURE_DOCUMENT_INTELLIGENCE_API_KEY"] = "your-api-key" +os.environ["AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT"] = "https://your-resource.cognitiveservices.azure.com" + +response = litellm.ocr( + model="azure_ai/doc-intelligence/prebuilt-layout", + document={ + "type": "document_url", + "document_url": "https://example.com/document.pdf" + } +) +``` + + + + +```yaml showLineNumbers title="proxy_config.yaml" +model_list: + - model_name: azure-layout + litellm_params: + model: azure_ai/doc-intelligence/prebuilt-layout + api_key: os.environ/AZURE_DOCUMENT_INTELLIGENCE_API_KEY + api_base: os.environ/AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT + model_info: + mode: ocr +``` + +**Usage:** +```bash +curl -X POST http://localhost:4000/ocr \ + -H "Authorization: Bearer your-api-key" \ + -d '{"model": "azure-layout", "document": {"type": "document_url", "document_url": "https://example.com/doc.pdf"}}' +``` + + + + +**Features:** +- Text extraction with markdown formatting +- Table detection and extraction +- Document structure analysis +- Paragraph and section recognition + +**Pricing:** $10 per 1,000 pages + +### prebuilt-read + +Optimized for reading text from documents - fastest and most cost-effective. + + + + +```python showLineNumbers title="Read Model - SDK" +import litellm +import os + +os.environ["AZURE_DOCUMENT_INTELLIGENCE_API_KEY"] = "your-api-key" +os.environ["AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT"] = "https://your-resource.cognitiveservices.azure.com" + +response = litellm.ocr( + model="azure_ai/doc-intelligence/prebuilt-read", + document={ + "type": "document_url", + "document_url": "https://example.com/document.pdf" + } +) +``` + + + + +```yaml showLineNumbers title="proxy_config.yaml" +model_list: + - model_name: azure-read + litellm_params: + model: azure_ai/doc-intelligence/prebuilt-read + api_key: os.environ/AZURE_DOCUMENT_INTELLIGENCE_API_KEY + api_base: os.environ/AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT + model_info: + mode: ocr +``` + +**Usage:** +```bash +curl -X POST http://localhost:4000/ocr \ + -H "Authorization: Bearer your-api-key" \ + -d '{"model": "azure-read", "document": {"type": "document_url", "document_url": "https://example.com/doc.pdf"}}' +``` + + + + +**Features:** +- Fast text extraction +- Optimized for reading-heavy documents +- Basic structure recognition + +**Pricing:** $1.50 per 1,000 pages + +### prebuilt-document + +General-purpose document analysis with key-value pairs. + + + + +```python showLineNumbers title="Document Model - SDK" +import litellm +import os + +os.environ["AZURE_DOCUMENT_INTELLIGENCE_API_KEY"] = "your-api-key" +os.environ["AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT"] = "https://your-resource.cognitiveservices.azure.com" + +response = litellm.ocr( + model="azure_ai/doc-intelligence/prebuilt-document", + document={ + "type": "document_url", + "document_url": "https://example.com/document.pdf" + } +) +``` + + + + +```yaml showLineNumbers title="proxy_config.yaml" +model_list: + - model_name: azure-document + litellm_params: + model: azure_ai/doc-intelligence/prebuilt-document + api_key: os.environ/AZURE_DOCUMENT_INTELLIGENCE_API_KEY + api_base: os.environ/AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT + model_info: + mode: ocr +``` + +**Usage:** +```bash +curl -X POST http://localhost:4000/ocr \ + -H "Authorization: Bearer your-api-key" \ + -d '{"model": "azure-document", "document": {"type": "document_url", "document_url": "https://example.com/doc.pdf"}}' +``` + + + + +**Pricing:** $10 per 1,000 pages + +## Document Types + +Azure Document Intelligence supports various document formats. + +### PDF Documents + +```python showLineNumbers title="PDF OCR" +response = litellm.ocr( + model="azure_ai/doc-intelligence/prebuilt-layout", + document={ + "type": "document_url", + "document_url": "https://example.com/document.pdf" + } +) +``` + +### Image Documents + +```python showLineNumbers title="Image OCR" +response = litellm.ocr( + model="azure_ai/doc-intelligence/prebuilt-layout", + document={ + "type": "image_url", + "image_url": "https://example.com/image.png" + } +) +``` + +**Supported image formats:** JPEG, PNG, BMP, TIFF + +### Base64 Encoded Documents + +```python showLineNumbers title="Base64 PDF" +import base64 + +# Read and encode PDF +with open("document.pdf", "rb") as f: + pdf_base64 = base64.b64encode(f.read()).decode() + +response = litellm.ocr( + model="azure_ai/doc-intelligence/prebuilt-layout", + document={ + "type": "document_url", + "document_url": f"data:application/pdf;base64,{pdf_base64}" + } +) +``` + +## Response Format + +```python showLineNumbers title="Response Structure" +# Response has the following structure +response.pages # List of pages with extracted text +response.model # Model used +response.object # "ocr" +response.usage_info # Token usage information + +# Access page content +for page in response.pages: + print(f"Page {page.index}:") + print(page.markdown) + + # Page dimensions (in pixels) + if page.dimensions: + print(f"Width: {page.dimensions.width}px") + print(f"Height: {page.dimensions.height}px") +``` + +## Async Support + +```python showLineNumbers title="Async Usage" +import litellm +import asyncio + +async def process_document(): + response = await litellm.aocr( + model="azure_ai/doc-intelligence/prebuilt-layout", + document={ + "type": "document_url", + "document_url": "https://example.com/document.pdf" + } + ) + return response + +# Run async function +response = asyncio.run(process_document()) +``` + +## Cost Tracking + +LiteLLM automatically tracks costs for Azure Document Intelligence OCR: + +| Model | Cost per 1,000 Pages | +|-------|---------------------| +| prebuilt-read | $1.50 | +| prebuilt-layout | $10.00 | +| prebuilt-document | $10.00 | + +```python showLineNumbers title="View Cost" +response = litellm.ocr( + model="azure_ai/doc-intelligence/prebuilt-layout", + document={"type": "document_url", "document_url": "https://..."} +) + +# Access cost information +print(f"Cost: ${response._hidden_params.get('response_cost', 0)}") +``` + +## Additional Resources + +- [Azure Document Intelligence Documentation](https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/) +- [Pricing Details](https://azure.microsoft.com/en-us/pricing/details/ai-document-intelligence/) +- [Supported File Formats](https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/concept-model-overview) +- [LiteLLM OCR Documentation](https://docs.litellm.ai/docs/ocr) + diff --git a/docs/my-website/docs/providers/azure_ocr.md b/docs/my-website/docs/providers/azure_ocr.md new file mode 100644 index 00000000000..5d79cc05338 --- /dev/null +++ b/docs/my-website/docs/providers/azure_ocr.md @@ -0,0 +1,154 @@ +# Azure AI OCR (Mistral) + +## Overview + +| Property | Details | +|-------|-------| +| Description | Azure AI OCR provides document intelligence capabilities powered by Mistral, enabling text extraction from PDFs and images | +| Provider Route on LiteLLM | `azure_ai/` | +| Supported Operations | `/ocr` | +| Link to Provider Doc | [Azure AI ↗](https://ai.azure.com/) + +Extract text from documents and images using Azure AI's OCR models, powered by Mistral. + +## Quick Start + +### **LiteLLM SDK** + +```python showLineNumbers title="SDK Usage" +import litellm +import os + +# Set environment variables +os.environ["AZURE_AI_API_KEY"] = "" +os.environ["AZURE_AI_API_BASE"] = "" + +# OCR with PDF URL +response = litellm.ocr( + model="azure_ai/mistral-document-ai-2505", + document={ + "type": "document_url", + "document_url": "https://example.com/document.pdf" + } +) + +# Access extracted text +for page in response.pages: + print(page.text) +``` + +### **LiteLLM PROXY** + +```yaml showLineNumbers title="proxy_config.yaml" +model_list: + - model_name: azure-ocr + litellm_params: + model: azure_ai/mistral-document-ai-2505 + api_key: "os.environ/AZURE_AI_API_KEY" + api_base: "os.environ/AZURE_AI_API_BASE" + model_info: + mode: ocr +``` + +## Document Types + +Azure AI OCR supports both PDFs and images. + +### PDF Documents + +```python showLineNumbers title="PDF OCR" +response = litellm.ocr( + model="azure_ai/mistral-document-ai-2505", + document={ + "type": "document_url", + "document_url": "https://example.com/document.pdf" + } +) +``` + +### Image Documents + +```python showLineNumbers title="Image OCR" +response = litellm.ocr( + model="azure_ai/mistral-document-ai-2505", + document={ + "type": "image_url", + "image_url": "https://example.com/image.png" + } +) +``` + +### Base64 Encoded Documents + +```python showLineNumbers title="Base64 PDF" +import base64 + +# Read and encode PDF +with open("document.pdf", "rb") as f: + pdf_base64 = base64.b64encode(f.read()).decode() + +response = litellm.ocr( + model="azure_ai/mistral-document-ai-2505", + document={ + "type": "document_url", + "document_url": f"data:application/pdf;base64,{pdf_base64}" + } +) +``` + +## Supported Parameters + +```python showLineNumbers title="All Parameters" +response = litellm.ocr( + model="azure_ai/mistral-document-ai-2505", + document={ # Required: Document to process + "type": "document_url", + "document_url": "https://..." + }, + include_image_base64=True, # Optional: Include base64 images + pages=[0, 1, 2], # Optional: Specific pages to process + image_limit=10 # Optional: Limit number of images +) +``` + +## Response Format + +```python showLineNumbers title="Response Structure" +# Response has the following structure +response.pages # List of pages with extracted text +response.model # Model used +response.object # "ocr" +response.usage_info # Token usage information + +# Access page content +for page in response.pages: + print(f"Page {page.page_number}:") + print(page.text) +``` + +## Async Support + +```python showLineNumbers title="Async Usage" +import litellm + +response = await litellm.aocr( + model="azure_ai/mistral-document-ai-2505", + document={ + "type": "document_url", + "document_url": "https://example.com/document.pdf" + } +) +``` + +## Important Notes + +:::info URL Conversion +Azure AI OCR endpoints don't have internet access. LiteLLM automatically converts public URLs to base64 data URIs before sending requests to Azure AI. +::: + +## Supported Models + +- `mistral-document-ai-2505` - Latest Mistral OCR model on Azure AI + +Use the Azure AI provider prefix: `azure_ai/` + diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index 28cae80cc42..f0b89615a0d 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -7,7 +7,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor | Property | Details | |-------|-------| | Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). | -| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1) | +| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models) | | Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) | | Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` | | Rerank Endpoint | `/rerank` | @@ -1734,7 +1734,69 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ +### Qwen3 Imported Models +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/qwen3/{model_arn}` | +| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Qwen3 Models](https://aws.amazon.com/about-aws/whats-new/2025/09/qwen3-models-fully-managed-amazon-bedrock/) | + + + + +```python +from litellm import completion +import os + +response = completion( + model="bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model", # bedrock/qwen3/{your-model-arn} + messages=[{"role": "user", "content": "Tell me a joke"}], + max_tokens=100, + temperature=0.7 +) +``` + + + + + +**1. Add to config** + +```yaml +model_list: + - model_name: Qwen3-32B + litellm_params: + model: bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model + +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "Qwen3-32B", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + ### OpenAI GPT OSS @@ -1937,203 +1999,13 @@ response = embedding( ### Advanced - [Pass model/provider-specific Params](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage) ## Image Generation -Use this for stable diffusion, and amazon nova canvas on bedrock + +See [Bedrock Image Generation](./bedrock_image_gen) for using Stable Diffusion and Amazon Nova Canvas models on Bedrock. -### Usage +## Rerank API - - - -```python -import os -from litellm import image_generation - -os.environ["AWS_ACCESS_KEY_ID"] = "" -os.environ["AWS_SECRET_ACCESS_KEY"] = "" -os.environ["AWS_REGION_NAME"] = "" - -response = image_generation( - prompt="A cute baby sea otter", - model="bedrock/stability.stable-diffusion-xl-v0", - ) -print(f"response: {response}") -``` - -**Set optional params** -```python -import os -from litellm import image_generation - -os.environ["AWS_ACCESS_KEY_ID"] = "" -os.environ["AWS_SECRET_ACCESS_KEY"] = "" -os.environ["AWS_REGION_NAME"] = "" - -response = image_generation( - prompt="A cute baby sea otter", - model="bedrock/stability.stable-diffusion-xl-v0", - ### OPENAI-COMPATIBLE ### - size="128x512", # width=128, height=512 - ### PROVIDER-SPECIFIC ### see `AmazonStabilityConfig` in bedrock.py for all params - seed=30 - ) -print(f"response: {response}") -``` - - - -1. Setup config.yaml - -```yaml -model_list: - - model_name: amazon.nova-canvas-v1:0 - litellm_params: - model: bedrock/amazon.nova-canvas-v1:0 - aws_region_name: "us-east-1" - aws_secret_access_key: my-key # OPTIONAL - all boto3 auth params supported - aws_secret_access_id: my-id # OPTIONAL - all boto3 auth params supported -``` - -2. Start proxy - -```bash -litellm --config /path/to/config.yaml -``` - -3. Test it! - -```bash -curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \ --H 'Content-Type: application/json' \ --H 'Authorization: Bearer $LITELLM_VIRTUAL_KEY' \ --d '{ - "model": "amazon.nova-canvas-v1:0", - "prompt": "A cute baby sea otter" -}' -``` - - - - -### Using Inference Profiles with Image Generation - -For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN: - - - - -```python -from litellm import image_generation - -response = image_generation( - model="bedrock/amazon.nova-canvas-v1:0", - model_id="arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0", - prompt="A cute baby sea otter" -) -print(f"response: {response}") -``` - - - - -```yaml -model_list: - - model_name: nova-canvas-inference-profile - litellm_params: - model: bedrock/amazon.nova-canvas-v1:0 - model_id: arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0 - aws_region_name: "eu-west-1" -``` - - - - -## Supported AWS Bedrock Image Generation Models - -| Model Name | Function Call | -|----------------------|---------------------------------------------| -| Stable Diffusion 3 - v0 | `embedding(model="bedrock/stability.stability.sd3-large-v1:0", prompt=prompt)` | -| Stable Diffusion - v0 | `embedding(model="bedrock/stability.stable-diffusion-xl-v0", prompt=prompt)` | -| Stable Diffusion - v0 | `embedding(model="bedrock/stability.stable-diffusion-xl-v1", prompt=prompt)` | - - -## Rerank API - -Use Bedrock's Rerank API in the Cohere `/rerank` format. - -Supported Cohere Rerank Params -- `model` - the foundation model ARN -- `query` - the query to rerank against -- `documents` - the list of documents to rerank -- `top_n` - the number of results to return - - - - -```python -from litellm import rerank -import os - -os.environ["AWS_ACCESS_KEY_ID"] = "" -os.environ["AWS_SECRET_ACCESS_KEY"] = "" -os.environ["AWS_REGION_NAME"] = "" - -response = rerank( - model="bedrock/arn:aws:bedrock:us-west-2::foundation-model/amazon.rerank-v1:0", # provide the model ARN - get this here https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/bedrock/client/list_foundation_models.html - query="hello", - documents=["hello", "world"], - top_n=2, -) - -print(response) -``` - - - - -1. Setup config.yaml - -```yaml -model_list: - - model_name: bedrock-rerank - litellm_params: - model: bedrock/arn:aws:bedrock:us-west-2::foundation-model/amazon.rerank-v1:0 - aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID - aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY - aws_region_name: os.environ/AWS_REGION_NAME -``` - -2. Start proxy server - -```bash -litellm --config config.yaml - -# RUNNING on http://0.0.0.0:4000 -``` - -3. Test it! - -```bash -curl http://0.0.0.0:4000/rerank \ - -H "Authorization: Bearer sk-1234" \ - -H "Content-Type: application/json" \ - -d '{ - "model": "bedrock-rerank", - "query": "What is the capital of the United States?", - "documents": [ - "Carson City is the capital city of the American state of Nevada.", - "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.", - "Washington, D.C. is the capital of the United States.", - "Capital punishment has existed in the United States since before it was a country." - ], - "top_n": 3 - - - }' -``` - - - +See [Bedrock Rerank](./bedrock_rerank) for using Bedrock's Rerank API in the Cohere `/rerank` format. ## Bedrock Application Inference Profile @@ -2428,38 +2300,6 @@ model_list: -Text to Image : -```bash -curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \ --H 'Content-Type: application/json' \ --H 'Authorization: Bearer $LITELLM_VIRTUAL_KEY' \ --d '{ - "model": "amazon.nova-canvas-v1:0", - "prompt": "A cute baby sea otter" -}' -``` - -Color Guided Generation: -```bash -curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \ --H 'Content-Type: application/json' \ --H 'Authorization: Bearer $LITELLM_VIRTUAL_KEY' \ --d '{ - "model": "amazon.nova-canvas-v1:0", - "prompt": "A cute baby sea otter", - "taskType": "COLOR_GUIDED_GENERATION", - "colorGuidedGenerationParams":{"colors":["#FFFFFF"]} -}' -``` - -| Model Name | Function Call | -|-------------------------|---------------------------------------------| -| Stable Diffusion 3 - v0 | `image_generation(model="bedrock/stability.stability.sd3-large-v1:0", prompt=prompt)` | -| Stable Diffusion - v0 | `image_generation(model="bedrock/stability.stable-diffusion-xl-v0", prompt=prompt)` | -| Stable Diffusion - v1 | `image_generation(model="bedrock/stability.stable-diffusion-xl-v1", prompt=prompt)` | -| Amazon Nova Canvas - v0 | `image_generation(model="bedrock/amazon.nova-canvas-v1:0", prompt=prompt)` | - - ### Passing an external BedrockRuntime.Client as a parameter - Completion() This is a deprecated flow. Boto3 is not async. And boto3.client does not let us make the http call through httpx. Pass in your aws params through the method above 👆. [See Auth Code](https://github.com/BerriAI/litellm/blob/55a20c7cce99a93d36a82bf3ae90ba3baf9a7f89/litellm/llms/bedrock_httpx.py#L284) [Add new auth flow](https://github.com/BerriAI/litellm/issues) diff --git a/docs/my-website/docs/providers/bedrock_agentcore.md b/docs/my-website/docs/providers/bedrock_agentcore.md new file mode 100644 index 00000000000..43df7f82519 --- /dev/null +++ b/docs/my-website/docs/providers/bedrock_agentcore.md @@ -0,0 +1,246 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Bedrock AgentCore + +Call Bedrock AgentCore in the OpenAI Request/Response format. + +| Property | Details | +|----------|---------| +| Description | Amazon Bedrock AgentCore provides direct access to hosted agent runtimes for executing agentic workflows with foundation models. | +| Provider Route on LiteLLM | `bedrock/agentcore/{AGENT_RUNTIME_ARN}` | +| Provider Doc | [AWS Bedrock AgentCore ↗](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgentRuntime.html) | + +## Quick Start + +### Model Format to LiteLLM + +To call a bedrock agent runtime through LiteLLM, use the following model format. + +Here the `model=bedrock/agentcore/` tells LiteLLM to call the bedrock `InvokeAgentRuntime` API. + +```shell showLineNumbers title="Model Format to LiteLLM" +bedrock/agentcore/{AGENT_RUNTIME_ARN} +``` + +**Example:** +- `bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime` + +You can find the Agent Runtime ARN in your AWS Bedrock console under AgentCore. + +### LiteLLM Python SDK + +```python showLineNumbers title="Basic AgentCore Completion" +import litellm + +# Make a completion request to your AgentCore runtime +response = litellm.completion( + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime", + messages=[ + { + "role": "user", + "content": "Explain machine learning in simple terms" + } + ], +) + +print(response.choices[0].message.content) +print(f"Usage: {response.usage}") +``` + +```python showLineNumbers title="Streaming AgentCore Responses" +import litellm + +# Stream responses from your AgentCore runtime +response = litellm.completion( + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime", + messages=[ + { + "role": "user", + "content": "What are the key principles of software architecture?" + } + ], + stream=True, +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") +``` + +### LiteLLM Proxy + +#### 1. Configure your model in config.yaml + + + + +```yaml showLineNumbers title="LiteLLM Proxy Configuration" +model_list: + - model_name: agentcore-runtime-1 + litellm_params: + model: bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: us-west-2 + + - model_name: agentcore-runtime-2 + litellm_params: + model: bedrock/agentcore/arn:aws:bedrock-agentcore:us-east-1:987654321098:runtime/production-runtime + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: us-east-1 +``` + + + + +#### 2. Start the LiteLLM Proxy + +```bash showLineNumbers title="Start LiteLLM Proxy" +litellm --config config.yaml +``` + +#### 3. Make requests to your AgentCore runtimes + + + + +```bash showLineNumbers title="Basic AgentCore Request" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -d '{ + "model": "agentcore-runtime-1", + "messages": [ + { + "role": "user", + "content": "Summarize the main benefits of cloud computing" + } + ] + }' +``` + +```bash showLineNumbers title="Streaming AgentCore Request" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -d '{ + "model": "agentcore-runtime-2", + "messages": [ + { + "role": "user", + "content": "Explain the differences between SQL and NoSQL databases" + } + ], + "stream": true + }' +``` + + + + + +```python showLineNumbers title="Using OpenAI SDK with LiteLLM Proxy" +from openai import OpenAI + +# Initialize client with your LiteLLM proxy URL +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-litellm-api-key" +) + +# Make a completion request to your AgentCore runtime +response = client.chat.completions.create( + model="agentcore-runtime-1", + messages=[ + { + "role": "user", + "content": "What are best practices for API design?" + } + ] +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Streaming with OpenAI SDK" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-litellm-api-key" +) + +# Stream AgentCore responses +stream = client.chat.completions.create( + model="agentcore-runtime-2", + messages=[ + { + "role": "user", + "content": "Describe the microservices architecture pattern" + } + ], + stream=True +) + +for chunk in stream: + if chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="") +``` + + + + +## Provider-specific Parameters + +AgentCore supports additional parameters that can be passed to customize the runtime invocation. + + + + +```python showLineNumbers title="Using AgentCore-specific parameters" +from litellm import completion + +response = litellm.completion( + model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime", + messages=[ + { + "role": "user", + "content": "Analyze this data and provide insights", + } + ], + qualifier="production", # PROVIDER-SPECIFIC: Runtime qualifier/version + runtimeSessionId="session-abc-123", # PROVIDER-SPECIFIC: Custom session ID +) +``` + + + + +```yaml showLineNumbers title="LiteLLM Proxy Configuration with Parameters" +model_list: + - model_name: agentcore-runtime-prod + litellm_params: + model: bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/my-agent-runtime + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: us-west-2 + qualifier: production +``` + + + + +### Available Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `qualifier` | string | Optional runtime qualifier/version to invoke a specific version of the agent runtime | +| `runtimeSessionId` | string | Optional custom session ID (must be 33+ characters). If not provided, LiteLLM generates one automatically | + +## Further Reading + +- [AWS Bedrock AgentCore Documentation](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgentRuntime.html) +- [LiteLLM Authentication to Bedrock](https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication) + diff --git a/docs/my-website/docs/providers/bedrock_batches.md b/docs/my-website/docs/providers/bedrock_batches.md index 57487f7d2c9..c262eef0e86 100644 --- a/docs/my-website/docs/providers/bedrock_batches.md +++ b/docs/my-website/docs/providers/bedrock_batches.md @@ -9,6 +9,7 @@ Use Amazon Bedrock Batch Inference API through LiteLLM. |----------|---------| | Description | Amazon Bedrock Batch Inference allows you to run inference on large datasets asynchronously | | Provider Doc | [AWS Bedrock Batch Inference ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html) | +| Cost Tracking | ✅ Supported | ## Overview diff --git a/docs/my-website/docs/providers/bedrock_embedding.md b/docs/my-website/docs/providers/bedrock_embedding.md index cd492084711..76c9606533e 100644 --- a/docs/my-website/docs/providers/bedrock_embedding.md +++ b/docs/my-website/docs/providers/bedrock_embedding.md @@ -2,11 +2,11 @@ ## Supported Embedding Models -| Provider | LiteLLM Route | AWS Documentation | -|----------|---------------|-------------------| -| Amazon Titan | `bedrock/amazon.*` | [Amazon Titan Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html) | -| Cohere | `bedrock/cohere.*` | [Cohere Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-embed.html) | -| TwelveLabs | `bedrock/us.twelvelabs.*` | [TwelveLabs](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-twelvelabs.html) | +| Provider | LiteLLM Route | AWS Documentation | Cost Tracking | +|----------|---------------|-------------------|---------------| +| Amazon Titan | `bedrock/amazon.*` | [Amazon Titan Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html) | ✅ | +| Cohere | `bedrock/cohere.*` | [Cohere Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-embed.html) | ✅ | +| TwelveLabs | `bedrock/us.twelvelabs.*` | [TwelveLabs](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-twelvelabs.html) | ✅ | ## Async Invoke Support diff --git a/docs/my-website/docs/providers/bedrock_image_gen.md b/docs/my-website/docs/providers/bedrock_image_gen.md new file mode 100644 index 00000000000..799c6d46437 --- /dev/null +++ b/docs/my-website/docs/providers/bedrock_image_gen.md @@ -0,0 +1,150 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# AWS Bedrock - Image Generation + +Use Bedrock for image generation with Stable Diffusion, Amazon Titan Image Generator, and Amazon Nova Canvas models. + +## Supported Models + +| Model Name | Function Call | Cost Tracking | +|-------------------------|---------------------------------------------|---------------| +| Stable Diffusion 3 - v0 | `image_generation(model="bedrock/stability.stability.sd3-large-v1:0", prompt=prompt)` | ✅ | +| Stable Diffusion - v0 | `image_generation(model="bedrock/stability.stable-diffusion-xl-v0", prompt=prompt)` | ✅ | +| Stable Diffusion - v1 | `image_generation(model="bedrock/stability.stable-diffusion-xl-v1", prompt=prompt)` | ✅ | +| Amazon Titan Image Generator - v1 | `image_generation(model="bedrock/amazon.titan-image-generator-v1", prompt=prompt)` | ✅ | +| Amazon Titan Image Generator - v2 | `image_generation(model="bedrock/amazon.titan-image-generator-v2:0", prompt=prompt)` | ✅ | +| Amazon Nova Canvas - v1 | `image_generation(model="bedrock/amazon.nova-canvas-v1:0", prompt=prompt)` | ✅ | + +## Usage + + + + +### Basic Usage + +```python +import os +from litellm import image_generation + +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "" + +response = image_generation( + prompt="A cute baby sea otter", + model="bedrock/stability.stable-diffusion-xl-v0", +) +print(f"response: {response}") +``` + +### Set Optional Parameters + +```python +import os +from litellm import image_generation + +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "" + +response = image_generation( + prompt="A cute baby sea otter", + model="bedrock/stability.stable-diffusion-xl-v0", + ### OPENAI-COMPATIBLE ### + size="128x512", # width=128, height=512 + ### PROVIDER-SPECIFIC ### see `AmazonStabilityConfig` in bedrock.py for all params + seed=30 +) +print(f"response: {response}") +``` + + + + +### 1. Setup config.yaml + +```yaml +model_list: + - model_name: amazon.nova-canvas-v1:0 + litellm_params: + model: bedrock/amazon.nova-canvas-v1:0 + aws_region_name: "us-east-1" + aws_secret_access_key: my-key # OPTIONAL - all boto3 auth params supported + aws_secret_access_id: my-id # OPTIONAL - all boto3 auth params supported +``` + +### 2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +### 3. Test it! + +**Text to Image:** + +```bash +curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer $LITELLM_VIRTUAL_KEY' \ +-d '{ + "model": "amazon.nova-canvas-v1:0", + "prompt": "A cute baby sea otter" +}' +``` + +**Color Guided Generation:** + +```bash +curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer $LITELLM_VIRTUAL_KEY' \ +-d '{ + "model": "amazon.nova-canvas-v1:0", + "prompt": "A cute baby sea otter", + "taskType": "COLOR_GUIDED_GENERATION", + "colorGuidedGenerationParams":{"colors":["#FFFFFF"]} +}' +``` + + + + +## Using Inference Profiles with Image Generation + +For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN: + + + + +```python +from litellm import image_generation + +response = image_generation( + model="bedrock/amazon.nova-canvas-v1:0", + model_id="arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0", + prompt="A cute baby sea otter" +) +print(f"response: {response}") +``` + + + + +```yaml +model_list: + - model_name: nova-canvas-inference-profile + litellm_params: + model: bedrock/amazon.nova-canvas-v1:0 + model_id: arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0 + aws_region_name: "eu-west-1" +``` + + + + +## Authentication + +All standard Bedrock authentication methods are supported for image generation. See [Bedrock Authentication](./bedrock#boto3---authentication) for details. + diff --git a/docs/my-website/docs/providers/bedrock_rerank.md b/docs/my-website/docs/providers/bedrock_rerank.md new file mode 100644 index 00000000000..86745eb5125 --- /dev/null +++ b/docs/my-website/docs/providers/bedrock_rerank.md @@ -0,0 +1,94 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# AWS Bedrock - Rerank API + +Use Bedrock's Rerank API in the Cohere `/rerank` format. + +:::info Cost Tracking + +✅ **Cost tracking is supported** for Bedrock Rerank API calls. + +::: + +## Supported Parameters + +- `model` - the foundation model ARN +- `query` - the query to rerank against +- `documents` - the list of documents to rerank +- `top_n` - the number of results to return + +## Usage + + + + +```python +from litellm import rerank +import os + +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "" + +response = rerank( + model="bedrock/arn:aws:bedrock:us-west-2::foundation-model/amazon.rerank-v1:0", # provide the model ARN - get this here https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/bedrock/client/list_foundation_models.html + query="hello", + documents=["hello", "world"], + top_n=2, +) + +print(response) +``` + + + + +### 1. Setup config.yaml + +```yaml +model_list: + - model_name: bedrock-rerank + litellm_params: + model: bedrock/arn:aws:bedrock:us-west-2::foundation-model/amazon.rerank-v1:0 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: os.environ/AWS_REGION_NAME +``` + +### 2. Start proxy server + +```bash +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +### 3. Test it! + +```bash +curl http://0.0.0.0:4000/rerank \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "bedrock-rerank", + "query": "What is the capital of the United States?", + "documents": [ + "Carson City is the capital city of the American state of Nevada.", + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.", + "Washington, D.C. is the capital of the United States.", + "Capital punishment has existed in the United States since before it was a country." + ], + "top_n": 3 + + + }' +``` + + + + +## Authentication + +All standard Bedrock authentication methods are supported for rerank. See [Bedrock Authentication](./bedrock#boto3---authentication) for details. + diff --git a/docs/my-website/docs/providers/bedrock_vector_store.md b/docs/my-website/docs/providers/bedrock_vector_store.md index 779c4fd0417..5fae0c76c11 100644 --- a/docs/my-website/docs/providers/bedrock_vector_store.md +++ b/docs/my-website/docs/providers/bedrock_vector_store.md @@ -138,7 +138,133 @@ print(response.choices[0].message.content) -Futher Reading Vector Stores: +## Filter Results + +Filter by metadata attributes. + +**Operators** (OpenAI-style, auto-translated): +- `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `in`, `nin` + +**AWS operators** (use directly): +- `equals`, `notEquals`, `greaterThan`, `greaterThanOrEquals`, `lessThan`, `lessThanOrEquals`, `in`, `notIn`, `startsWith`, `listContains`, `stringContains` + + + + +```python +response = await litellm.acompletion( + model="anthropic/claude-3-5-sonnet", + messages=[{"role": "user", "content": "What are the latest updates?"}], + tools=[{ + "type": "file_search", + "vector_store_ids": ["YOUR_KNOWLEDGE_BASE_ID"], + "filters": { + "key": "category", + "value": "updates", + "operator": "eq" + } + }] +) +``` + + + + + +```python +response = await litellm.acompletion( + model="anthropic/claude-3-5-sonnet", + messages=[{"role": "user", "content": "What are the policies?"}], + tools=[{ + "type": "file_search", + "vector_store_ids": ["YOUR_KNOWLEDGE_BASE_ID"], + "filters": { + "and": [ + {"key": "category", "value": "policy", "operator": "eq"}, + {"key": "year", "value": 2024, "operator": "gte"} + ] + } + }] +) +``` + + + + + +```python +response = await litellm.acompletion( + model="anthropic/claude-3-5-sonnet", + messages=[{"role": "user", "content": "Show me technical docs"}], + tools=[{ + "type": "file_search", + "vector_store_ids": ["YOUR_KNOWLEDGE_BASE_ID"], + "filters": { + "or": [ + {"key": "category", "value": "api", "operator": "eq"}, + {"key": "category", "value": "sdk", "operator": "eq"} + ] + } + }] +) +``` + + + + + +```python +response = await litellm.acompletion( + model="anthropic/claude-3-5-sonnet", + messages=[{"role": "user", "content": "Find docs"}], + tools=[{ + "type": "file_search", + "vector_store_ids": ["YOUR_KNOWLEDGE_BASE_ID"], + "filters": { + "and": [ + {"key": "title", "value": "Guide", "operator": "stringContains"}, + {"key": "tags", "value": "important", "operator": "listContains"} + ] + } + }] +) +``` + + + + + +```bash +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -d '{ + "model": "claude-3-5-sonnet", + "messages": [{"role": "user", "content": "What are our policies?"}], + "tools": [{ + "type": "file_search", + "vector_store_ids": ["YOUR_KNOWLEDGE_BASE_ID"], + "filters": { + "and": [ + {"key": "department", "value": "engineering", "operator": "eq"}, + {"key": "type", "value": "policy", "operator": "eq"} + ] + } + }] + }' +``` + + + + +## Accessing Search Results + +See how to access vector store search results in your response: +- [Accessing Search Results (Non-Streaming & Streaming)](../completion/knowledgebase#accessing-search-results-citations) + +## Further Reading + +Vector Stores: - [Always on Vector Stores](https://docs.litellm.ai/docs/completion/knowledgebase#always-on-for-a-model) - [Listing available vector stores on litellm proxy](https://docs.litellm.ai/docs/completion/knowledgebase#listing-available-vector-stores) - [How LiteLLM Vector Stores Work](https://docs.litellm.ai/docs/completion/knowledgebase#how-it-works) \ No newline at end of file diff --git a/docs/my-website/docs/providers/clarifai.md b/docs/my-website/docs/providers/clarifai.md index cb498650385..eb46901db22 100644 --- a/docs/my-website/docs/providers/clarifai.md +++ b/docs/my-website/docs/providers/clarifai.md @@ -1,21 +1,27 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + # Clarifai -Anthropic, OpenAI, Mistral, Llama and Gemini LLMs are Supported on Clarifai. +Anthropic, OpenAI, Qwen, xAI, Gemini and most of Open soured LLMs are Supported on Clarifai. -:::warning - -Streaming is not yet supported on using clarifai and litellm. Tracking support here: https://github.com/BerriAI/litellm/issues/4162 - -::: +| Property | Details | +|-------|-------| +| Description | Clarifai is a powerful AI platform that provides access to a wide range of LLMs through a unified API. LiteLLM enables seamless integration with Clarifai's models using an OpenAI-compatible interface. | +| Provider Doc | [Clarifai ↗](https://docs.clarifai.com/) | +|OpenAI compatible Endpoint for Provider | `https://api.clarifai.com/v2/ext/openai/v1` | +| Supported Endpoints | `/chat/completions` | ## Pre-Requisites -`pip install litellm` + +```bash +pip install litellm +``` ## Required Environment Variables -To obtain your Clarifai Personal access token follow this [link](https://docs.clarifai.com/clarifai-basics/authentication/personal-access-tokens/). Optionally the PAT can also be passed in `completion` function. +To obtain your Clarifai Personal access token follow this [link](https://docs.clarifai.com/clarifai-basics/authentication/personal-access-tokens/). ```python -os.environ["CLARIFAI_API_KEY"] = "YOUR_CLARIFAI_PAT" # CLARIFAI_PAT - +os.environ["CLARIFAI_PAT"] = "CLARIFAI_API_KEY" # CLARIFAI_PAT ``` ## Usage @@ -27,154 +33,231 @@ from litellm import completion os.environ["CLARIFAI_API_KEY"] = "" response = completion( - model="clarifai/mistralai.completion.mistral-large", + model="clarifai/openai.chat-completion.gpt-oss-20b", messages=[{ "content": "Tell me a joke about physics?","role": "user"}] ) ``` +## Streaming Support -**Output** -```json -{ - "id": "chatcmpl-572701ee-9ab2-411c-ac75-46c1ba18e781", - "choices": [ - { - "finish_reason": "stop", - "index": 1, - "message": { - "content": "Sure, here's a physics joke for you:\n\nWhy can't you trust an atom?\n\nBecause they make up everything!", - "role": "assistant" - } - } +LiteLLM supports streaming responses with Clarifai models: + +```python +import litellm + +for chunk in litellm.completion( + model="clarifai/openai.chat-completion.gpt-oss-20b", + api_key="CLARIFAI_API_KEY", + messages=[ + {"role": "user", "content": "Tell me a fun fact about space."} ], - "created": 1714410197, - "model": "https://api.clarifai.com/v2/users/mistralai/apps/completion/models/mistral-large/outputs", - "object": "chat.completion", - "system_fingerprint": null, - "usage": { - "prompt_tokens": 14, - "completion_tokens": 24, - "total_tokens": 38 + stream=True, +): + print(chunk.choices[0].delta) +``` + +## Tool Calling (Function Calling) + +Clarifai models accessed via LiteLLM support function calling: + +```python +import litellm + +tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current temperature for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City and country e.g. Tokyo, Japan" + } + }, + "required": ["location"], + "additionalProperties": False + }, } } +}] + +response = litellm.completion( + model="clarifai/openai.chat-completion.gpt-oss-20b", + api_key="CLARIFAI_API_KEY", + messages=[{"role": "user", "content": "What is the weather in Paris today?"}], + tools=tools, +) + +print(response.choices[0].message.tool_calls) ``` ## Clarifai models liteLLM supports all models on [Clarifai community](https://clarifai.com/explore/models?filterData=%5B%7B%22field%22%3A%22use_cases%22%2C%22value%22%3A%5B%22llm%22%5D%7D%5D&page=1&perPage=24) -Example Usage - Note: liteLLM supports all models deployed on Clarifai - -## Llama LLMs -| Model Name | Function Call | ----------------------------|---------------------------------| -| clarifai/meta.Llama-2.llama2-7b-chat | `completion('clarifai/meta.Llama-2.llama2-7b-chat', messages)` -| clarifai/meta.Llama-2.llama2-13b-chat | `completion('clarifai/meta.Llama-2.llama2-13b-chat', messages)` -| clarifai/meta.Llama-2.llama2-70b-chat | `completion('clarifai/meta.Llama-2.llama2-70b-chat', messages)` | -| clarifai/meta.Llama-2.codeLlama-70b-Python | `completion('clarifai/meta.Llama-2.codeLlama-70b-Python', messages)`| -| clarifai/meta.Llama-2.codeLlama-70b-Instruct | `completion('clarifai/meta.Llama-2.codeLlama-70b-Instruct', messages)` | - -## Mistral LLMs -| Model Name | Function Call | -|---------------------------------------------|------------------------------------------------------------------------| -| clarifai/mistralai.completion.mixtral-8x22B | `completion('clarifai/mistralai.completion.mixtral-8x22B', messages)` | -| clarifai/mistralai.completion.mistral-large | `completion('clarifai/mistralai.completion.mistral-large', messages)` | -| clarifai/mistralai.completion.mistral-medium | `completion('clarifai/mistralai.completion.mistral-medium', messages)` | -| clarifai/mistralai.completion.mistral-small | `completion('clarifai/mistralai.completion.mistral-small', messages)` | -| clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1 | `completion('clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1', messages)` -| clarifai/mistralai.completion.mistral-7B-OpenOrca | `completion('clarifai/mistralai.completion.mistral-7B-OpenOrca', messages)` | -| clarifai/mistralai.completion.openHermes-2-mistral-7B | `completion('clarifai/mistralai.completion.openHermes-2-mistral-7B', messages)` | +### 🧠 OpenAI Models +- [gpt-oss-20b](https://clarifai.com/openai/chat-completion/models/gpt-oss-20b) +- [gpt-oss-120b](https://clarifai.com/openai/chat-completion/models/gpt-oss-120b) +- [gpt-5-nano](https://clarifai.com/openai/chat-completion/models/gpt-5-nano) +- [gpt-5-mini](https://clarifai.com/openai/chat-completion/models/gpt-5-mini) +- [gpt-5](https://clarifai.com/openai/chat-completion/models/gpt-5) +- [gpt-4o](https://clarifai.com/openai/chat-completion/models/gpt-4o) +- [o3](https://clarifai.com/openai/chat-completion/models/o3) +- Many more... -## Jurassic LLMs -| Model Name | Function Call | -|-----------------------------------------------|---------------------------------------------------------------------| -| clarifai/ai21.complete.Jurassic2-Grande | `completion('clarifai/ai21.complete.Jurassic2-Grande', messages)` | -| clarifai/ai21.complete.Jurassic2-Grande-Instruct | `completion('clarifai/ai21.complete.Jurassic2-Grande-Instruct', messages)` | -| clarifai/ai21.complete.Jurassic2-Jumbo-Instruct | `completion('clarifai/ai21.complete.Jurassic2-Jumbo-Instruct', messages)` | -| clarifai/ai21.complete.Jurassic2-Jumbo | `completion('clarifai/ai21.complete.Jurassic2-Jumbo', messages)` | -| clarifai/ai21.complete.Jurassic2-Large | `completion('clarifai/ai21.complete.Jurassic2-Large', messages)` | - -## Wizard LLMs - -| Model Name | Function Call | -|-----------------------------------------------|---------------------------------------------------------------------| -| clarifai/wizardlm.generate.wizardCoder-Python-34B | `completion('clarifai/wizardlm.generate.wizardCoder-Python-34B', messages)` | -| clarifai/wizardlm.generate.wizardLM-70B | `completion('clarifai/wizardlm.generate.wizardLM-70B', messages)` | -| clarifai/wizardlm.generate.wizardLM-13B | `completion('clarifai/wizardlm.generate.wizardLM-13B', messages)` | -| clarifai/wizardlm.generate.wizardCoder-15B | `completion('clarifai/wizardlm.generate.wizardCoder-15B', messages)` | - -## Anthropic models - -| Model Name | Function Call | -|-----------------------------------------------|---------------------------------------------------------------------| -| clarifai/anthropic.completion.claude-v1 | `completion('clarifai/anthropic.completion.claude-v1', messages)` | -| clarifai/anthropic.completion.claude-instant-1_2 | `completion('clarifai/anthropic.completion.claude-instant-1_2', messages)` | -| clarifai/anthropic.completion.claude-instant | `completion('clarifai/anthropic.completion.claude-instant', messages)` | -| clarifai/anthropic.completion.claude-v2 | `completion('clarifai/anthropic.completion.claude-v2', messages)` | -| clarifai/anthropic.completion.claude-2_1 | `completion('clarifai/anthropic.completion.claude-2_1', messages)` | -| clarifai/anthropic.completion.claude-3-opus | `completion('clarifai/anthropic.completion.claude-3-opus', messages)` | -| clarifai/anthropic.completion.claude-3-sonnet | `completion('clarifai/anthropic.completion.claude-3-sonnet', messages)` | - -## OpenAI GPT LLMs - -| Model Name | Function Call | -|-----------------------------------------------|---------------------------------------------------------------------| -| clarifai/openai.chat-completion.GPT-4 | `completion('clarifai/openai.chat-completion.GPT-4', messages)` | -| clarifai/openai.chat-completion.GPT-3_5-turbo | `completion('clarifai/openai.chat-completion.GPT-3_5-turbo', messages)` | -| clarifai/openai.chat-completion.gpt-4-turbo | `completion('clarifai/openai.chat-completion.gpt-4-turbo', messages)` | -| clarifai/openai.completion.gpt-3_5-turbo-instruct | `completion('clarifai/openai.completion.gpt-3_5-turbo-instruct', messages)` | - -## GCP LLMs - -| Model Name | Function Call | -|-----------------------------------------------|---------------------------------------------------------------------| -| clarifai/gcp.generate.gemini-1_5-pro | `completion('clarifai/gcp.generate.gemini-1_5-pro', messages)` | -| clarifai/gcp.generate.imagen-2 | `completion('clarifai/gcp.generate.imagen-2', messages)` | -| clarifai/gcp.generate.code-gecko | `completion('clarifai/gcp.generate.code-gecko', messages)` | -| clarifai/gcp.generate.code-bison | `completion('clarifai/gcp.generate.code-bison', messages)` | -| clarifai/gcp.generate.text-bison | `completion('clarifai/gcp.generate.text-bison', messages)` | -| clarifai/gcp.generate.gemma-2b-it | `completion('clarifai/gcp.generate.gemma-2b-it', messages)` | -| clarifai/gcp.generate.gemma-7b-it | `completion('clarifai/gcp.generate.gemma-7b-it', messages)` | -| clarifai/gcp.generate.gemini-pro | `completion('clarifai/gcp.generate.gemini-pro', messages)` | -| clarifai/gcp.generate.gemma-1_1-7b-it | `completion('clarifai/gcp.generate.gemma-1_1-7b-it', messages)` | - -## Cohere LLMs -| Model Name | Function Call | -|-----------------------------------------------|---------------------------------------------------------------------| -| clarifai/cohere.generate.cohere-generate-command | `completion('clarifai/cohere.generate.cohere-generate-command', messages)` | - clarifai/cohere.generate.command-r-plus' | `completion('clarifai/clarifai/cohere.generate.command-r-plus', messages)`| - -## Databricks LLMs - -| Model Name | Function Call | -|---------------------------------------------------|---------------------------------------------------------------------| -| clarifai/databricks.drbx.dbrx-instruct | `completion('clarifai/databricks.drbx.dbrx-instruct', messages)` | -| clarifai/databricks.Dolly-v2.dolly-v2-12b | `completion('clarifai/databricks.Dolly-v2.dolly-v2-12b', messages)`| - -## Microsoft LLMs - -| Model Name | Function Call | -|---------------------------------------------------|---------------------------------------------------------------------| -| clarifai/microsoft.text-generation.phi-2 | `completion('clarifai/microsoft.text-generation.phi-2', messages)` | -| clarifai/microsoft.text-generation.phi-1_5 | `completion('clarifai/microsoft.text-generation.phi-1_5', messages)`| - -## Salesforce models - -| Model Name | Function Call | -|-----------------------------------------------------------|-------------------------------------------------------------------------------| -| clarifai/salesforce.blip.general-english-image-caption-blip-2 | `completion('clarifai/salesforce.blip.general-english-image-caption-blip-2', messages)` | -| clarifai/salesforce.xgen.xgen-7b-8k-instruct | `completion('clarifai/salesforce.xgen.xgen-7b-8k-instruct', messages)` | +### 🤖 Anthropic Models +- [claude-sonnet-4](https://clarifai.com/anthropic/completion/models/claude-sonnet-4) +- [claude-opus-4](https://clarifai.com/anthropic/completion/models/claude-opus-4) +- [claude-3_5-haiku](https://clarifai.com/anthropic/completion/models/claude-3_5-haiku) +- [claude-3_7-sonnet](https://clarifai.com/anthropic/completion/models/claude-3_7-sonnet) +- Many more... -## Other Top performing LLMs +### 🪄 xAI Models +- [grok-3](https://clarifai.com/xai/chat-completion/models/grok-3) +- [grok-2-vision-1212](https://clarifai.com/xai/chat-completion/models/grok-2-vision-1212) +- [grok-2-1212](https://clarifai.com/xai/chat-completion/models/grok-2-1212) +- [grok-code-fast-1](https://clarifai.com/xai/chat-completion/models/grok-code-fast-1) +- [grok-2-image-1212](https://clarifai.com/xai/image-generation/models/grok-2-image-1212) +- Many more... -| Model Name | Function Call | -|---------------------------------------------------|---------------------------------------------------------------------| -| clarifai/deci.decilm.deciLM-7B-instruct | `completion('clarifai/deci.decilm.deciLM-7B-instruct', messages)` | -| clarifai/upstage.solar.solar-10_7b-instruct | `completion('clarifai/upstage.solar.solar-10_7b-instruct', messages)` | -| clarifai/openchat.openchat.openchat-3_5-1210 | `completion('clarifai/openchat.openchat.openchat-3_5-1210', messages)` | -| clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B | `completion('clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B', messages)` | -| clarifai/fblgit.una-cybertron.una-cybertron-7b-v2 | `completion('clarifai/fblgit.una-cybertron.una-cybertron-7b-v2', messages)` | -| clarifai/tiiuae.falcon.falcon-40b-instruct | `completion('clarifai/tiiuae.falcon.falcon-40b-instruct', messages)` | -| clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat | `completion('clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat', messages)` | -| clarifai/bigcode.code.StarCoder | `completion('clarifai/bigcode.code.StarCoder', messages)` | -| clarifai/mosaicml.mpt.mpt-7b-instruct | `completion('clarifai/mosaicml.mpt.mpt-7b-instruct', messages)` | + +### 🔷 Google Gemini Models +- [gemini-2_5-pro](https://clarifai.com/gcp/generate/models/gemini-2_5-pro) +- [gemini-2_5-flash-lite](https://clarifai.com/gcp/generate/models/gemini-2_5-flash-lite) +- [gemini-2_0-flash](https://clarifai.com/gcp/generate/models/gemini-2_0-flash) +- [gemini-2_0-flash-lite](https://clarifai.com/gcp/generate/models/gemini-2_0-flash-lite) +- Many more... + + +### 🧩 Qwen Models +- [Qwen3-30B-A3B-Instruct-2507](https://clarifai.com/qwen/qwenLM/models/Qwen3-30B-A3B-Instruct-2507) +- [Qwen3-30B-A3B-Thinking-2507](https://clarifai.com/qwen/qwenLM/models/Qwen3-30B-A3B-Thinking-2507) +- [Qwen3-14B](https://clarifai.com/qwen/qwenLM/models/Qwen3-14B) +- [QwQ-32B-AWQ](https://clarifai.com/qwen/qwenLM/models/QwQ-32B-AWQ) +- [Qwen2_5-VL-7B-Instruct](https://clarifai.com/qwen/qwen-VL/models/Qwen2_5-VL-7B-Instruct) +- [Qwen3-Coder-30B-A3B-Instruct](https://clarifai.com/qwen/qwenCoder/models/Qwen3-Coder-30B-A3B-Instruct) +- Many more... + + +### 💡 MiniCPM (OpenBMB) Models +- [MiniCPM-o-2_6-language](https://clarifai.com/openbmb/miniCPM/models/MiniCPM-o-2_6-language) +- [MiniCPM3-4B](https://clarifai.com/openbmb/miniCPM/models/MiniCPM3-4B) +- [MiniCPM4-8B](https://clarifai.com/openbmb/miniCPM/models/MiniCPM4-8B) +- Many more... + + +### 🧬 Microsoft Phi Models +- [Phi-4-reasoning-plus](https://clarifai.com/microsoft/text-generation/models/Phi-4-reasoning-plus) +- [phi-4](https://clarifai.com/microsoft/text-generation/models/phi-4) +- Many more... + + +### 🦙 Meta Llama Models +- [Llama-3_2-3B-Instruct](https://clarifai.com/meta/Llama-3/models/Llama-3_2-3B-Instruct) +- Many more... + + +### 🔍 DeepSeek Models +- [DeepSeek-R1-0528-Qwen3-8B](https://clarifai.com/deepseek-ai/deepseek-chat/models/DeepSeek-R1-0528-Qwen3-8B) +- Many more... + +## Usage with LiteLLM Proxy + +Here's how to call Clarifai with the LiteLLM Proxy Server + +### 1. Save key in your environment + +```bash +export CLARIFAI_PAT="CLARIFAI_API_KEY" +``` + +### 2. Start the proxy + + + + +```yaml +model_list: + - model_name: clarifai-model + litellm_params: + model: clarifai/openai.chat-completion.gpt-oss-20b + api_key: os.environ/CLARIFAI_PAT +``` + +```bash +litellm --config /path/to/config.yaml + +# Server running on http://0.0.0.0:4000 +``` + + + +### 3. Test it + + + + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data ' { + "model": "clarifai-model", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="clarifai-model", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ] +) + +print(response) +``` + + + +## Important Notes + +- Always prefix Clarifai model IDs with `clarifai/` when specifying the model name +- Use your Clarifai Personal Access Token (PAT) as the API key +- Usage is tracked and billed through Clarifai +- API rate limits are subject to your Clarifai account settings +- Most OpenAI parameters are supported, but some advanced features may vary by model + + +## FAQs + +| Question | Answer | +|----------|---------| +| Can I use all Clarifai models with LiteLLM? | Most chat-completion models are supported. Use the Clarifai model URL as the `model`. | +| Do I need a separate Clarifai PAT? | Yes, you must use a valid Clarifai Personal Access Token. | +| Is tool calling supported? | Yes, provided the underlying Clarifai model supports function/tool calling. | +| How is billing handled? | Clarifai usage is billed independently via Clarifai. | + +## Additional Resources + +- [Clarifai Documentation](https://docs.clarifai.com/) +- [LiteLLM GitHub](https://github.com/BerriAI/litellm) +- [Clarifai Runners Examples](https://github.com/Clarifai/runners-examples) \ No newline at end of file diff --git a/docs/my-website/docs/providers/cohere.md b/docs/my-website/docs/providers/cohere.md index 9c424010570..1c3181d1884 100644 --- a/docs/my-website/docs/providers/cohere.md +++ b/docs/my-website/docs/providers/cohere.md @@ -15,30 +15,51 @@ os.environ["COHERE_API_KEY"] = "" ### LiteLLM Python SDK +#### Cohere v2 API (Default) + ```python showLineNumbers from litellm import completion ## set ENV variables os.environ["COHERE_API_KEY"] = "cohere key" -# cohere call +# cohere v2 call response = completion( - model="command-r", + model="cohere_chat/command-a-03-2025", + messages = [{ "content": "Hello, how are you?","role": "user"}] +) +``` + +#### Cohere v1 API + +To use the Cohere v1/chat API, prefix your model name with `cohere_chat/v1/`: + +```python showLineNumbers +from litellm import completion + +## set ENV variables +os.environ["COHERE_API_KEY"] = "cohere key" + +# cohere v1 call +response = completion( + model="cohere_chat/v1/command-a-03-2025", messages = [{ "content": "Hello, how are you?","role": "user"}] ) ``` #### Streaming +**Cohere v2 Streaming:** + ```python showLineNumbers from litellm import completion ## set ENV variables os.environ["COHERE_API_KEY"] = "cohere key" -# cohere call +# cohere v2 streaming response = completion( - model="command-r", + model="cohere_chat/command-a-03-2025", messages = [{ "content": "Hello, how are you?","role": "user"}], stream=True ) @@ -48,6 +69,25 @@ for chunk in response: ``` +**Cohere v1 Streaming:** + +```python showLineNumbers +from litellm import completion + +## set ENV variables +os.environ["COHERE_API_KEY"] = "cohere key" + +# cohere v1 streaming +response = completion( + model="cohere_chat/v1/command-a-03-2025", + messages = [{ "content": "Hello, how are you?","role": "user"}], + stream=True +) + +for chunk in response: + print(chunk) +``` + ## Usage with LiteLLM Proxy @@ -63,11 +103,21 @@ export COHERE_API_KEY="your-api-key" Define the cohere models you want to use in the config.yaml +**For Cohere v1 models:** ```yaml showLineNumbers model_list: - model_name: command-a-03-2025 litellm_params: - model: command-a-03-2025 + model: cohere_chat/v1/command-a-03-2025 + api_key: "os.environ/COHERE_API_KEY" +``` + +**For Cohere v2 models:** +```yaml showLineNumbers +model_list: + - model_name: command-a-03-2025-v2 + litellm_params: + model: cohere_chat/command-a-03-2025 api_key: "os.environ/COHERE_API_KEY" ``` @@ -78,9 +128,8 @@ litellm --config /path/to/config.yaml ### 3. Test it - - + ```shell showLineNumbers curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -98,7 +147,25 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ' ``` - + + +```shell showLineNumbers +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data ' { + "model": "command-a-03-2025-v2", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + ```python showLineNumbers import openai @@ -107,7 +174,7 @@ client = openai.OpenAI( base_url="http://0.0.0.0:4000" ) -# request sent to model set on litellm proxy +# request sent to cohere v1 model response = client.chat.completions.create(model="command-a-03-2025", messages = [ { "role": "user", @@ -116,7 +183,26 @@ response = client.chat.completions.create(model="command-a-03-2025", messages = ]) print(response) +``` + + +```python showLineNumbers +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +# request sent to cohere v2 model +response = client.chat.completions.create(model="command-a-03-2025-v2", messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } +]) + +print(response) ``` diff --git a/docs/my-website/docs/providers/cometapi.md b/docs/my-website/docs/providers/cometapi.md index 1245bacfad4..a7f6e65519d 100644 --- a/docs/my-website/docs/providers/cometapi.md +++ b/docs/my-website/docs/providers/cometapi.md @@ -1,6 +1,10 @@ # CometAPI LiteLLM supports all AI models from [CometAPI](https://www.cometapi.com/). CometAPI provides access to 500+ AI models through a unified API interface, including cutting-edge models like GPT-5, Claude Opus 4.1, and various other state-of-the-art language models. + + Open In Colab + + ## Authentication To use CometAPI models, you need to obtain an API key from [CometAPI Token Console](https://api.cometapi.com/console/token). CometAPI offers free tokens for new users - you can get your free API key instantly by registering. diff --git a/docs/my-website/docs/providers/custom.md b/docs/my-website/docs/providers/custom.md deleted file mode 100644 index 81b92f0a031..00000000000 --- a/docs/my-website/docs/providers/custom.md +++ /dev/null @@ -1,69 +0,0 @@ -# Custom LLM API-Endpoints -LiteLLM supports Custom deploy api endpoints - -LiteLLM Expects the following input and output for custom LLM API endpoints - -### Model Details - -For calls to your custom API base ensure: -* Set `api_base="your-api-base"` -* Add `custom/` as a prefix to the `model` param. If your API expects `meta-llama/Llama-2-13b-hf` set `model=custom/meta-llama/Llama-2-13b-hf` - -| Model Name | Function Call | -|------------------|--------------------------------------------| -| meta-llama/Llama-2-13b-hf | `response = completion(model="custom/meta-llama/Llama-2-13b-hf", messages=messages, api_base="https://your-custom-inference-endpoint")` | -| meta-llama/Llama-2-13b-hf | `response = completion(model="custom/meta-llama/Llama-2-13b-hf", messages=messages, api_base="https://api.autoai.dev/inference")` | - -### Example Call to Custom LLM API using LiteLLM -```python -from litellm import completion -response = completion( - model="custom/meta-llama/Llama-2-13b-hf", - messages= [{"content": "what is custom llama?", "role": "user"}], - temperature=0.2, - max_tokens=10, - api_base="https://api.autoai.dev/inference", - request_timeout=300, -) -print("got response\n", response) -``` - -#### Setting your Custom API endpoint - -Inputs to your custom LLM api bases should follow this format: - -```python -resp = requests.post( - your-api_base, - json={ - 'model': 'meta-llama/Llama-2-13b-hf', # model name - 'params': { - 'prompt': ["The capital of France is P"], - 'max_tokens': 32, - 'temperature': 0.7, - 'top_p': 1.0, - 'top_k': 40, - } - } -) -``` - -Outputs from your custom LLM api bases should follow this format: -```python -{ - 'data': [ - { - 'prompt': 'The capital of France is P', - 'output': [ - 'The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France' - ], - 'params': { - 'temperature': 0.7, - 'top_k': 40, - 'top_p': 1 - } - } - ], - 'message': 'ok' -} -``` \ No newline at end of file diff --git a/docs/my-website/docs/providers/fal_ai.md b/docs/my-website/docs/providers/fal_ai.md new file mode 100644 index 00000000000..e50ef919da0 --- /dev/null +++ b/docs/my-website/docs/providers/fal_ai.md @@ -0,0 +1,311 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Fal AI + +Fal AI provides fast, scalable access to state-of-the-art image generation models including FLUX, Stable Diffusion, Imagen, and more. + +## Overview + +| Property | Details | +|----------|---------| +| Description | Fal AI offers optimized infrastructure for running image generation models at scale with low latency. | +| Provider Route on LiteLLM | `fal_ai/` | +| Provider Doc | [Fal AI Documentation ↗](https://fal.ai/models) | +| Supported Operations | [`/images/generations`](#image-generation) | + +## Setup + +### API Key + +```python showLineNumbers +import os + +# Set your Fal AI API key +os.environ["FAL_AI_API_KEY"] = "your-fal-api-key" +``` + +Get your API key from [fal.ai](https://fal.ai/). + +## Supported Models + +| Model Name | Description | Documentation | +|------------|-------------|---------------| +| `fal_ai/flux/schnell` | Flux Schnell - Low-latency generation with `image_size` support | [Docs ↗](https://fal.ai/models/fal-ai/flux/schnell) | +| `fal_ai/fal-ai/flux-pro/v1.1-ultra` | FLUX Pro v1.1 Ultra - High-quality image generation | [Docs ↗](https://fal.ai/models/fal-ai/flux-pro/v1.1-ultra) | +| `fal_ai/fal-ai/imagen4/preview` | Google's Imagen 4 - Highest quality model | [Docs ↗](https://fal.ai/models/fal-ai/imagen4/preview) | +| `fal_ai/fal-ai/recraft/v3/text-to-image` | Recraft v3 - Multiple style options | [Docs ↗](https://fal.ai/models/fal-ai/recraft/v3/text-to-image) | +| `fal_ai/fal-ai/stable-diffusion-v35-medium` | Stable Diffusion v3.5 Medium | [Docs ↗](https://fal.ai/models/fal-ai/stable-diffusion-v35-medium) | +| `fal_ai/bria/text-to-image/3.2` | Bria 3.2 - Commercial-grade generation | [Docs ↗](https://fal.ai/models/bria/text-to-image/3.2) | + +## Image Generation + +### Usage - LiteLLM Python SDK + + + + +```python showLineNumbers title="Basic Image Generation" +import litellm +import os + +# Set your API key +os.environ["FAL_AI_API_KEY"] = "your-fal-api-key" + +# Generate an image +response = litellm.image_generation( + model="fal_ai/fal-ai/flux-pro/v1.1-ultra", + prompt="A serene mountain landscape at sunset with vibrant colors" +) + +print(response.data[0].url) +``` + + + + + +```python showLineNumbers title="Google Imagen 4 Generation" +import litellm +import os + +os.environ["FAL_AI_API_KEY"] = "your-fal-api-key" + +# Generate with Imagen 4 +response = litellm.image_generation( + model="fal_ai/fal-ai/imagen4/preview", + prompt="A vintage 1960s kitchen with flour package on countertop", + aspect_ratio="16:9", + num_images=1 +) + +print(response.data[0].url) +``` + + + + + +```python showLineNumbers title="Recraft v3 with Style" +import litellm +import os + +os.environ["FAL_AI_API_KEY"] = "your-fal-api-key" + +# Generate with specific style +response = litellm.image_generation( + model="fal_ai/fal-ai/recraft/v3/text-to-image", + prompt="A red panda eating bamboo", + style="realistic_image", + image_size="landscape_4_3" +) + +print(response.data[0].url) +``` + + + + + +```python showLineNumbers title="Async Image Generation" +import litellm +import asyncio +import os + +async def generate_image(): + os.environ["FAL_AI_API_KEY"] = "your-fal-api-key" + + response = await litellm.aimage_generation( + model="fal_ai/fal-ai/stable-diffusion-v35-medium", + prompt="A cyberpunk cityscape with neon lights", + guidance_scale=7.5, + num_inference_steps=50 + ) + + print(response.data[0].url) + return response + +asyncio.run(generate_image()) +``` + + + + + +```python showLineNumbers title="Advanced FLUX Pro Generation" +import litellm +import os + +os.environ["FAL_AI_API_KEY"] = "your-fal-api-key" + +# Generate with advanced parameters +response = litellm.image_generation( + model="fal_ai/fal-ai/flux-pro/v1.1-ultra", + prompt="A majestic dragon soaring over mountains", + n=2, + size="1792x1024", # Maps to aspect_ratio="16:9" + seed=42, + safety_tolerance="2", + enhance_prompt=True +) + +for image in response.data: + print(f"Generated image: {image.url}") +``` + + + + +### Usage - LiteLLM Proxy Server + +#### 1. Configure your config.yaml + +```yaml showLineNumbers title="Fal AI Image Generation Configuration" +model_list: + - model_name: flux-ultra + litellm_params: + model: fal_ai/fal-ai/flux-pro/v1.1-ultra + api_key: os.environ/FAL_AI_API_KEY + model_info: + mode: image_generation + + - model_name: imagen4 + litellm_params: + model: fal_ai/fal-ai/imagen4/preview + api_key: os.environ/FAL_AI_API_KEY + model_info: + mode: image_generation + + - model_name: stable-diffusion + litellm_params: + model: fal_ai/fal-ai/stable-diffusion-v35-medium + api_key: os.environ/FAL_AI_API_KEY + model_info: + mode: image_generation + +general_settings: + master_key: sk-1234 +``` + +#### 2. Start LiteLLM Proxy Server + +```bash showLineNumbers title="Start Proxy Server" +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +#### 3. Make requests + + + + +```python showLineNumbers title="Generate via Proxy - OpenAI SDK" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="sk-1234" +) + +response = client.images.generate( + model="flux-ultra", + prompt="A beautiful sunset over the ocean", + n=1, + size="1024x1024" +) + +print(response.data[0].url) +``` + + + + + +```python showLineNumbers title="Generate via Proxy - LiteLLM SDK" +import litellm + +response = litellm.image_generation( + model="litellm_proxy/imagen4", + prompt="A cozy coffee shop interior", + api_base="http://localhost:4000", + api_key="sk-1234" +) + +print(response.data[0].url) +``` + + + + + +```bash showLineNumbers title="Generate via Proxy - cURL" +curl --location 'http://localhost:4000/v1/images/generations' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer sk-1234' \ +--data '{ + "model": "stable-diffusion", + "prompt": "A serene Japanese garden with cherry blossoms", + "n": 1, + "size": "1024x1024" +}' +``` + + + + + + +## Using Model-Specific Parameters + +LiteLLM forwards any additional parameters directly to the Fal AI API. You can pass model-specific parameters in your request and they will be sent to Fal AI. + +```python showLineNumbers title="Pass Model-Specific Parameters" +import litellm + +# Any parameters beyond the standard ones are forwarded to Fal AI +response = litellm.image_generation( + model="fal_ai/fal-ai/flux-pro/v1.1-ultra", + prompt="A beautiful sunset", + # Model-specific Fal AI parameters + aspect_ratio="16:9", + safety_tolerance="2", + enhance_prompt=True, + seed=42 +) +``` + +For the complete list of parameters supported by each model, see: +- [FLUX Pro v1.1-ultra Parameters ↗](https://fal.ai/models/fal-ai/flux-pro/v1.1-ultra/api) +- [Imagen 4 Parameters ↗](https://fal.ai/models/fal-ai/imagen4/preview/api) +- [Recraft v3 Parameters ↗](https://fal.ai/models/fal-ai/recraft/v3/text-to-image/api) +- [Stable Diffusion v3.5 Parameters ↗](https://fal.ai/models/fal-ai/stable-diffusion-v35-medium/api) +- [Bria 3.2 Parameters ↗](https://fal.ai/models/bria/text-to-image/3.2/api) + +## Supported Parameters + +Standard OpenAI-compatible parameters that work across all models: + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `prompt` | string | Text description of desired image | Required | +| `model` | string | Fal AI model to use | Required | +| `n` | integer | Number of images to generate (1-4) | `1` | +| `size` | string | Image dimensions (maps to model-specific format) | Model default | +| `api_key` | string | Your Fal AI API key | Environment variable | + +## Getting Started + +1. Sign up at [fal.ai](https://fal.ai/) +2. Get your API key from your account settings +3. Set `FAL_AI_API_KEY` environment variable +4. Choose a model from the [Fal AI model gallery](https://fal.ai/models) +5. Start generating images with LiteLLM + +## Additional Resources + +- [Fal AI Documentation](https://fal.ai/docs) +- [Model Gallery](https://fal.ai/models) +- [API Reference](https://fal.ai/docs/api-reference) +- [Pricing](https://fal.ai/pricing) + diff --git a/docs/my-website/docs/providers/fireworks_ai.md b/docs/my-website/docs/providers/fireworks_ai.md index 98d7c33ce7e..b1b10cd71b5 100644 --- a/docs/my-website/docs/providers/fireworks_ai.md +++ b/docs/my-website/docs/providers/fireworks_ai.md @@ -204,7 +204,7 @@ from litellm import completion import os os.environ["FIREWORKS_AI_API_KEY"] = "YOUR_API_KEY" -os.environ["FIREWORKS_AI_API_BASE"] = "https://audio-prod.us-virginia-1.direct.fireworks.ai/v1" +os.environ["FIREWORKS_AI_API_BASE"] = "https://audio-prod.api.fireworks.ai/v1" completion = litellm.completion( model="fireworks_ai/accounts/fireworks/models/llama-v3p3-70b-instruct", @@ -343,7 +343,7 @@ from litellm import transcription import os os.environ["FIREWORKS_AI_API_KEY"] = "YOUR_API_KEY" -os.environ["FIREWORKS_AI_API_BASE"] = "https://audio-prod.us-virginia-1.direct.fireworks.ai/v1" +os.environ["FIREWORKS_AI_API_BASE"] = "https://audio-prod.api.fireworks.ai/v1" response = transcription( model="fireworks_ai/whisper-v3", @@ -363,7 +363,7 @@ model_list: - model_name: whisper-v3 litellm_params: model: fireworks_ai/whisper-v3 - api_base: https://audio-prod.us-virginia-1.direct.fireworks.ai/v1 + api_base: https://audio-prod.api.fireworks.ai/v1 api_key: os.environ/FIREWORKS_API_KEY model_info: mode: audio_transcription diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index 40d64656528..c5014fc2ff3 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -10,7 +10,7 @@ import TabItem from '@theme/TabItem'; | Provider Route on LiteLLM | `gemini/` | | Provider Doc | [Google AI Studio ↗](https://aistudio.google.com/) | | API Endpoint for Provider | https://generativelanguage.googleapis.com | -| Supported OpenAI Endpoints | `/chat/completions`, [`/embeddings`](../embedding/supported_embedding#gemini-ai-embedding-models), `/completions` | +| Supported OpenAI Endpoints | `/chat/completions`, [`/embeddings`](../embedding/supported_embedding#gemini-ai-embedding-models), `/completions`, [`/videos`](./gemini/videos.md), [`/images/edits`](../image_edits.md) | | Pass-through Endpoint | [Supported](../pass_through/google_ai_studio.md) |
@@ -64,16 +64,21 @@ response = completion( LiteLLM translates OpenAI's `reasoning_effort` to Gemini's `thinking` parameter. [Code](https://github.com/BerriAI/litellm/blob/620664921902d7a9bfb29897a7b27c1a7ef4ddfb/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py#L362) -Added an additional non-OpenAI standard "disable" value for non-reasoning Gemini requests. +**Cost Optimization:** Use `reasoning_effort="none"` (OpenAI standard) for significant cost savings - up to 96% cheaper. [Google's docs](https://ai.google.dev/gemini-api/docs/openai) + +:::info +Note: Reasoning cannot be turned off on Gemini 2.5 Pro models. +::: **Mapping** -| reasoning_effort | thinking | -| ---------------- | -------- | -| "disable" | "budget_tokens": 0 | -| "low" | "budget_tokens": 1024 | -| "medium" | "budget_tokens": 2048 | -| "high" | "budget_tokens": 4096 | +| reasoning_effort | thinking | Notes | +| ---------------- | -------- | ----- | +| "none" | "budget_tokens": 0, "includeThoughts": false | 💰 **Recommended for cost optimization** - OpenAI-compatible, always 0 | +| "disable" | "budget_tokens": DEFAULT (0), "includeThoughts": false | LiteLLM-specific, configurable via env var | +| "low" | "budget_tokens": 1024 | | +| "medium" | "budget_tokens": 2048 | | +| "high" | "budget_tokens": 4096 | | @@ -81,6 +86,14 @@ Added an additional non-OpenAI standard "disable" value for non-reasoning Gemini ```python from litellm import completion +# Cost-optimized: Use reasoning_effort="none" for best pricing +resp = completion( + model="gemini/gemini-2.0-flash-thinking-exp-01-21", + messages=[{"role": "user", "content": "What is the capital of France?"}], + reasoning_effort="none", # Up to 96% cheaper! +) + +# Or use other levels: "low", "medium", "high" resp = completion( model="gemini/gemini-2.5-flash-preview-04-17", messages=[{"role": "user", "content": "What is the capital of France?"}], diff --git a/docs/my-website/docs/providers/gemini/videos.md b/docs/my-website/docs/providers/gemini/videos.md new file mode 100644 index 00000000000..5b5d5a8a636 --- /dev/null +++ b/docs/my-website/docs/providers/gemini/videos.md @@ -0,0 +1,409 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Gemini Video Generation (Veo) + +LiteLLM supports Google's Veo video generation models through a unified API interface. + +| Property | Details | +|-------|-------| +| Description | Google's Veo AI video generation models | +| Provider Route on LiteLLM | `gemini/` | +| Supported Models | `veo-3.0-generate-preview`, `veo-3.1-generate-preview` | +| Cost Tracking | ✅ Duration-based pricing | +| Logging Support | ✅ Full request/response logging | +| Proxy Server Support | ✅ Full proxy integration with virtual keys | +| Spend Management | ✅ Budget tracking and rate limiting | +| Link to Provider Doc | [Google Veo Documentation ↗](https://ai.google.dev/gemini-api/docs/video) | + +## Quick Start + +### Required API Keys + +```python +import os +os.environ["GEMINI_API_KEY"] = "your-google-api-key" +# OR +os.environ["GOOGLE_API_KEY"] = "your-google-api-key" +``` + +### Basic Usage + +```python +from litellm import video_generation, video_status, video_content +import os +import time + +os.environ["GEMINI_API_KEY"] = "your-google-api-key" + +# Step 1: Generate video +response = video_generation( + model="gemini/veo-3.0-generate-preview", + prompt="A cat playing with a ball of yarn in a sunny garden" +) + +print(f"Video ID: {response.id}") +print(f"Initial Status: {response.status}") # "processing" + +# Step 2: Poll for completion +while True: + status_response = video_status( + video_id=response.id + ) + + print(f"Current Status: {status_response.status}") + + if status_response.status == "completed": + break + elif status_response.status == "failed": + print("Video generation failed") + break + + time.sleep(10) # Wait 10 seconds before checking again + +# Step 3: Download video content +video_bytes = video_content( + video_id=response.id +) + +# Save to file +with open("generated_video.mp4", "wb") as f: + f.write(video_bytes) + +print("Video downloaded successfully!") +``` + +## Supported Models + +| Model Name | Description | Max Duration | Status | +|------------|-------------|--------------|--------| +| veo-3.0-generate-preview | Veo 3.0 video generation | 8 seconds | Preview | +| veo-3.1-generate-preview | Veo 3.1 video generation | 8 seconds | Preview | + +## Video Generation Parameters + +LiteLLM automatically maps OpenAI-style parameters to Veo's format: + +| OpenAI Parameter | Veo Parameter | Description | Example | +|------------------|---------------|-------------|---------| +| `prompt` | `prompt` | Text description of the video | "A cat playing" | +| `size` | `aspectRatio` | Video dimensions → aspect ratio | "1280x720" → "16:9" | +| `seconds` | `durationSeconds` | Duration in seconds | "8" → 8 | +| `input_reference` | `image` | Reference image to animate | File object or path | +| `model` | `model` | Model to use | "gemini/veo-3.0-generate-preview" | + +### Size to Aspect Ratio Mapping + +LiteLLM automatically converts size dimensions to Veo's aspect ratio format: +- `"1280x720"`, `"1920x1080"` → `"16:9"` (landscape) +- `"720x1280"`, `"1080x1920"` → `"9:16"` (portrait) + +### Supported Veo Parameters + +Based on Veo's API: +- **prompt** (required): Text description with optional audio cues +- **aspectRatio**: `"16:9"` (default) or `"9:16"` +- **resolution**: `"720p"` (default) or `"1080p"` (Veo 3.1 only, 16:9 aspect ratio only) +- **durationSeconds**: Video length (max 8 seconds for most models) +- **image**: Reference image for animation +- **negativePrompt**: What to exclude from the video (Veo 3.1) +- **referenceImages**: Style and content references (Veo 3.1 only) + +## Complete Workflow Example + +```python +import litellm +import time + +def generate_and_download_veo_video( + prompt: str, + output_file: str = "video.mp4", + size: str = "1280x720", + seconds: str = "8" +): + """ + Complete workflow for Veo video generation. + + Args: + prompt: Text description of the video + output_file: Where to save the video + size: Video dimensions (e.g., "1280x720" for 16:9) + seconds: Duration in seconds + + Returns: + bool: True if successful + """ + print(f"🎬 Generating video: {prompt}") + + # Step 1: Initiate generation + response = litellm.video_generation( + model="gemini/veo-3.0-generate-preview", + prompt=prompt, + size=size, # Maps to aspectRatio + seconds=seconds # Maps to durationSeconds + ) + + video_id = response.id + print(f"✓ Video generation started (ID: {video_id})") + + # Step 2: Wait for completion + max_wait_time = 600 # 10 minutes + start_time = time.time() + + while time.time() - start_time < max_wait_time: + status_response = litellm.video_status(video_id=video_id) + + if status_response.status == "completed": + print("✓ Video generation completed!") + break + elif status_response.status == "failed": + print("✗ Video generation failed") + return False + + print(f"⏳ Status: {status_response.status}") + time.sleep(10) + else: + print("✗ Timeout waiting for video generation") + return False + + # Step 3: Download video + print("⬇️ Downloading video...") + video_bytes = litellm.video_content(video_id=video_id) + + with open(output_file, "wb") as f: + f.write(video_bytes) + + print(f"✓ Video saved to {output_file}") + return True + +# Use it +generate_and_download_veo_video( + prompt="A serene lake at sunset with mountains in the background", + output_file="sunset_lake.mp4" +) +``` + +## Async Usage + +```python +from litellm import avideo_generation, avideo_status, avideo_content +import asyncio + +async def async_video_workflow(): + # Generate video + response = await avideo_generation( + model="gemini/veo-3.0-generate-preview", + prompt="A cat playing with a ball of yarn" + ) + + # Poll for completion + while True: + status = await avideo_status(video_id=response.id) + if status.status == "completed": + break + await asyncio.sleep(10) + + # Download content + video_bytes = await avideo_content(video_id=response.id) + + with open("video.mp4", "wb") as f: + f.write(video_bytes) + +# Run it +asyncio.run(async_video_workflow()) +``` + +## LiteLLM Proxy Usage + +### Configuration + +Add Veo models to your `config.yaml`: + +```yaml +model_list: + - model_name: veo-3 + litellm_params: + model: gemini/veo-3.0-generate-preview + api_key: os.environ/GEMINI_API_KEY +``` + +Start the proxy: + +```bash +litellm --config config.yaml +# Server running on http://0.0.0.0:4000 +``` + +### Making Requests + + + + +```bash +# Step 1: Generate video +curl --location 'http://0.0.0.0:4000/v1/videos' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer sk-1234' \ +--data '{ + "model": "veo-3", + "prompt": "A cat playing with a ball of yarn in a sunny garden" +}' + +# Response: {"id": "gemini::operations/generate_12345::...", "status": "processing", ...} + +# Step 2: Check status +curl --location 'http://localhost:4000/v1/videos/{video_id}' \ +--header 'x-litellm-api-key: sk-1234' + +# Step 3: Download video (when status is "completed") +curl --location 'http://localhost:4000/v1/videos/{video_id}/content' \ +--header 'x-litellm-api-key: sk-1234' \ +--output video.mp4 +``` + + + + +```python +import litellm + +litellm.api_base = "http://0.0.0.0:4000" +litellm.api_key = "sk-1234" + +# Generate video +response = litellm.video_generation( + model="veo-3", + prompt="A cat playing with a ball of yarn in a sunny garden" +) + +# Check status +import time +while True: + status = litellm.video_status(video_id=response.id) + if status.status == "completed": + break + time.sleep(10) + +# Download video +video_bytes = litellm.video_content(video_id=response.id) +with open("video.mp4", "wb") as f: + f.write(video_bytes) +``` + + + + +## Cost Tracking + +LiteLLM automatically tracks costs for Veo video generation: + +```python +response = litellm.video_generation( + model="gemini/veo-3.0-generate-preview", + prompt="A beautiful sunset" +) + +# Cost is calculated based on video duration +# Veo pricing: ~$0.10 per second (estimated) +# Default video duration: ~5 seconds +# Estimated cost: ~$0.50 +``` + +## Differences from OpenAI Video API + +| Feature | OpenAI (Sora) | Gemini (Veo) | +|---------|---------------|--------------| +| Reference Images | ✅ Supported | ❌ Not supported | +| Size Control | ✅ Supported | ❌ Not supported | +| Duration Control | ✅ Supported | ❌ Not supported | +| Video Remix/Edit | ✅ Supported | ❌ Not supported | +| Video List | ✅ Supported | ❌ Not supported | +| Prompt-based Generation | ✅ Supported | ✅ Supported | +| Async Operations | ✅ Supported | ✅ Supported | + +## Error Handling + +```python +from litellm import video_generation, video_status, video_content +from litellm.exceptions import APIError, Timeout + +try: + response = video_generation( + model="gemini/veo-3.0-generate-preview", + prompt="A beautiful landscape" + ) + + # Poll with timeout + max_attempts = 60 # 10 minutes (60 * 10s) + for attempt in range(max_attempts): + status = video_status(video_id=response.id) + + if status.status == "completed": + video_bytes = video_content(video_id=response.id) + with open("video.mp4", "wb") as f: + f.write(video_bytes) + break + elif status.status == "failed": + raise APIError("Video generation failed") + + time.sleep(10) + else: + raise Timeout("Video generation timed out") + +except APIError as e: + print(f"API Error: {e}") +except Timeout as e: + print(f"Timeout: {e}") +except Exception as e: + print(f"Unexpected error: {e}") +``` + +## Best Practices + +1. **Always poll for completion**: Veo video generation is asynchronous and can take several minutes +2. **Set reasonable timeouts**: Allow at least 5-10 minutes for video generation +3. **Handle failures gracefully**: Check for `failed` status and implement retry logic +4. **Use descriptive prompts**: More detailed prompts generally produce better results +5. **Store video IDs**: Save the operation ID/video ID to resume polling if your application restarts + +## Troubleshooting + +### Video generation times out + +```python +# Increase polling timeout +max_wait_time = 900 # 15 minutes instead of 10 +``` + +### Video not found when downloading + +```python +# Make sure video is completed before downloading +status = video_status(video_id=video_id) +if status.status != "completed": + print("Video not ready yet!") +``` + +### API key errors + +```python +# Verify your API key is set +import os +print(os.environ.get("GEMINI_API_KEY")) + +# Or pass it explicitly +response = video_generation( + model="gemini/veo-3.0-generate-preview", + prompt="...", + api_key="your-api-key-here" +) +``` + +## See Also + +- [OpenAI Video Generation](../openai/videos.md) +- [Azure Video Generation](../azure/videos.md) +- [Vertex AI Video Generation](../vertex_ai/videos.md) +- [Video Generation API Reference](/docs/videos) +- [Veo Pass-through Endpoints](/docs/pass_through/google_ai_studio#example-4-video-generation-with-veo) + diff --git a/docs/my-website/docs/providers/github.md b/docs/my-website/docs/providers/github.md index b9e525ef5c1..51220166140 100644 --- a/docs/my-website/docs/providers/github.md +++ b/docs/my-website/docs/providers/github.md @@ -1,7 +1,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# 🆕 Github +# Github https://github.com/marketplace/models :::tip diff --git a/docs/my-website/docs/providers/google_ai_studio/files.md b/docs/my-website/docs/providers/google_ai_studio/files.md index 500f1d57185..ce61ce1a90b 100644 --- a/docs/my-website/docs/providers/google_ai_studio/files.md +++ b/docs/my-website/docs/providers/google_ai_studio/files.md @@ -39,7 +39,7 @@ encoded_string = base64.b64encode(wav_data).decode('utf-8') file = create_file( file=wav_data, purpose="user_data", - extra_body={"custom_llm_provider": "gemini"}, + extra_headers={"custom-llm-provider": "gemini"}, api_key=os.getenv("GEMINI_API_KEY"), ) diff --git a/docs/my-website/docs/providers/milvus_vector_stores.md b/docs/my-website/docs/providers/milvus_vector_stores.md new file mode 100644 index 00000000000..84f16fbc74a --- /dev/null +++ b/docs/my-website/docs/providers/milvus_vector_stores.md @@ -0,0 +1,528 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Milvus - Vector Store + +Use Milvus as a vector store for RAG. + +## Quick Start + +You need three things: +1. A Milvus instance (cloud or self-hosted) +2. An embedding model (to convert your queries to vectors) +3. A Milvus collection with vector fields + +## Usage + + + + +### Basic Search + +```python +from litellm import vector_stores +import os + +# Set your credentials +os.environ["MILVUS_API_KEY"] = "your-milvus-api-key" +os.environ["MILVUS_API_BASE"] = "https://your-milvus-instance.milvus.io" + +# Search the vector store +response = vector_stores.search( + vector_store_id="my-collection-name", # Your Milvus collection name + query="What is the capital of France?", + custom_llm_provider="milvus", + litellm_embedding_model="azure/text-embedding-3-large", + litellm_embedding_config={ + "api_base": "your-embedding-endpoint", + "api_key": "your-embedding-api-key", + "api_version": "2025-09-01" + }, + milvus_text_field="book_intro", # Field name that contains text content + api_key=os.getenv("MILVUS_API_KEY"), +) + +print(response) +``` + +### Async Search + +```python +from litellm import vector_stores + +response = await vector_stores.asearch( + vector_store_id="my-collection-name", + query="What is the capital of France?", + custom_llm_provider="milvus", + litellm_embedding_model="azure/text-embedding-3-large", + litellm_embedding_config={ + "api_base": "your-embedding-endpoint", + "api_key": "your-embedding-api-key", + "api_version": "2025-09-01" + }, + milvus_text_field="book_intro", + api_key=os.getenv("MILVUS_API_KEY"), +) + +print(response) +``` + +### Advanced Options + +```python +from litellm import vector_stores + +response = vector_stores.search( + vector_store_id="my-collection-name", + query="What is the capital of France?", + custom_llm_provider="milvus", + litellm_embedding_model="azure/text-embedding-3-large", + litellm_embedding_config={ + "api_base": "your-embedding-endpoint", + "api_key": "your-embedding-api-key", + }, + milvus_text_field="book_intro", + api_key=os.getenv("MILVUS_API_KEY"), + # Milvus-specific parameters + limit=10, # Number of results to return + offset=0, # Pagination offset + dbName="default", # Database name + annsField="book_intro_vector", # Vector field name + outputFields=["id", "book_intro", "title"], # Fields to return + filter='book_id > 0', # Metadata filter expression + searchParams={"metric_type": "L2", "params": {"nprobe": 10}}, # Search parameters +) + +print(response) +``` + + + + + +### Setup Config + +Add this to your config.yaml: + +```yaml +vector_store_registry: + - vector_store_name: "milvus-knowledgebase" + litellm_params: + vector_store_id: "my-collection-name" + custom_llm_provider: "milvus" + api_key: os.environ/MILVUS_API_KEY + api_base: https://your-milvus-instance.milvus.io + litellm_embedding_model: "azure/text-embedding-3-large" + litellm_embedding_config: + api_base: https://your-endpoint.cognitiveservices.azure.com/ + api_key: os.environ/AZURE_API_KEY + api_version: "2025-09-01" + milvus_text_field: "book_intro" + # Optional Milvus parameters + annsField: "book_intro_vector" + limit: 10 +``` + +### Start Proxy + +```bash +litellm --config /path/to/config.yaml +``` + +### Search via API + +```bash +curl -X POST 'http://0.0.0.0:4000/v1/vector_stores/my-collection-name/search' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "query": "What is the capital of France?" +}' +``` + + + + +## Required Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `vector_store_id` | string | Your Milvus collection name | +| `custom_llm_provider` | string | Set to `"milvus"` | +| `litellm_embedding_model` | string | Model to generate query embeddings (e.g., `"azure/text-embedding-3-large"`) | +| `litellm_embedding_config` | dict | Config for the embedding model (api_base, api_key, api_version) | +| `milvus_text_field` | string | Field name in your collection that contains text content | +| `api_key` | string | Your Milvus API key (or set `MILVUS_API_KEY` env var) | +| `api_base` | string | Your Milvus API base URL (or set `MILVUS_API_BASE` env var) | + +## Optional Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `dbName` | string | Database name (default: "default") | +| `annsField` | string | Vector field name to search (default: "book_intro_vector") | +| `limit` | integer | Maximum number of results to return | +| `offset` | integer | Pagination offset | +| `filter` | string | Filter expression for metadata filtering | +| `groupingField` | string | Field to group results by | +| `outputFields` | list | List of fields to return in results | +| `searchParams` | dict | Search parameters like metric type and search parameters | +| `partitionNames` | list | List of partition names to search | +| `consistencyLevel` | string | Consistency level for the search | + +## Supported Features + +| Feature | Status | Notes | +|---------|--------|-------| +| Logging | ✅ Supported | Full logging support available | +| Guardrails | ❌ Not Yet Supported | Guardrails are not currently supported for vector stores | +| Cost Tracking | ✅ Supported | Cost is $0 for Milvus searches | +| Unified API | ✅ Supported | Call via OpenAI compatible `/v1/vector_stores/search` endpoint | +| Passthrough | ✅ Supported | Use native Milvus API format | + +## Response Format + +The response follows the standard LiteLLM vector store format: + +```json +{ + "object": "vector_store.search_results.page", + "search_query": "What is the capital of France?", + "data": [ + { + "score": 0.95, + "content": [ + { + "text": "Paris is the capital of France...", + "type": "text" + } + ], + "file_id": null, + "filename": null, + "attributes": { + "id": "123", + "title": "France Geography" + } + } + ] +} +``` + +## Passthrough API (Native Milvus Format) + +Use this to allow developers to **create** and **search** vector stores using the native Milvus API format, without giving them the Milvus credentials. + +This is for the proxy only. + +### Admin Flow + +#### 1. Add the vector store to LiteLLM + +```yaml +model_list: + - model_name: embedding-model + litellm_params: + model: azure/text-embedding-3-large + api_base: https://your-endpoint.cognitiveservices.azure.com/ + api_key: os.environ/AZURE_API_KEY + api_version: "2025-09-01" + +vector_store_registry: + - vector_store_name: "milvus-store" + litellm_params: + vector_store_id: "can-be-anything" # vector store id can be anything for the purpose of passthrough api + custom_llm_provider: "milvus" + api_key: os.environ/MILVUS_API_KEY + api_base: https://your-milvus-instance.milvus.io + +general_settings: + database_url: "postgresql://user:password@host:port/database" + master_key: "sk-1234" +``` + +Add your vector store credentials to LiteLLM. + +#### 2. Start the proxy + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +#### 3. Create a virtual index + +```bash +curl -L -X POST 'http://0.0.0.0:4000/v1/indexes' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "index_name": "dall-e-6", + "litellm_params": { + "vector_store_index": "real-collection-name", + "vector_store_name": "milvus-store" + } +}' +``` + +This is a virtual index, which the developer can use to create and search vector stores. + +#### 4. Create a key with the vector store permissions + +```bash +curl -L -X POST 'http://0.0.0.0:4000/key/generate' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "allowed_vector_store_indexes": [{"index_name": "dall-e-6", "index_permissions": ["write", "read"]}], + "models": ["embedding-model"] +}' +``` + +Give the key access to the virtual index and the embedding model. + +**Expected response** + +```json +{ + "key": "sk-my-virtual-key" +} +``` + +### Developer Flow + +#### 1. Create a collection with schema + +Note: Use the `/milvus` endpoint for the passthrough api that uses the `milvus` provider in your config. + +```python +from milvus_rest_client import MilvusRESTClient, DataType +import random +import time + +# Configuration +uri = "http://0.0.0.0:4000/milvus" # IMPORTANT: Use the '/milvus' endpoint for passthrough +token = "sk-my-virtual-key" +collection_name = "dall-e-6" # Virtual index name + +# Initialize client +milvus_client = MilvusRESTClient(uri=uri, token=token) +print(f"Connected to DB: {uri} successfully") + +# Check if the collection exists and drop if it does +check_collection = milvus_client.has_collection(collection_name) +if check_collection: + milvus_client.drop_collection(collection_name) + print(f"Dropped the existing collection {collection_name} successfully") + +# Define schema +dim = 64 # Vector dimension + +print("Start to create the collection schema") +schema = milvus_client.create_schema() +schema.add_field( + "book_id", DataType.INT64, is_primary=True, description="customized primary id" +) +schema.add_field("word_count", DataType.INT64, description="word count") +schema.add_field( + "book_intro", DataType.FLOAT_VECTOR, dim=dim, description="book introduction" +) + +# Prepare index parameters +print("Start to prepare index parameters with default AUTOINDEX") +index_params = milvus_client.prepare_index_params() +index_params.add_index("book_intro", metric_type="L2") + +# Create collection +print(f"Start to create example collection: {collection_name}") +milvus_client.create_collection( + collection_name, schema=schema, index_params=index_params +) +collection_property = milvus_client.describe_collection(collection_name) +print("Collection details: %s" % collection_property) +``` + +#### 2. Insert data into the collection + +```python +# Insert data with customized ids +nb = 1000 +insert_rounds = 2 +start = 0 # first primary key id +total_rt = 0 # total response time for insert + +print( + f"Start to insert {nb*insert_rounds} entities into example collection: {collection_name}" +) +for i in range(insert_rounds): + vector = [random.random() for _ in range(dim)] + rows = [ + {"book_id": i, "word_count": random.randint(1, 100), "book_intro": vector} + for i in range(start, start + nb) + ] + t0 = time.time() + milvus_client.insert(collection_name, rows) + ins_rt = time.time() - t0 + start += nb + total_rt += ins_rt +print(f"Insert completed in {round(total_rt, 4)} seconds") + +# Flush the collection +print("Start to flush") +start_flush = time.time() +milvus_client.flush(collection_name) +end_flush = time.time() +print(f"Flush completed in {round(end_flush - start_flush, 4)} seconds") +``` + +#### 3. Search the collection + +```python +# Search configuration +nq = 3 # Number of query vectors +search_params = {"metric_type": "L2", "params": {"level": 2}} +limit = 2 # Number of results to return + +# Perform searches +for i in range(5): + search_vectors = [[random.random() for _ in range(dim)] for _ in range(nq)] + t0 = time.time() + results = milvus_client.search( + collection_name, + data=search_vectors, + limit=limit, + search_params=search_params, + anns_field="book_intro", + ) + t1 = time.time() + print(f"Search {i} results: {results}") + print(f"Search {i} latency: {round(t1-t0, 4)} seconds") +``` + +#### Complete Example + +Here's a full working example: + +```python +from milvus_rest_client import MilvusRESTClient, DataType +import random +import time + +# ---------------------------- +# 🔐 CONFIGURATION +# ---------------------------- +uri = "http://0.0.0.0:4000/milvus" # IMPORTANT: Use the '/milvus' endpoint +token = "sk-my-virtual-key" +collection_name = "dall-e-6" # Your virtual index name + +# ---------------------------- +# 📋 STEP 1 — Initialize Client +# ---------------------------- +milvus_client = MilvusRESTClient(uri=uri, token=token) +print(f"✅ Connected to DB: {uri} successfully") + +# ---------------------------- +# 🗑️ STEP 2 — Drop Existing Collection (if needed) +# ---------------------------- +check_collection = milvus_client.has_collection(collection_name) +if check_collection: + milvus_client.drop_collection(collection_name) + print(f"🗑️ Dropped the existing collection {collection_name} successfully") + +# ---------------------------- +# 📐 STEP 3 — Create Collection Schema +# ---------------------------- +dim = 64 # Vector dimension + +print("📐 Creating the collection schema") +schema = milvus_client.create_schema() +schema.add_field( + "book_id", DataType.INT64, is_primary=True, description="customized primary id" +) +schema.add_field("word_count", DataType.INT64, description="word count") +schema.add_field( + "book_intro", DataType.FLOAT_VECTOR, dim=dim, description="book introduction" +) + +# ---------------------------- +# 🔍 STEP 4 — Create Index +# ---------------------------- +print("🔍 Preparing index parameters with default AUTOINDEX") +index_params = milvus_client.prepare_index_params() +index_params.add_index("book_intro", metric_type="L2") + +# ---------------------------- +# 🏗️ STEP 5 — Create Collection +# ---------------------------- +print(f"🏗️ Creating collection: {collection_name}") +milvus_client.create_collection( + collection_name, schema=schema, index_params=index_params +) +collection_property = milvus_client.describe_collection(collection_name) +print(f"✅ Collection created: {collection_property}") + +# ---------------------------- +# 📤 STEP 6 — Insert Data +# ---------------------------- +nb = 1000 +insert_rounds = 2 +start = 0 +total_rt = 0 + +print(f"📤 Inserting {nb*insert_rounds} entities into collection") +for i in range(insert_rounds): + vector = [random.random() for _ in range(dim)] + rows = [ + {"book_id": i, "word_count": random.randint(1, 100), "book_intro": vector} + for i in range(start, start + nb) + ] + t0 = time.time() + milvus_client.insert(collection_name, rows) + ins_rt = time.time() - t0 + start += nb + total_rt += ins_rt +print(f"✅ Insert completed in {round(total_rt, 4)} seconds") + +# ---------------------------- +# 💾 STEP 7 — Flush Collection +# ---------------------------- +print("💾 Flushing collection") +start_flush = time.time() +milvus_client.flush(collection_name) +end_flush = time.time() +print(f"✅ Flush completed in {round(end_flush - start_flush, 4)} seconds") + +# ---------------------------- +# 🔍 STEP 8 — Search +# ---------------------------- +nq = 3 +search_params = {"metric_type": "L2", "params": {"level": 2}} +limit = 2 + +print(f"🔍 Performing {5} search operations") +for i in range(5): + search_vectors = [[random.random() for _ in range(dim)] for _ in range(nq)] + t0 = time.time() + results = milvus_client.search( + collection_name, + data=search_vectors, + limit=limit, + search_params=search_params, + anns_field="book_intro", + ) + t1 = time.time() + print(f"✅ Search {i} results: {results}") + print(f" Search {i} latency: {round(t1-t0, 4)} seconds") +``` + +## How It Works + +When you search: + +1. LiteLLM converts your query to a vector using the embedding model you specified +2. It sends the vector to your Milvus instance via the `/v2/vectordb/entities/search` endpoint +3. Milvus finds the most similar documents in your collection using vector similarity search +4. Results come back with distance scores + +The embedding model can be any model supported by LiteLLM - Azure OpenAI, OpenAI, Bedrock, etc. + diff --git a/docs/my-website/docs/providers/oci.md b/docs/my-website/docs/providers/oci.md index 1f52fba04f3..cea5d6824a0 100644 --- a/docs/my-website/docs/providers/oci.md +++ b/docs/my-website/docs/providers/oci.md @@ -29,17 +29,38 @@ Check the [OCI Models List](https://docs.oracle.com/en-us/iaas/Content/generativ ## Authentication -LiteLLM uses OCI signing key authentication. Follow the [official Oracle tutorial](https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm) to create a signing key and obtain the following parameters: +LiteLLM supports two authentication methods for OCI: + +### Method 1: Manual Credentials +Provide individual OCI credentials directly to LiteLLM. Follow the [official Oracle tutorial](https://docs.oracle.com/en-us/iaas/Content/API/Concepts/apisigningkey.htm) to create a signing key and obtain the following parameters: - `user` - `fingerprint` - `tenancy` - `region` -- `key_file` +- `key_file` or `key` +- `compartment_id` + +This is the default method for LiteLLM AI Gateway (LLM Proxy) access to OCI GenAI models. + +### Method 2: OCI SDK Signer +Use an OCI SDK `Signer` object for authentication. This method: +- Leverages the official [OCI SDK for signing](https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html) +- Supports additional authentication methods (instance principals, workload identity, etc.) + +To use this method, install the OCI SDK: +```bash +pip install oci +``` + +This method is an alternative when using the LiteLLM SDK on Oracle Cloud Infrastructure (instances or Oracle Kubernetes Engine). ## Usage -Input the parameters obtained from the OCI signing key creation process into the `completion` function. + + + +Input the parameters obtained from the OCI signing key creation process into the `completion` function: ```python import os @@ -64,10 +85,119 @@ response = completion( print(response) ``` + + + +Use the OCI SDK `Signer` for authentication: + +```python +from litellm import completion +from oci.signer import Signer + +# Create an OCI Signer +signer = Signer( + tenancy="ocid1.tenancy.oc1..", + user="ocid1.user.oc1..", + fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx", + private_key_file_location="~/.oci/key.pem", + # Or use private_key_content="" +) + +messages = [{"role": "user", "content": "Hey! how's it going?"}] +response = completion( + model="oci/xai.grok-4", + messages=messages, + oci_signer=signer, + oci_region="us-chicago-1", # Optional, defaults to us-ashburn-1 + oci_serving_mode="ON_DEMAND", # Optional, default is "ON_DEMAND". Other option is "DEDICATED" + oci_compartment_id="", +) +print(response) +``` + +**Alternative: Use OCI Config File** + +The OCI SDK can automatically load credentials from `~/.oci/config`: + +```python +from litellm import completion +from oci.config import from_file +from oci.signer import Signer + +# Load config from file +config = from_file("~/.oci/config", "DEFAULT") # "DEFAULT" is the profile name +signer = Signer( + tenancy=config["tenancy"], + user=config["user"], + fingerprint=config["fingerprint"], + private_key_file_location=config["key_file"], + pass_phrase=config.get("pass_phrase") # Optional if key is encrypted +) + +messages = [{"role": "user", "content": "Hey! how's it going?"}] +response = completion( + model="oci/xai.grok-4", + messages=messages, + oci_signer=signer, + oci_region=config["region"], + oci_compartment_id="", +) +print(response) +``` + +**Instance Principal Authentication** + +For applications running on OCI compute instances: + +```python +from litellm import completion +from oci.auth.signers import InstancePrincipalsSecurityTokenSigner + +oci.auth.signers.get_oke_workload_identity_resource_principal_signer() +# Use instance principal authentication +signer = InstancePrincipalsSecurityTokenSigner() + +messages = [{"role": "user", "content": "Hey! how's it going?"}] +response = completion( + model="oci/xai.grok-4", + messages=messages, + oci_signer=signer, + oci_region="us-chicago-1", + oci_compartment_id="", +) +print(response) +``` + +**Use workload identity authentication** + +For applications running in Oracle Kubernetes Engine (OKE): + +```python +from litellm import completion +from oci.auth.signers import get_oke_workload_identity_resource_principal_signer + +# Use instance principal authentication +signer = get_oke_workload_identity_resource_principal_signer() + +messages = [{"role": "user", "content": "Hey! how's it going?"}] +response = completion( + model="oci/xai.grok-4", + messages=messages, + oci_signer=signer, + oci_region="us-chicago-1", + oci_compartment_id="", +) +print(response) +``` + + ## Usage - Streaming Just set `stream=True` when calling completion. + + + ```python import os from litellm import completion @@ -93,10 +223,68 @@ for chunk in response: print(chunk["choices"][0]["delta"]["content"]) # same as openai format ``` + + + +```python +from litellm import completion +from oci.signer import Signer + +signer = Signer( + tenancy="ocid1.tenancy.oc1..", + user="ocid1.user.oc1..", + fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx", + private_key_file_location="~/.oci/key.pem", +) + +messages = [{"role": "user", "content": "Hey! how's it going?"}] +response = completion( + model="oci/xai.grok-4", + messages=messages, + stream=True, + oci_signer=signer, + oci_region="us-chicago-1", + oci_compartment_id="", +) +for chunk in response: + print(chunk["choices"][0]["delta"]["content"]) # same as openai format +``` + + + + ## Usage Examples by Model Type ### Using Cohere Models + + + +```python +from litellm import completion +from oci.signer import Signer + +signer = Signer( + tenancy="ocid1.tenancy.oc1..", + user="ocid1.user.oc1..", + fingerprint="xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx", + private_key_file_location="~/.oci/key.pem", +) + +messages = [{"role": "user", "content": "Explain quantum computing"}] +response = completion( + model="oci/cohere.command-latest", + messages=messages, + oci_signer=signer, + oci_region="us-chicago-1", + oci_compartment_id="", +) +print(response) +``` + + + + ```python from litellm import completion @@ -112,4 +300,7 @@ response = completion( oci_compartment_id=, ) print(response) -``` \ No newline at end of file +``` + + + \ No newline at end of file diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index 3fad78dc80e..51ebc881d22 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -4,6 +4,10 @@ import TabItem from '@theme/TabItem'; # OpenAI LiteLLM supports OpenAI Chat + Embedding calls. +:::tip +**We recommend using `litellm.responses()` / Responses API** for the latest OpenAI models (GPT-5, gpt-5-codex, o3-mini, etc.) +::: + ### Required API Keys ```python @@ -406,6 +410,77 @@ Expected Response: ``` +### Advanced: Using `reasoning_effort` with `summary` field + +By default, `reasoning_effort` accepts a string value (`"low"`, `"medium"`, `"high"`, `"minimal"`) and only sets the effort level without including a reasoning summary. + +To opt-in to the `summary` feature, you can pass `reasoning_effort` as a dictionary. **Note:** The `summary` field requires your OpenAI organization to have verification status. Using `summary` without verification will result in a 400 error from OpenAI. + + + +```python +# Option 1: String format (default - no summary) +response = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=[{"role": "user", "content": "What is the capital of France?"}], + reasoning_effort="high" # Only sets effort level +) + +# Option 2: Dict format (with optional summary - requires org verification) +response = litellm.completion( + model="openai/responses/gpt-5-mini", + messages=[{"role": "user", "content": "What is the capital of France?"}], + reasoning_effort={"effort": "high", "summary": "auto"} # "auto", "detailed", or "concise" (not all supported by all models) +) +``` + + + +```bash +# Option 1: String format (default - no summary) +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "openai/responses/gpt-5-mini", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "reasoning_effort": "high" +}' + +# Option 2: Dict format (with optional summary - requires org verification) +# summary options: "auto", "detailed", or "concise" (not all supported by all models) +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "openai/responses/gpt-5-mini", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "reasoning_effort": {"effort": "high", "summary": "auto"} +}' +``` + + + +**Summary field options:** +- `"auto"`: System automatically determines the appropriate summary level based on the model +- `"concise"`: Provides a shorter summary (not supported by GPT-5 series models) +- `"detailed"`: Offers a comprehensive reasoning summary + +**Note:** GPT-5 series models support `"auto"` and `"detailed"`, but do not support `"concise"`. O-series models (o3-pro, o4-mini, o3) support all three options. Some models like o3-mini and o1 do not support reasoning summaries at all. + +**Supported `reasoning_effort` values by model:** + +| Model | Default (when not set) | Supported Values | +|-------|----------------------|------------------| +| `gpt-5` | `medium` | `minimal`, `low`, `medium`, `high` | +| `gpt-5-mini` | `medium` | `minimal`, `low`, `medium`, `high` | +| `gpt-5-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) | +| `gpt-5-pro` | `high` | `high` only | + +**Note:** `gpt-5-pro` only accepts `reasoning_effort="high"`. Other values will return an error. When `reasoning_effort` is not set (None), OpenAI defaults to the value shown in the "Default" column. + +See [OpenAI Reasoning documentation](https://platform.openai.com/docs/guides/reasoning) for more details on organization verification requirements. + ## OpenAI Chat Completion to Responses API Bridge Call any Responses API model from OpenAI's `/chat/completions` endpoint. @@ -836,4 +911,10 @@ response = completion( model="gpt-5-pro", messages=[{"role": "user", "content": "Solve this complex reasoning problem..."}] ) -``` \ No newline at end of file +``` + +## Video Generation + +LiteLLM supports OpenAI's video generation models including Sora. + +For detailed documentation on video generation, see [OpenAI Video Generation →](./openai/video_generation.md) \ No newline at end of file diff --git a/docs/my-website/docs/providers/openai/text_to_speech.md b/docs/my-website/docs/providers/openai/text_to_speech.md index 34cd0f069e6..a4aeb9e5257 100644 --- a/docs/my-website/docs/providers/openai/text_to_speech.md +++ b/docs/my-website/docs/providers/openai/text_to_speech.md @@ -4,6 +4,18 @@ import TabItem from '@theme/TabItem'; # OpenAI - Text-to-speech +## Overview + +| Feature | Supported | Notes | +|---------|-----------|-------| +| Cost Tracking | ✅ | Works with all supported models | +| Logging | ✅ | Works across all integrations | +| End-user Tracking | ✅ | | +| Fallbacks | ✅ | Works between supported models | +| Loadbalancing | ✅ | Works between supported models | +| Guardrails | ✅ | Applies to input text | +| Supported Models | tts-1, tts-1-hd, gpt-4o-mini-tts | | + ## **LiteLLM Python SDK Usage** ### Quick Start diff --git a/docs/my-website/docs/providers/openai/videos.md b/docs/my-website/docs/providers/openai/videos.md new file mode 100644 index 00000000000..202c79c2446 --- /dev/null +++ b/docs/my-website/docs/providers/openai/videos.md @@ -0,0 +1,247 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# OpenAI Video Generation + +LiteLLM supports OpenAI's video generation models including Sora. + +## Quick Start + +### Required API Keys + +```python +import os +os.environ["OPENAI_API_KEY"] = "your-api-key" +``` + +### Basic Usage + +```python +from litellm import video_generation, video_content +import os + +os.environ["OPENAI_API_KEY"] = "your-api-key" + +# Generate a video +response = video_generation( + prompt="A cat playing with a ball of yarn in a sunny garden", + model="sora-2", + seconds="8", + size="720x1280" +) + +print(f"Video ID: {response.id}") +print(f"Status: {response.status}") + +# Download video content when ready +video_bytes = video_content( + video_id=response.id, +) + +# Save to file +with open("generated_video.mp4", "wb") as f: + f.write(video_bytes) +``` + +## **LiteLLM Proxy Usage** + +LiteLLM provides OpenAI API compatible video endpoints for complete video generation workflow: + +- `/videos/generations` - Generate new videos +- `/videos/remix` - Edit existing videos with reference images +- `/videos/status` - Check video generation status +- `/videos/retrieval` - Download completed videos + +**Setup** + +Add this to your litellm proxy config.yaml + +```yaml +model_list: + - model_name: sora-2 + litellm_params: + model: openai/sora-2 + api_key: os.environ/OPENAI_API_KEY +``` + +Start litellm + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +Test video generation request + +```bash +curl --location 'http://localhost:4000/v1/videos' \ +--header 'Content-Type: application/json' \ +--header 'x-litellm-api-key: sk-1234' \ +--data '{ + "model": "sora-2", + "prompt": "A beautiful sunset over the ocean" +}' +``` + +Test video status request + +```bash +# Using custom-llm-provider header +curl --location 'http://localhost:4000/v1/videos/video_id' \ +--header 'Accept: application/json' \ +--header 'x-litellm-api-key: sk-1234' \ +--header 'custom-llm-provider: openai' +``` + +Test video retrieval request + +```bash +# Using custom-llm-provider header +curl --location 'http://localhost:4000/v1/videos/video_id/content' \ +--header 'Accept: application/json' \ +--header 'x-litellm-api-key: sk-1234' \ +--header 'custom-llm-provider: openai' \ +--output video.mp4 + +# Or using query parameter +curl --location 'http://localhost:4000/v1/videos/video_id/content?custom_llm_provider=openai' \ +--header 'Accept: application/json' \ +--header 'x-litellm-api-key: sk-1234' \ +--output video.mp4 +``` + +Test video remix request + +```bash +# Using custom_llm_provider in request body +curl --location --request POST 'http://localhost:4000/v1/videos/video_id/remix' \ +--header 'Accept: application/json' \ +--header 'Content-Type: application/json' \ +--header 'x-litellm-api-key: sk-1234' \ +--data '{ + "prompt": "New remix instructions", + "custom_llm_provider": "openai" +}' + +# Or using custom-llm-provider header +curl --location --request POST 'http://localhost:4000/v1/videos/video_id/remix' \ +--header 'Accept: application/json' \ +--header 'Content-Type: application/json' \ +--header 'x-litellm-api-key: sk-1234' \ +--header 'custom-llm-provider: openai' \ +--data '{ + "prompt": "New remix instructions" +}' +``` + +Test OpenAI video generation request + +```bash +curl http://localhost:4000/v1/videos \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "sora-2", + "prompt": "A cat playing with a ball of yarn in a sunny garden", + "seconds": "8", + "size": "720x1280" + }' +``` + + +## Supported Models + +| Model Name | Description | Max Duration | Supported Sizes | +|------------|-------------|--------------|-----------------| +| sora-2 | OpenAI's latest video generation model | 8 seconds | 720x1280, 1280x720 | + +## Video Generation Parameters + +- `prompt` (required): Text description of the desired video +- `model` (optional): Model to use, defaults to "sora-2" +- `seconds` (optional): Video duration in seconds (e.g., "8", "16") +- `size` (optional): Video dimensions (e.g., "720x1280", "1280x720") +- `input_reference` (optional): Reference image for video editing +- `user` (optional): User identifier for tracking + +## Video Content Retrieval + +```python +# Download video content +video_bytes = video_content( + video_id="video_1234567890" +) + +# Save to file +with open("video.mp4", "wb") as f: + f.write(video_bytes) +``` + +## Complete Workflow + +```python +import litellm +import time + +def generate_and_download_video(prompt): + # Step 1: Generate video + response = litellm.video_generation( + prompt=prompt, + model="sora-2", + seconds="8", + size="720x1280" + ) + + video_id = response.id + print(f"Video ID: {video_id}") + + # Step 2: Wait for processing (in practice, poll status) + time.sleep(30) + + # Step 3: Download video + video_bytes = litellm.video_content( + video_id=video_id + ) + + # Step 4: Save to file + with open(f"video_{video_id}.mp4", "wb") as f: + f.write(video_bytes) + + return f"video_{video_id}.mp4" + +# Usage +video_file = generate_and_download_video( + "A cat playing with a ball of yarn in a sunny garden" +) +``` + + +## Video Editing with Reference Images + +```python +# Video editing with reference image +response = litellm.video_generation( + prompt="Make the cat jump higher", + input_reference=open("path/to/image.jpg", "rb"), # Reference image + model="sora-2", + seconds="8" +) + +print(f"Video ID: {response.id}") +``` + +## Error Handling + +```python +from litellm.exceptions import BadRequestError, AuthenticationError + +try: + response = video_generation( + prompt="A cat playing with a ball of yarn" + ) +except AuthenticationError as e: + print(f"Authentication failed: {e}") +except BadRequestError as e: + print(f"Bad request: {e}") +``` diff --git a/docs/my-website/docs/providers/openrouter.md b/docs/my-website/docs/providers/openrouter.md index 58a87f68495..327634909b3 100644 --- a/docs/my-website/docs/providers/openrouter.md +++ b/docs/my-website/docs/providers/openrouter.md @@ -9,10 +9,9 @@ LiteLLM supports all the text / chat / vision models from [OpenRouter](https://o ```python import os from litellm import completion + os.environ["OPENROUTER_API_KEY"] = "" os.environ["OPENROUTER_API_BASE"] = "" # [OPTIONAL] defaults to https://openrouter.ai/api/v1 - - os.environ["OR_SITE_URL"] = "" # [OPTIONAL] os.environ["OR_APP_NAME"] = "" # [OPTIONAL] @@ -22,8 +21,32 @@ response = completion( ) ``` -## OpenRouter Completion Models +## Configuration with Environment Variables +For production environments, you can dynamically configure the base_url using environment variables: + +```python +import os +from litellm import completion + +# Configure with environment variables +OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY") +OPENROUTER_BASE_URL = os.getenv("OPENROUTER_API_BASE", "https://openrouter.ai/api/v1") + +# Set environment for LiteLLM +os.environ["OPENROUTER_API_KEY"] = OPENROUTER_API_KEY +os.environ["OPENROUTER_API_BASE"] = OPENROUTER_BASE_URL + +response = completion( + model="openrouter/google/palm-2-chat-bison", + messages=messages, + base_url=OPENROUTER_BASE_URL # Explicitly pass base_url for clarity +) +``` + +This approach provides better flexibility for managing configurations across different environments (dev, staging, production) and makes it easier to switch between self-hosted and cloud endpoints. + +## OpenRouter Completion Models 🚨 LiteLLM supports ALL OpenRouter models, send `model=openrouter/` to send it to open router. See all openrouter models [here](https://openrouter.ai/models) | Model Name | Function Call | @@ -40,12 +63,12 @@ response = completion( | openrouter/meta-llama/llama-2-70b-chat | `completion('openrouter/meta-llama/llama-2-70b-chat', messages)` | `os.environ['OR_SITE_URL']`,`os.environ['OR_APP_NAME']`,`os.environ['OPENROUTER_API_KEY']` | ## Passing OpenRouter Params - transforms, models, route - Pass `transforms`, `models`, `route`as arguments to `litellm.completion()` ```python import os from litellm import completion + os.environ["OPENROUTER_API_KEY"] = "" response = completion( @@ -54,4 +77,4 @@ response = completion( transforms = [""], route= "" ) -``` \ No newline at end of file +``` diff --git a/docs/my-website/docs/providers/runwayml/images.md b/docs/my-website/docs/providers/runwayml/images.md new file mode 100644 index 00000000000..00146d10baa --- /dev/null +++ b/docs/my-website/docs/providers/runwayml/images.md @@ -0,0 +1,198 @@ +# RunwayML - Image Generation + +## Overview + +| Property | Details | +|-------|-------| +| Description | RunwayML provides advanced AI-powered image generation with high-quality results | +| Provider Route on LiteLLM | `runwayml/` | +| Supported Operations | [`/images/generations`](#quick-start) | +| Link to Provider Doc | [RunwayML API ↗](https://docs.dev.runwayml.com/) | + +LiteLLM supports RunwayML's Gen-4 image generation API, allowing you to generate high-quality images from text prompts. + +## Quick Start + +```python showLineNumbers title="Basic Image Generation" +from litellm import image_generation +import os + +os.environ["RUNWAYML_API_KEY"] = "your-api-key" + +response = image_generation( + model="runwayml/gen4_image", + prompt="A serene mountain landscape at sunset", + size="1920x1080" +) + +print(response.data[0].url) +``` + +## Authentication + +Set your RunwayML API key: + +```python showLineNumbers title="Set API Key" +import os + +os.environ["RUNWAYML_API_KEY"] = "your-api-key" +``` + +## Supported Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `model` | string | Yes | Model to use (e.g., `runwayml/gen4_image`) | +| `prompt` | string | Yes | Text description for the image | +| `size` | string | No | Image dimensions (default: `1920x1080`) | + +### Supported Sizes + +- `1024x1024` +- `1792x1024` +- `1024x1792` +- `1920x1080` (default) +- `1080x1920` + +## Async Usage + +```python showLineNumbers title="Async Image Generation" +from litellm import aimage_generation +import os +import asyncio + +os.environ["RUNWAYML_API_KEY"] = "your-api-key" + +async def generate_image(): + response = await aimage_generation( + model="runwayml/gen4_image", + prompt="A futuristic city skyline at night", + size="1920x1080" + ) + + print(response.data[0].url) + +asyncio.run(generate_image()) +``` + +## LiteLLM Proxy Usage + +Add RunwayML to your proxy configuration: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: gen4-image + litellm_params: + model: runwayml/gen4_image + api_key: os.environ/RUNWAYML_API_KEY +``` + +Start the proxy: + +```bash +litellm --config /path/to/config.yaml +``` + +Generate images through the proxy: + +```bash showLineNumbers title="Proxy Request" +curl --location 'http://localhost:4000/v1/images/generations' \ +--header 'Content-Type: application/json' \ +--header 'x-litellm-api-key: sk-1234' \ +--data '{ + "model": "runwayml/gen4_image", + "prompt": "A serene mountain landscape at sunset", + "size": "1920x1080" +}' +``` + +## Supported Models + +| Model | Description | Default Size | +|-------|-------------|--------------| +| `runwayml/gen4_image` | High-quality image generation | 1920x1080 | + +## Cost Tracking + +LiteLLM automatically tracks RunwayML image generation costs: + +```python showLineNumbers title="Cost Tracking" +from litellm import image_generation, completion_cost + +response = image_generation( + model="runwayml/gen4_image", + prompt="A serene mountain landscape at sunset", + size="1920x1080" +) + +cost = completion_cost(completion_response=response) +print(f"Image generation cost: ${cost}") +``` + +## Supported Features + +| Feature | Supported | +|---------|-----------| +| Image Generation | ✅ | +| Cost Tracking | ✅ | +| Logging | ✅ | +| Fallbacks | ✅ | +| Load Balancing | ✅ | + + + +## How It Works + +RunwayML uses an asynchronous task-based API pattern. LiteLLM handles the polling and response transformation automatically. + +### Complete Flow Diagram + +```mermaid +sequenceDiagram + participant Client + box rgb(200, 220, 255) LiteLLM AI Gateway + participant LiteLLM + end + participant RunwayML as RunwayML API + + Client->>LiteLLM: POST /images/generations (OpenAI format) + Note over LiteLLM: Transform to RunwayML format + + LiteLLM->>RunwayML: POST v1/text_to_image + RunwayML-->>LiteLLM: 200 OK + task ID + + Note over LiteLLM: Automatic Polling + loop Every 2 seconds + LiteLLM->>RunwayML: GET v1/tasks/{task_id} + RunwayML-->>LiteLLM: Status: RUNNING + end + + LiteLLM->>RunwayML: GET v1/tasks/{task_id} + RunwayML-->>LiteLLM: Status: SUCCEEDED + image URL + + Note over LiteLLM: Transform to OpenAI format + LiteLLM-->>Client: Image Response (OpenAI format) +``` + +### What LiteLLM Does For You + +When you call `litellm.image_generation()` or `/v1/images/generations`: + +1. **Request Transformation**: Converts OpenAI image generation format → RunwayML format +2. **Submits Task**: Sends transformed request to RunwayML API +3. **Receives Task ID**: Captures the task ID from the initial response +4. **Automatic Polling**: + - Polls the task status endpoint every 2 seconds + - Continues until status is `SUCCEEDED` or `FAILED` + - Default timeout: 10 minutes (configurable via `RUNWAYML_POLLING_TIMEOUT`) +5. **Response Transformation**: Converts RunwayML format → OpenAI format +6. **Returns Result**: Sends unified OpenAI format response to client + +**Polling Configuration:** +- Default timeout: 600 seconds (10 minutes) +- Configurable via `RUNWAYML_POLLING_TIMEOUT` environment variable +- Uses sync (`time.sleep()`) or async (`await asyncio.sleep()`) based on call type + +:::info +**Typical processing time**: 10-30 seconds depending on image size and complexity +::: diff --git a/docs/my-website/docs/providers/runwayml/text-to-speech.md b/docs/my-website/docs/providers/runwayml/text-to-speech.md new file mode 100644 index 00000000000..020269863c6 --- /dev/null +++ b/docs/my-website/docs/providers/runwayml/text-to-speech.md @@ -0,0 +1,244 @@ +# RunwayML - Text-to-Speech + +## Overview + +| Property | Details | +|-------|-------| +| Description | RunwayML provides high-quality AI-powered text-to-speech with natural-sounding voices | +| Provider Route on LiteLLM | `runwayml/` | +| Supported Operations | [`/audio/speech`](#quick-start) | +| Link to Provider Doc | [RunwayML API ↗](https://docs.dev.runwayml.com/) | + +LiteLLM supports RunwayML's text-to-speech API with automatic task polling, allowing you to generate natural-sounding audio from text. + +## Quick Start + +```python showLineNumbers title="Basic Text-to-Speech" +from litellm import speech +import os + +os.environ["RUNWAYML_API_KEY"] = "your-api-key" + +response = speech( + model="runwayml/eleven_multilingual_v2", + input="Step right up, ladies and gentlemen! Have you ever wished for a toaster that's not just a toaster but a marvel of modern ingenuity?", + voice="alloy" +) + +# Save the audio +with open("output.mp3", "wb") as f: + f.write(response.content) +``` + +## Authentication + +Set your RunwayML API key: + +```python showLineNumbers title="Set API Key" +import os + +os.environ["RUNWAYML_API_KEY"] = "your-api-key" +``` + +## Supported Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `model` | string | Yes | Model to use (e.g., `runwayml/eleven_multilingual_v2`) | +| `input` | string | Yes | Text to convert to speech | +| `voice` | string or dict | Yes | Voice to use (OpenAI name, RunwayML preset, or voice config) | + +## Voice Options + +### Using OpenAI Voice Names + +OpenAI voice names are automatically mapped to appropriate RunwayML voices: + +```python showLineNumbers title="OpenAI Voice Names" +from litellm import speech + +# These OpenAI voice names work automatically +response = speech( + model="runwayml/eleven_multilingual_v2", + input="Hello, world!", + voice="alloy" # Maya - neutral, balanced female voice +) +``` + +**Voice Mappings:** +- `alloy` → Maya (neutral, balanced female voice) +- `echo` → James (male voice) +- `fable` → Bernard (warm, storytelling voice) +- `onyx` → Vincent (deep male voice) +- `nova` → Serene (warm, expressive female voice) +- `shimmer` → Ella (clear, friendly female voice) + +### Using RunwayML Preset Voices + +You can directly specify any RunwayML preset voice by passing the preset name as a string: + +```python showLineNumbers title="RunwayML Preset Names" +from litellm import speech + +# Pass the RunwayML voice name as a string +response = speech( + model="runwayml/eleven_multilingual_v2", + input="Hello, world!", + voice="Maya" # LiteLLM automatically formats this for RunwayML +) + +# Try different RunwayML voices +response = speech( + model="runwayml/eleven_multilingual_v2", + input="Step right up, ladies and gentlemen!", + voice="Bernard" # Great for storytelling +) +``` + +**Available RunwayML Voices:** + +Maya, Arjun, Serene, Bernard, Billy, Mark, Clint, Mabel, Chad, Leslie, Eleanor, Elias, Elliot, Grungle, Brodie, Sandra, Kirk, Kylie, Lara, Lisa, Malachi, Marlene, Martin, Miriam, Monster, Paula, Pip, Rusty, Ragnar, Xylar, Maggie, Jack, Katie, Noah, James, Rina, Ella, Mariah, Frank, Claudia, Niki, Vincent, Kendrick, Myrna, Tom, Wanda, Benjamin, Kiana, Rachel + +:::tip +Simply pass the voice name as a string - LiteLLM automatically handles the internal RunwayML API format conversion. +::: + +## Async Usage + +```python showLineNumbers title="Async Text-to-Speech" +from litellm import aspeech +import os +import asyncio + +os.environ["RUNWAYML_API_KEY"] = "your-api-key" + +async def generate_speech(): + response = await aspeech( + model="runwayml/eleven_multilingual_v2", + input="This is an asynchronous text-to-speech request.", + voice="nova" + ) + + with open("output.mp3", "wb") as f: + f.write(response.content) + + print("Audio generated successfully!") + +asyncio.run(generate_speech()) +``` + +## LiteLLM Proxy Usage + +Add RunwayML to your proxy configuration: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: runway-tts + litellm_params: + model: runwayml/eleven_multilingual_v2 + api_key: os.environ/RUNWAYML_API_KEY +``` + +Start the proxy: + +```bash +litellm --config /path/to/config.yaml +``` + +Generate speech through the proxy: + +```bash showLineNumbers title="Proxy Request" +curl --location 'http://localhost:4000/v1/audio/speech' \ +--header 'Content-Type: application/json' \ +--header 'x-litellm-api-key: sk-1234' \ +--data '{ + "model": "runwayml/eleven_multilingual_v2", + "input": "Hello from the LiteLLM proxy!", + "voice": "alloy" +}' +``` + +With RunwayML-specific voice: + +```bash showLineNumbers title="Proxy Request with RunwayML Voice" +curl --location 'http://localhost:4000/v1/audio/speech' \ +--header 'Content-Type: application/json' \ +--header 'x-litellm-api-key: sk-1234' \ +--data '{ + "model": "runwayml/eleven_multilingual_v2", + "input": "Hello with a custom RunwayML voice!", + "voice": "Bernard" +}' +``` + +## Supported Models + +| Model | Description | +|-------|-------------| +| `runwayml/eleven_multilingual_v2` | High-quality multilingual text-to-speech | + +## Cost Tracking + +LiteLLM automatically tracks RunwayML text-to-speech costs: + +```python showLineNumbers title="Cost Tracking" +from litellm import speech, completion_cost + +response = speech( + model="runwayml/eleven_multilingual_v2", + input="Hello, world!", + voice="alloy" +) + +cost = completion_cost(completion_response=response) +print(f"Text-to-speech cost: ${cost}") +``` + +## Supported Features + +| Feature | Supported | +|---------|-----------| +| Text-to-Speech | ✅ | +| Cost Tracking | ✅ | +| Logging | ✅ | +| Fallbacks | ✅ | +| Load Balancing | ✅ | +| 50+ Voice Presets | ✅ | + +## How It Works + +RunwayML uses an asynchronous task-based API pattern. LiteLLM handles the polling and response transformation automatically. + +### Complete Flow Diagram + +```mermaid +sequenceDiagram + participant Client + box rgb(200, 220, 255) LiteLLM AI Gateway + participant LiteLLM + end + participant RunwayML as RunwayML API + participant Storage as Audio Storage + + Client->>LiteLLM: POST /audio/speech (OpenAI format) + Note over LiteLLM: Transform to RunwayML format
Map voice to preset ID + + LiteLLM->>RunwayML: POST v1/text_to_speech + RunwayML-->>LiteLLM: 200 OK + task ID + + Note over LiteLLM: Automatic Polling + loop Every 2 seconds + LiteLLM->>RunwayML: GET v1/tasks/{task_id} + RunwayML-->>LiteLLM: Status: RUNNING + end + + LiteLLM->>RunwayML: GET v1/tasks/{task_id} + RunwayML-->>LiteLLM: Status: SUCCEEDED + audio URL + + LiteLLM->>Storage: GET audio URL + Storage-->>LiteLLM: Audio data (MP3) + + Note over LiteLLM: Return audio content + LiteLLM-->>Client: Audio Response (binary) +``` + diff --git a/docs/my-website/docs/providers/runwayml/videos.md b/docs/my-website/docs/providers/runwayml/videos.md new file mode 100644 index 00000000000..33621509a31 --- /dev/null +++ b/docs/my-website/docs/providers/runwayml/videos.md @@ -0,0 +1,266 @@ +# RunwayML - Video Generation + +LiteLLM supports RunwayML's Gen-4 video generation API, allowing you to generate videos from text prompts and images. + +## Quick Start + +```python showLineNumbers title="Basic Video Generation" +from litellm import video_generation +import os + +os.environ["RUNWAYML_API_KEY"] = "your-api-key" + +# Generate video from text and image +response = video_generation( + model="runwayml/gen4_turbo", + prompt="A high quality demo video of litellm ai gateway", + input_reference="https://media.licdn.com/dms/image/v2/D4D0BAQFqOrIAJEgtLw/company-logo_200_200/company-logo_200_200/0/1714076049190/berri_ai_logo?e=2147483647&v=beta&t=7tG_KRZZ4MPGc7Iin79PcFcrpvf5Hu6rBM4ptHGU1DY", + seconds=5, + size="1280x720" +) + +print(f"Video ID: {response.id}") +print(f"Status: {response.status}") +``` + +## Authentication + +Set your RunwayML API key: + +```python showLineNumbers title="Set API Key" +import os + +os.environ["RUNWAYML_API_KEY"] = "your-api-key" +``` + +## Supported Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `model` | string | Yes | Model to use (e.g., `runwayml/gen4_turbo`) | +| `prompt` | string | Yes | Text description for the video | +| `input_reference` | string/file | Yes | URL or file path to reference image | +| `seconds` | int | No | Video duration (5 or 10 seconds) | +| `size` | string | No | Video dimensions (`1280x720` or `720x1280`). Can also use `ratio` format (`1280:720`) | + +## Complete Workflow + +```python showLineNumbers title="Complete Video Generation Workflow" +from litellm import video_generation, video_status, video_content +import os +import time + +os.environ["RUNWAYML_API_KEY"] = "your-api-key" + +# 1. Generate video +response = video_generation( + model="runwayml/gen4_turbo", + prompt="A high quality demo video of litellm ai gateway", + input_reference="https://media.licdn.com/dms/image/v2/D4D0BAQFqOrIAJEgtLw/company-logo_200_200/company-logo_200_200/0/1714076049190/berri_ai_logo?e=2147483647&v=beta&t=7tG_KRZZ4MPGc7Iin79PcFcrpvf5Hu6rBM4ptHGU1DY", + seconds=5, + size="1280x720" +) + +video_id = response.id +print(f"Video generation started: {video_id}") + +# 2. Check status until completed +while True: + status_response = video_status(video_id=video_id) + print(f"Status: {status_response.status}") + + if status_response.status == "completed": + print("Video generation completed!") + break + elif status_response.status == "failed": + print("Video generation failed") + break + + time.sleep(10) # Wait 10 seconds before checking again + +# 3. Download video content +video_bytes = video_content(video_id=video_id) + +# 4. Save to file +with open("generated_video.mp4", "wb") as f: + f.write(video_bytes) + +print("Video saved successfully!") +``` + +## Async Usage + +```python showLineNumbers title="Async Video Generation" +from litellm import avideo_generation, avideo_status, avideo_content +import os +import asyncio + +os.environ["RUNWAYML_API_KEY"] = "your-api-key" + +async def generate_video(): + # Generate video + response = await avideo_generation( + model="runwayml/gen4_turbo", + prompt="A serene lake with mountains in the background", + input_reference="https://example.com/lake.jpg", + seconds=5, + size="1280x720" + ) + + video_id = response.id + print(f"Video generation started: {video_id}") + + # Poll for completion + while True: + status_response = await avideo_status(video_id=video_id) + print(f"Status: {status_response.status}") + + if status_response.status == "completed": + break + elif status_response.status == "failed": + print("Video generation failed") + return + + await asyncio.sleep(10) + + # Download video + video_bytes = await avideo_content(video_id=video_id) + + # Save to file + with open("generated_video.mp4", "wb") as f: + f.write(video_bytes) + + print("Video saved successfully!") + +asyncio.run(generate_video()) +``` + +## LiteLLM Proxy Usage + +Add RunwayML to your proxy configuration: + +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: gen4-turbo + litellm_params: + model: runwayml/gen4_turbo + api_key: os.environ/RUNWAYML_API_KEY +``` + +Start the proxy: + +```bash +litellm --config /path/to/config.yaml +``` + +Generate videos through the proxy: + +```bash showLineNumbers title="Proxy Request" +curl --location 'http://localhost:4000/v1/videos' \ +--header 'Content-Type: application/json' \ +--header 'x-litellm-api-key: sk-1234' \ +--data '{ + "model": "runwayml/gen4_turbo", + "prompt": "A high quality demo video of litellm ai gateway", + "input_reference": "https://media.licdn.com/dms/image/v2/D4D0BAQFqOrIAJEgtLw/company-logo_200_200/company-logo_200_200/0/1714076049190/berri_ai_logo?e=2147483647&v=beta&t=7tG_KRZZ4MPGc7Iin79PcFcrpvf5Hu6rBM4ptHGU1DY", + "ratio": "1280:720" +}' +``` + +Check video status: + +```bash showLineNumbers title="Check Status" +curl --location 'http://localhost:4000/v1/videos/{video_id}' \ +--header 'x-litellm-api-key: sk-1234' +``` + +Download video content: + +```bash showLineNumbers title="Download Video" +curl --location 'http://localhost:4000/v1/videos/{video_id}/content' \ +--header 'x-litellm-api-key: sk-1234' \ +--output video.mp4 +``` + +## Supported Models + +| Model | Description | Duration | Aspect Ratios | +|-------|-------------|----------|---------------| +| `runwayml/gen4_turbo` | Fast video generation | 5-10s | 1280x720, 720x1280 | + +## Error Handling + +```python showLineNumbers title="Error Handling" +from litellm import video_generation, video_status +import time + +try: + response = video_generation( + model="runwayml/gen4_turbo", + prompt="A scenic mountain view", + input_reference="https://example.com/mountain.jpg", + seconds=5 + ) + + # Poll for completion + max_attempts = 60 # 10 minutes max + attempts = 0 + + while attempts < max_attempts: + status_response = video_status(video_id=response.id) + + if status_response.status == "completed": + print("Video generation completed!") + break + elif status_response.status == "failed": + error = status_response.error or {} + print(f"Video generation failed: {error.get('message', 'Unknown error')}") + break + + time.sleep(10) + attempts += 1 + + if attempts >= max_attempts: + print("Video generation timed out") + +except Exception as e: + print(f"Error: {str(e)}") +``` + +## Cost Tracking + +LiteLLM automatically tracks RunwayML video generation costs: + +```python showLineNumbers title="Cost Tracking" +from litellm import video_generation, completion_cost + +response = video_generation( + model="runwayml/gen4_turbo", + prompt="A high quality demo video of litellm ai gateway", + input_reference="https://media.licdn.com/dms/image/v2/D4D0BAQFqOrIAJEgtLw/company-logo_200_200/company-logo_200_200/0/1714076049190/berri_ai_logo?e=2147483647&v=beta&t=7tG_KRZZ4MPGc7Iin79PcFcrpvf5Hu6rBM4ptHGU1DY", + seconds=5, + size="1280x720" +) + +# Calculate cost +cost = completion_cost(completion_response=response) +print(f"Video generation cost: ${cost}") +``` + +## API Reference + +For complete API details, see the [OpenAI Video Generation API specification](https://platform.openai.com/docs/guides/video-generation) which LiteLLM follows. + +## Supported Features + +| Feature | Supported | +|---------|-----------| +| Video Generation | ✅ | +| Image-to-Video | ✅ | +| Status Checking | ✅ | +| Content Download | ✅ | +| Cost Tracking | ✅ | +| Logging | ✅ | +| Fallbacks | ✅ | +| Load Balancing | ✅ | + diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 3b6562b51ae..4d7e85f3888 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -12,7 +12,7 @@ import TabItem from '@theme/TabItem'; | Provider Route on LiteLLM | `vertex_ai/` | | Link to Provider Doc | [Vertex AI ↗](https://cloud.google.com/vertex-ai) | | Base URL | 1. Regional endpoints
`https://{vertex_location}-aiplatform.googleapis.com/`
2. Global endpoints (limited availability)
`https://aiplatform.googleapis.com/`| -| Supported Operations | [`/chat/completions`](#sample-usage), `/completions`, [`/embeddings`](#embedding-models), [`/audio/speech`](#text-to-speech-apis), [`/fine_tuning`](#fine-tuning-apis), [`/batches`](#batch-apis), [`/files`](#batch-apis), [`/images`](#image-generation-models) | +| Supported Operations | [`/chat/completions`](#sample-usage), `/completions`, [`/embeddings`](#embedding-models), [`/audio/speech`](#text-to-speech-apis), [`/fine_tuning`](#fine-tuning-apis), [`/batches`](#batch-apis), [`/files`](#batch-apis), [`/images`](#image-generation-models), [`/rerank`](#rerank-api) |
@@ -1604,6 +1604,53 @@ litellm.vertex_location = "us-central1 # Your Location | gemini-2.5-flash-preview-09-2025 | `completion('gemini-2.5-flash-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-preview-09-2025', messages)` | | gemini-2.5-flash-lite-preview-09-2025 | `completion('gemini-2.5-flash-lite-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-lite-preview-09-2025', messages)` | +## Private Service Connect (PSC) Endpoints + +LiteLLM supports Vertex AI models deployed to Private Service Connect (PSC) endpoints, allowing you to use custom `api_base` URLs for private deployments. + +### Usage + +```python +from litellm import completion + +# Use PSC endpoint with custom api_base +response = completion( + model="vertex_ai/1234567890", # Numeric endpoint ID + messages=[{"role": "user", "content": "Hello!"}], + api_base="http://10.96.32.8", # Your PSC endpoint + vertex_project="my-project-id", + vertex_location="us-central1" +) +``` + +**Key Features:** +- Supports both numeric endpoint IDs and custom model names +- Works with both completion and embedding endpoints +- Automatically constructs full PSC URL: `{api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}` +- Compatible with streaming requests + +### Configuration + +Add PSC endpoints to your `config.yaml`: + +```yaml +model_list: + - model_name: psc-gemini + litellm_params: + model: vertex_ai/1234567890 # Numeric endpoint ID + api_base: "http://10.96.32.8" # Your PSC endpoint + vertex_project: "my-project-id" + vertex_location: "us-central1" + vertex_credentials: "/path/to/service_account.json" + - model_name: psc-embedding + litellm_params: + model: vertex_ai/text-embedding-004 + api_base: "http://10.96.32.8" # Your PSC endpoint + vertex_project: "my-project-id" + vertex_location: "us-central1" + vertex_credentials: "/path/to/service_account.json" +``` + ## Fine-tuned Models You can call fine-tuned Vertex AI Gemini models through LiteLLM @@ -2042,515 +2089,6 @@ curl http://0.0.0.0:4000/v1/chat/completions \ | code-gecko@latest| `completion('code-gecko@latest', messages)` | -## **Embedding Models** - -#### Usage - Embedding - - - - -```python -import litellm -from litellm import embedding -litellm.vertex_project = "hardy-device-38811" # Your Project ID -litellm.vertex_location = "us-central1" # proj location - -response = embedding( - model="vertex_ai/textembedding-gecko", - input=["good morning from litellm"], -) -print(response) -``` - - - - - -1. Add model to config.yaml -```yaml -model_list: - - model_name: snowflake-arctic-embed-m-long-1731622468876 - litellm_params: - model: vertex_ai/ - vertex_project: "adroit-crow-413218" - vertex_location: "us-central1" - vertex_credentials: adroit-crow-413218-a956eef1a2a8.json - -litellm_settings: - drop_params: True -``` - -2. Start Proxy - -``` -$ litellm --config /path/to/config.yaml -``` - -3. Make Request using OpenAI Python SDK, Langchain Python SDK - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -response = client.embeddings.create( - model="snowflake-arctic-embed-m-long-1731622468876", - input = ["good morning from litellm", "this is another item"], -) - -print(response) -``` - - - - - -#### Supported Embedding Models -All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a0249f630a6792d49dffc2c5d9b7/model_prices_and_context_window.json#L835) are supported - -| Model Name | Function Call | -|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| text-embedding-004 | `embedding(model="vertex_ai/text-embedding-004", input)` | -| text-multilingual-embedding-002 | `embedding(model="vertex_ai/text-multilingual-embedding-002", input)` | -| textembedding-gecko | `embedding(model="vertex_ai/textembedding-gecko", input)` | -| textembedding-gecko-multilingual | `embedding(model="vertex_ai/textembedding-gecko-multilingual", input)` | -| textembedding-gecko-multilingual@001 | `embedding(model="vertex_ai/textembedding-gecko-multilingual@001", input)` | -| textembedding-gecko@001 | `embedding(model="vertex_ai/textembedding-gecko@001", input)` | -| textembedding-gecko@003 | `embedding(model="vertex_ai/textembedding-gecko@003", input)` | -| text-embedding-preview-0409 | `embedding(model="vertex_ai/text-embedding-preview-0409", input)` | -| text-multilingual-embedding-preview-0409 | `embedding(model="vertex_ai/text-multilingual-embedding-preview-0409", input)` | -| Fine-tuned OR Custom Embedding models | `embedding(model="vertex_ai/", input)` | - -### Supported OpenAI (Unified) Params - -| [param](../embedding/supported_embedding.md#input-params-for-litellmembedding) | type | [vertex equivalent](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api) | -|-------|-------------|--------------------| -| `input` | **string or List[string]** | `instances` | -| `dimensions` | **int** | `output_dimensionality` | -| `input_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | `task_type` | - -#### Usage with OpenAI (Unified) Params - - - - - -```python -response = litellm.embedding( - model="vertex_ai/text-embedding-004", - input=["good morning from litellm", "gm"] - input_type = "RETRIEVAL_DOCUMENT", - dimensions=1, -) -``` - - - - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -response = client.embeddings.create( - model="text-embedding-004", - input = ["good morning from litellm", "gm"], - dimensions=1, - extra_body = { - "input_type": "RETRIEVAL_QUERY", - } -) - -print(response) -``` - - - - -### Supported Vertex Specific Params - -| param | type | -|-------|-------------| -| `auto_truncate` | **bool** | -| `task_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | -| `title` | **str** | - -#### Usage with Vertex Specific Params (Use `task_type` and `title`) - -You can pass any vertex specific params to the embedding model. Just pass them to the embedding function like this: - -[Relevant Vertex AI doc with all embedding params](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#request_body) - - - - -```python -response = litellm.embedding( - model="vertex_ai/text-embedding-004", - input=["good morning from litellm", "gm"] - task_type = "RETRIEVAL_DOCUMENT", - title = "test", - dimensions=1, - auto_truncate=True, -) -``` - - - - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -response = client.embeddings.create( - model="text-embedding-004", - input = ["good morning from litellm", "gm"], - dimensions=1, - extra_body = { - "task_type": "RETRIEVAL_QUERY", - "auto_truncate": True, - "title": "test", - } -) - -print(response) -``` - - - -## **Multi-Modal Embeddings** - - -Known Limitations: -- Only supports 1 image / video / image per request -- Only supports GCS or base64 encoded images / videos - -### Usage - - - - -Using GCS Images - -```python -response = await litellm.aembedding( - model="vertex_ai/multimodalembedding@001", - input="gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" # will be sent as a gcs image -) -``` - -Using base 64 encoded images - -```python -response = await litellm.aembedding( - model="vertex_ai/multimodalembedding@001", - input="data:image/jpeg;base64,..." # will be sent as a base64 encoded image -) -``` - - - - -1. Add model to config.yaml -```yaml -model_list: - - model_name: multimodalembedding@001 - litellm_params: - model: vertex_ai/multimodalembedding@001 - vertex_project: "adroit-crow-413218" - vertex_location: "us-central1" - vertex_credentials: adroit-crow-413218-a956eef1a2a8.json - -litellm_settings: - drop_params: True -``` - -2. Start Proxy - -``` -$ litellm --config /path/to/config.yaml -``` - -3. Make Request use OpenAI Python SDK, Langchain Python SDK - - - - - - -Requests with GCS Image / Video URI - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -# # request sent to model set on litellm proxy, `litellm --model` -response = client.embeddings.create( - model="multimodalembedding@001", - input = "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", -) - -print(response) -``` - -Requests with base64 encoded images - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -# # request sent to model set on litellm proxy, `litellm --model` -response = client.embeddings.create( - model="multimodalembedding@001", - input = "data:image/jpeg;base64,...", -) - -print(response) -``` - - - - - -Requests with GCS Image / Video URI -```python -from langchain_openai import OpenAIEmbeddings - -embeddings_models = "multimodalembedding@001" - -embeddings = OpenAIEmbeddings( - model="multimodalembedding@001", - base_url="http://0.0.0.0:4000", - api_key="sk-1234", # type: ignore -) - - -query_result = embeddings.embed_query( - "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" -) -print(query_result) - -``` - -Requests with base64 encoded images - -```python -from langchain_openai import OpenAIEmbeddings - -embeddings_models = "multimodalembedding@001" - -embeddings = OpenAIEmbeddings( - model="multimodalembedding@001", - base_url="http://0.0.0.0:4000", - api_key="sk-1234", # type: ignore -) - - -query_result = embeddings.embed_query( - "data:image/jpeg;base64,..." -) -print(query_result) - -``` - - - - - - - - - -1. Add model to config.yaml -```yaml -default_vertex_config: - vertex_project: "adroit-crow-413218" - vertex_location: "us-central1" - vertex_credentials: adroit-crow-413218-a956eef1a2a8.json -``` - -2. Start Proxy - -``` -$ litellm --config /path/to/config.yaml -``` - -3. Make Request use OpenAI Python SDK - -```python -import vertexai - -from vertexai.vision_models import Image, MultiModalEmbeddingModel, Video -from vertexai.vision_models import VideoSegmentConfig -from google.auth.credentials import Credentials - - -LITELLM_PROXY_API_KEY = "sk-1234" -LITELLM_PROXY_BASE = "http://0.0.0.0:4000/vertex-ai" - -import datetime - -class CredentialsWrapper(Credentials): - def __init__(self, token=None): - super().__init__() - self.token = token - self.expiry = None # or set to a future date if needed - - def refresh(self, request): - pass - - def apply(self, headers, token=None): - headers['Authorization'] = f'Bearer {self.token}' - - @property - def expired(self): - return False # Always consider the token as non-expired - - @property - def valid(self): - return True # Always consider the credentials as valid - -credentials = CredentialsWrapper(token=LITELLM_PROXY_API_KEY) - -vertexai.init( - project="adroit-crow-413218", - location="us-central1", - api_endpoint=LITELLM_PROXY_BASE, - credentials = credentials, - api_transport="rest", - -) - -model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding") -image = Image.load_from_file( - "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" -) - -embeddings = model.get_embeddings( - image=image, - contextual_text="Colosseum", - dimension=1408, -) -print(f"Image Embedding: {embeddings.image_embedding}") -print(f"Text Embedding: {embeddings.text_embedding}") -``` - - - - - -### Text + Image + Video Embeddings - - - - -Text + Image - -```python -response = await litellm.aembedding( - model="vertex_ai/multimodalembedding@001", - input=["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"] # will be sent as a gcs image -) -``` - -Text + Video - -```python -response = await litellm.aembedding( - model="vertex_ai/multimodalembedding@001", - input=["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image -) -``` - -Image + Video - -```python -response = await litellm.aembedding( - model="vertex_ai/multimodalembedding@001", - input=["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image -) -``` - - - - - -1. Add model to config.yaml -```yaml -model_list: - - model_name: multimodalembedding@001 - litellm_params: - model: vertex_ai/multimodalembedding@001 - vertex_project: "adroit-crow-413218" - vertex_location: "us-central1" - vertex_credentials: adroit-crow-413218-a956eef1a2a8.json - -litellm_settings: - drop_params: True -``` - -2. Start Proxy - -``` -$ litellm --config /path/to/config.yaml -``` - -3. Make Request use OpenAI Python SDK, Langchain Python SDK - - -Text + Image - -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -# # request sent to model set on litellm proxy, `litellm --model` -response = client.embeddings.create( - model="multimodalembedding@001", - input = ["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"], -) - -print(response) -``` - -Text + Video -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -# # request sent to model set on litellm proxy, `litellm --model` -response = client.embeddings.create( - model="multimodalembedding@001", - input = ["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"], -) - -print(response) -``` - -Image + Video -```python -import openai - -client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - -# # request sent to model set on litellm proxy, `litellm --model` -response = client.embeddings.create( - model="multimodalembedding@001", - input = ["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"], -) - -print(response) -``` - - - - - ## **Gemini TTS (Text-to-Speech) Audio Output** :::info @@ -2935,7 +2473,7 @@ finetune_settings: ft_job = await client.fine_tuning.jobs.create( model="gemini-1.0-pro-002", # Vertex model you want to fine-tune training_file="gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl", # file_id from create file response - extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm proxy which provider to use + extra_headers={"custom-llm-provider": "vertex_ai"}, # tell litellm proxy which provider to use ) ``` @@ -2946,8 +2484,8 @@ ft_job = await client.fine_tuning.jobs.create( curl http://localhost:4000/v1/fine_tuning/jobs \ -H "Content-Type: application/json" \ -H "Authorization: Bearer sk-1234" \ + -H "custom-llm-provider: vertex_ai" \ -d '{ - "custom_llm_provider": "vertex_ai", "model": "gemini-1.0-pro-002", "training_file": "gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl" }' @@ -2975,9 +2513,7 @@ ft_job = client.fine_tuning.jobs.create( "learning_rate_multiplier": 0.1, # learning_rate_multiplier on Vertex "adapter_size": "ADAPTER_SIZE_ONE" # type: ignore, vertex specific hyperparameter }, - extra_body={ - "custom_llm_provider": "vertex_ai", - }, + extra_headers={"custom-llm-provider": "vertex_ai"}, ) ``` @@ -2988,8 +2524,8 @@ ft_job = client.fine_tuning.jobs.create( curl http://localhost:4000/v1/fine_tuning/jobs \ -H "Content-Type: application/json" \ -H "Authorization: Bearer sk-1234" \ + -H "custom-llm-provider: vertex_ai" \ -d '{ - "custom_llm_provider": "vertex_ai", "model": "gemini-1.0-pro-002", "training_file": "gs://cloud-samples-data/ai-platform/generative_ai/sft_train_data.jsonl", "hyperparameters": { @@ -3114,3 +2650,101 @@ Once that's done, when you deploy the new container in the Google Cloud Run serv s/o @[Darien Kindlund](https://www.linkedin.com/in/kindlund/) for this tutorial + +## **Rerank API** + +Vertex AI supports reranking through the Discovery Engine API, providing semantic ranking capabilities for document retrieval. + +### Setup + +Set your Google Cloud project ID: + +```bash +export VERTEXAI_PROJECT="your-project-id" +``` + +### Usage + +```python +from litellm import rerank + +# Using the latest model (recommended) +response = rerank( + model="vertex_ai/semantic-ranker-default@latest", + query="What is Google Gemini?", + documents=[ + "Gemini is a cutting edge large language model created by Google.", + "The Gemini zodiac symbol often depicts two figures standing side-by-side.", + "Gemini is a constellation that can be seen in the night sky." + ], + top_n=2, + return_documents=True # Set to False for ID-only responses +) + +# Using specific model versions +response_v003 = rerank( + model="vertex_ai/semantic-ranker-default-003", + query="What is Google Gemini?", + documents=documents, + top_n=2 +) + +print(response.results) +``` + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `model` | string | Model name (e.g., `vertex_ai/semantic-ranker-default@latest`) | +| `query` | string | Search query | +| `documents` | list | Documents to rank | +| `top_n` | int | Number of top results to return | +| `return_documents` | bool | Return full content (True) or IDs only (False) | + +### Supported Models + +- `semantic-ranker-default@latest` +- `semantic-ranker-fast@latest` +- `semantic-ranker-default-003` +- `semantic-ranker-default-002` + +For detailed model specifications, see the [Google Cloud ranking API documentation](https://cloud.google.com/generative-ai-app-builder/docs/ranking#rank_or_rerank_a_set_of_records_according_to_a_query). + +### Proxy Usage + +Add to your `config.yaml`: + +```yaml +model_list: + - model_name: semantic-ranker-default@latest + litellm_params: + model: vertex_ai/semantic-ranker-default@latest + vertex_ai_project: "your-project-id" + vertex_ai_location: "us-central1" + vertex_ai_credentials: "path/to/service-account.json" +``` + +Start the proxy: + +```bash +litellm --config /path/to/config.yaml +``` + +Test with curl: + +```bash +curl http://0.0.0.0:4000/rerank \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "semantic-ranker-default@latest", + "query": "What is Google Gemini?", + "documents": [ + "Gemini is a cutting edge large language model created by Google.", + "The Gemini zodiac symbol often depicts two figures standing side-by-side.", + "Gemini is a constellation that can be seen in the night sky." + ], + "top_n": 2 + }' +``` diff --git a/docs/my-website/docs/providers/vertex_ai/videos.md b/docs/my-website/docs/providers/vertex_ai/videos.md new file mode 100644 index 00000000000..4aaf74354b1 --- /dev/null +++ b/docs/my-website/docs/providers/vertex_ai/videos.md @@ -0,0 +1,268 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Vertex AI Video Generation (Veo) + +LiteLLM supports Vertex AI's Veo video generation models using the unified OpenAI video API surface. + +| Property | Details | +|-------|-------| +| Description | Google Cloud Vertex AI Veo video generation models | +| Provider Route on LiteLLM | `vertex_ai/` | +| Supported Models | `veo-2.0-generate-001`, `veo-3.0-generate-preview`, `veo-3.0-fast-generate-preview`, `veo-3.1-generate-preview`, `veo-3.1-fast-generate-preview` | +| Cost Tracking | ✅ Duration-based pricing | +| Logging Support | ✅ Full request/response logging | +| Proxy Server Support | ✅ Full proxy integration with virtual keys | +| Spend Management | ✅ Budget tracking and rate limiting | +| Link to Provider Doc | [Vertex AI Veo Documentation ↗](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/veo-video-generation) | + +## Quick Start + +### Required Environment Setup + +```python +import json +import os + +os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id" +os.environ["VERTEXAI_LOCATION"] = "us-central1" + +# Option 1: Point to a service account file +os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/path/to/service_account.json" + +# Option 2: Store the service account JSON directly +with open("/path/to/service_account.json", "r", encoding="utf-8") as f: + os.environ["VERTEXAI_CREDENTIALS"] = f.read() +``` + +### Basic Usage + +```python +from litellm import video_generation, video_status, video_content +import json +import os +import time + +with open("/path/to/service_account.json", "r", encoding="utf-8") as f: + vertex_credentials = f.read() + +response = video_generation( + model="vertex_ai/veo-3.0-generate-preview", + prompt="A cat playing with a ball of yarn in a sunny garden", + vertex_project="your-gcp-project-id", + vertex_location="us-central1", + vertex_credentials=vertex_credentials, + seconds="8", + size="1280x720", +) + +print(f"Video ID: {response.id}") +print(f"Initial Status: {response.status}") + +# Poll for completion +while True: + status = video_status( + video_id=response.id, + vertex_project="your-gcp-project-id", + vertex_location="us-central1", + vertex_credentials=vertex_credentials, + ) + + print(f"Current Status: {status.status}") + + if status.status == "completed": + break + if status.status == "failed": + raise RuntimeError("Video generation failed") + + time.sleep(10) + +# Download the rendered video +video_bytes = video_content( + video_id=response.id, + vertex_project="your-gcp-project-id", + vertex_location="us-central1", + vertex_credentials=vertex_credentials, +) + +with open("generated_video.mp4", "wb") as f: + f.write(video_bytes) +``` + +## Supported Models + +| Model Name | Description | Max Duration | Status | +|------------|-------------|--------------|--------| +| veo-2.0-generate-001 | Veo 2.0 video generation | 5 seconds | GA | +| veo-3.0-generate-preview | Veo 3.0 high quality | 8 seconds | Preview | +| veo-3.0-fast-generate-preview | Veo 3.0 fast generation | 8 seconds | Preview | +| veo-3.1-generate-preview | Veo 3.1 high quality | 10 seconds | Preview | +| veo-3.1-fast-generate-preview | Veo 3.1 fast | 10 seconds | Preview | + +## Video Generation Parameters + +LiteLLM converts OpenAI-style parameters to Veo's API shape automatically: + +| OpenAI Parameter | Vertex AI Parameter | Description | Example | +|------------------|---------------------|-------------|---------| +| `prompt` | `instances[].prompt` | Text description of the video | "A cat playing" | +| `size` | `parameters.aspectRatio` | Converted to `16:9` or `9:16` | "1280x720" → `16:9` | +| `seconds` | `parameters.durationSeconds` | Clip length in seconds | "8" → `8` | +| `input_reference` | `instances[].image` | Reference image for animation | `open("image.jpg", "rb")` | +| Provider-specific params | `extra_body` | Forwarded to Vertex API | `{"negativePrompt": "blurry"}` | + +### Size to Aspect Ratio Mapping + +- `1280x720`, `1920x1080` → `16:9` +- `720x1280`, `1080x1920` → `9:16` +- Unknown sizes default to `16:9` + +## Async Usage + +```python +from litellm import avideo_generation, avideo_status, avideo_content +import asyncio +import json + +with open("/path/to/service_account.json", "r", encoding="utf-8") as f: + vertex_credentials = f.read() + + +async def workflow(): + response = await avideo_generation( + model="vertex_ai/veo-3.1-generate-preview", + prompt="Slow motion water droplets splashing into a pool", + seconds="10", + vertex_project="your-gcp-project-id", + vertex_location="us-central1", + vertex_credentials=vertex_credentials, + ) + + while True: + status = await avideo_status( + video_id=response.id, + vertex_project="your-gcp-project-id", + vertex_location="us-central1", + vertex_credentials=vertex_credentials, + ) + + if status.status == "completed": + break + if status.status == "failed": + raise RuntimeError("Video generation failed") + + await asyncio.sleep(10) + + video_bytes = await avideo_content( + video_id=response.id, + vertex_project="your-gcp-project-id", + vertex_location="us-central1", + vertex_credentials=vertex_credentials, + ) + + with open("veo_water.mp4", "wb") as f: + f.write(video_bytes) + +asyncio.run(workflow()) +``` + +## LiteLLM Proxy Usage + +Add Veo models to your `config.yaml`: + +```yaml +model_list: + - model_name: veo-3 + litellm_params: + model: vertex_ai/veo-3.0-generate-preview + vertex_project: os.environ/VERTEXAI_PROJECT + vertex_location: os.environ/VERTEXAI_LOCATION + vertex_credentials: os.environ/VERTEXAI_CREDENTIALS +``` + +Start the proxy and make requests: + + + + +```bash +# Step 1: Generate video +curl --location 'http://0.0.0.0:4000/videos' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer sk-1234' \ +--data '{ + "model": "veo-3", + "prompt": "Aerial shot over a futuristic city at sunrise", + "seconds": "8" +}' + +# Step 2: Poll status +curl --location 'http://localhost:4000/v1/videos/{video_id}' \ +--header 'x-litellm-api-key: sk-1234' + +# Step 3: Download video +curl --location 'http://localhost:4000/v1/videos/{video_id}/content' \ +--header 'x-litellm-api-key: sk-1234' \ +--output video.mp4 +``` + + + + +```python +import litellm + +litellm.api_base = "http://0.0.0.0:4000" +litellm.api_key = "sk-1234" + +response = litellm.video_generation( + model="veo-3", + prompt="Aerial shot over a futuristic city at sunrise", +) + +status = litellm.video_status(video_id=response.id) +while status.status not in ["completed", "failed"]: + status = litellm.video_status(video_id=response.id) + +if status.status == "completed": + content = litellm.video_content(video_id=response.id) + with open("veo_city.mp4", "wb") as f: + f.write(content) +``` + + + + +## Cost Tracking + +LiteLLM records the duration returned by Veo so you can apply duration-based pricing. + +```python +with open("/path/to/service_account.json", "r", encoding="utf-8") as f: + vertex_credentials = f.read() + +response = video_generation( + model="vertex_ai/veo-2.0-generate-001", + prompt="Flowers blooming in fast forward", + seconds="5", + vertex_project="your-gcp-project-id", + vertex_location="us-central1", + vertex_credentials=vertex_credentials, +) + +print(response.usage) # {"duration_seconds": 5.0} +``` + +## Troubleshooting + +- **`vertex_project is required`**: set `VERTEXAI_PROJECT` env var or pass `vertex_project` in the request. +- **`Permission denied`**: ensure the service account has the `Vertex AI User` role and the correct region enabled. +- **Video stuck in `processing`**: Veo operations are long-running. Continue polling every 10–15 seconds up to ~10 minutes. + +## See Also + +- [OpenAI Video Generation](../openai/videos.md) +- [Azure Video Generation](../azure/videos.md) +- [Gemini Video Generation](../gemini/videos.md) +- [Video Generation API Reference](/docs/videos) + diff --git a/docs/my-website/docs/providers/vertex_batch.md b/docs/my-website/docs/providers/vertex_batch.md index 046c60f2ebd..01052ba32e3 100644 --- a/docs/my-website/docs/providers/vertex_batch.md +++ b/docs/my-website/docs/providers/vertex_batch.md @@ -50,7 +50,7 @@ oai_client = OpenAI( file_obj = oai_client.files.create( file=open("batch_requests.jsonl", "rb"), purpose="batch", - extra_body={"custom_llm_provider": "vertex_ai"} + extra_headers={"custom-llm-provider": "vertex_ai"} ) print(f"File uploaded with ID: {file_obj.id}") @@ -63,9 +63,9 @@ print(f"File uploaded with ID: {file_obj.id}") curl --request POST \ --url http://localhost:4000/v1/files \ --header 'Content-Type: multipart/form-data' \ + --header 'custom-llm-provider: vertex_ai' \ --form purpose=batch \ - --form file=@batch_requests.jsonl \ - --form custom_llm_provider=vertex_ai + --form file=@batch_requests.jsonl ``` @@ -100,7 +100,7 @@ create_batch_response = oai_client.batches.create( completion_window="24h", endpoint="/v1/chat/completions", input_file_id=batch_input_file_id, # e.g. "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd" - extra_body={"custom_llm_provider": "vertex_ai"} + extra_headers={"custom-llm-provider": "vertex_ai"} ) print(f"Batch created with ID: {create_batch_response.id}") @@ -113,11 +113,11 @@ print(f"Batch created with ID: {create_batch_response.id}") curl --request POST \ --url http://localhost:4000/v1/batches \ --header 'Content-Type: application/json' \ + --header 'custom-llm-provider: vertex_ai' \ --data '{ "input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd", "endpoint": "/v1/chat/completions", - "completion_window": "24h", - "custom_llm_provider": "vertex_ai" + "completion_window": "24h" }' ``` @@ -162,7 +162,7 @@ Check the status of your batch job. The batch will progress through states: `val ```python showLineNumbers title="retrieve_batch.py" retrieved_batch = oai_client.batches.retrieve( batch_id=create_batch_response.id, # Created batch id, e.g. 7814463557919047680 - extra_body={"custom_llm_provider": "vertex_ai"} + extra_headers={"custom-llm-provider": "vertex_ai"} ) print(f"Batch status: {retrieved_batch.status}") @@ -230,7 +230,7 @@ encoded_file_id = urllib.parse.quote_plus(output_file_id) # Get file content file_content = oai_client.files.content( file_id=encoded_file_id, - extra_body={"custom_llm_provider": "vertex_ai"} + extra_headers={"custom-llm-provider": "vertex_ai"} ) # Process the results diff --git a/docs/my-website/docs/providers/vertex_embedding.md b/docs/my-website/docs/providers/vertex_embedding.md new file mode 100644 index 00000000000..5656ade337b --- /dev/null +++ b/docs/my-website/docs/providers/vertex_embedding.md @@ -0,0 +1,587 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Vertex AI Embedding + +## Usage - Embedding + + + + +```python +import litellm +from litellm import embedding +litellm.vertex_project = "hardy-device-38811" # Your Project ID +litellm.vertex_location = "us-central1" # proj location + +response = embedding( + model="vertex_ai/textembedding-gecko", + input=["good morning from litellm"], +) +print(response) +``` + + + + + +1. Add model to config.yaml +```yaml +model_list: + - model_name: snowflake-arctic-embed-m-long-1731622468876 + litellm_params: + model: vertex_ai/ + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: adroit-crow-413218-a956eef1a2a8.json + +litellm_settings: + drop_params: True +``` + +2. Start Proxy + +``` +$ litellm --config /path/to/config.yaml +``` + +3. Make Request using OpenAI Python SDK, Langchain Python SDK + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.embeddings.create( + model="snowflake-arctic-embed-m-long-1731622468876", + input = ["good morning from litellm", "this is another item"], +) + +print(response) +``` + + + + + +#### Supported Embedding Models +All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a0249f630a6792d49dffc2c5d9b7/model_prices_and_context_window.json#L835) are supported + +| Model Name | Function Call | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| text-embedding-004 | `embedding(model="vertex_ai/text-embedding-004", input)` | +| text-multilingual-embedding-002 | `embedding(model="vertex_ai/text-multilingual-embedding-002", input)` | +| textembedding-gecko | `embedding(model="vertex_ai/textembedding-gecko", input)` | +| textembedding-gecko-multilingual | `embedding(model="vertex_ai/textembedding-gecko-multilingual", input)` | +| textembedding-gecko-multilingual@001 | `embedding(model="vertex_ai/textembedding-gecko-multilingual@001", input)` | +| textembedding-gecko@001 | `embedding(model="vertex_ai/textembedding-gecko@001", input)` | +| textembedding-gecko@003 | `embedding(model="vertex_ai/textembedding-gecko@003", input)` | +| text-embedding-preview-0409 | `embedding(model="vertex_ai/text-embedding-preview-0409", input)` | +| text-multilingual-embedding-preview-0409 | `embedding(model="vertex_ai/text-multilingual-embedding-preview-0409", input)` | +| Fine-tuned OR Custom Embedding models | `embedding(model="vertex_ai/", input)` | + +### Supported OpenAI (Unified) Params + +| [param](../embedding/supported_embedding.md#input-params-for-litellmembedding) | type | [vertex equivalent](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api) | +|-------|-------------|--------------------| +| `input` | **string or List[string]** | `instances` | +| `dimensions` | **int** | `output_dimensionality` | +| `input_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | `task_type` | + +#### Usage with OpenAI (Unified) Params + + + + + +```python +response = litellm.embedding( + model="vertex_ai/text-embedding-004", + input=["good morning from litellm", "gm"] + input_type = "RETRIEVAL_DOCUMENT", + dimensions=1, +) +``` + + + + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.embeddings.create( + model="text-embedding-004", + input = ["good morning from litellm", "gm"], + dimensions=1, + extra_body = { + "input_type": "RETRIEVAL_QUERY", + } +) + +print(response) +``` + + + + +### Supported Vertex Specific Params + +| param | type | +|-------|-------------| +| `auto_truncate` | **bool** | +| `task_type` | **Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]** | +| `title` | **str** | + +#### Usage with Vertex Specific Params (Use `task_type` and `title`) + +You can pass any vertex specific params to the embedding model. Just pass them to the embedding function like this: + +[Relevant Vertex AI doc with all embedding params](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#request_body) + + + + +```python +response = litellm.embedding( + model="vertex_ai/text-embedding-004", + input=["good morning from litellm", "gm"] + task_type = "RETRIEVAL_DOCUMENT", + title = "test", + dimensions=1, + auto_truncate=True, +) +``` + + + + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.embeddings.create( + model="text-embedding-004", + input = ["good morning from litellm", "gm"], + dimensions=1, + extra_body = { + "task_type": "RETRIEVAL_QUERY", + "auto_truncate": True, + "title": "test", + } +) + +print(response) +``` + + + +## **BGE Embeddings** + +Use BGE (Baidu General Embedding) models deployed on Vertex AI. + +### Usage + + + + +```python showLineNumbers title="Using BGE on Vertex AI" +import litellm + +response = litellm.embedding( + model="vertex_ai/bge/", + input=["Hello", "World"], + vertex_project="your-project-id", + vertex_location="your-location" +) + +print(response) +``` + + + + + +1. Add model to config.yaml +```yaml showLineNumbers title="config.yaml" +model_list: + - model_name: bge-embedding + litellm_params: + model: vertex_ai/bge/ + vertex_project: "your-project-id" + vertex_location: "us-central1" + vertex_credentials: your-credentials.json + +litellm_settings: + drop_params: True +``` + +2. Start Proxy + +```bash +$ litellm --config /path/to/config.yaml +``` + +3. Make Request using OpenAI Python SDK + +```python showLineNumbers title="Making requests to BGE" +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +response = client.embeddings.create( + model="bge-embedding", + input=["good morning from litellm", "this is another item"] +) + +print(response) +``` + +Using a Private Service Connect (PSC) endpoint + +```yaml showLineNumbers title="config.yaml (PSC)" +model_list: + - model_name: bge-small-en-v1.5 + litellm_params: + model: vertex_ai/bge/1234567890 + api_base: http://10.96.32.8 # Your PSC IP + vertex_project: my-project-id #optional + vertex_location: us-central1 #optional +``` + + + + +## **Multi-Modal Embeddings** + + +Known Limitations: +- Only supports 1 image / video / image per request +- Only supports GCS or base64 encoded images / videos + +### Usage + + + + +Using GCS Images + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input="gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" # will be sent as a gcs image +) +``` + +Using base 64 encoded images + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input="data:image/jpeg;base64,..." # will be sent as a base64 encoded image +) +``` + + + + +1. Add model to config.yaml +```yaml +model_list: + - model_name: multimodalembedding@001 + litellm_params: + model: vertex_ai/multimodalembedding@001 + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: adroit-crow-413218-a956eef1a2a8.json + +litellm_settings: + drop_params: True +``` + +2. Start Proxy + +``` +$ litellm --config /path/to/config.yaml +``` + +3. Make Request use OpenAI Python SDK, Langchain Python SDK + + + + + + +Requests with GCS Image / Video URI + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", +) + +print(response) +``` + +Requests with base64 encoded images + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = "data:image/jpeg;base64,...", +) + +print(response) +``` + + + + + +Requests with GCS Image / Video URI +```python +from langchain_openai import OpenAIEmbeddings + +embeddings_models = "multimodalembedding@001" + +embeddings = OpenAIEmbeddings( + model="multimodalembedding@001", + base_url="http://0.0.0.0:4000", + api_key="sk-1234", # type: ignore +) + + +query_result = embeddings.embed_query( + "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" +) +print(query_result) + +``` + +Requests with base64 encoded images + +```python +from langchain_openai import OpenAIEmbeddings + +embeddings_models = "multimodalembedding@001" + +embeddings = OpenAIEmbeddings( + model="multimodalembedding@001", + base_url="http://0.0.0.0:4000", + api_key="sk-1234", # type: ignore +) + + +query_result = embeddings.embed_query( + "data:image/jpeg;base64,..." +) +print(query_result) + +``` + + + + + + + + + +1. Add model to config.yaml +```yaml +default_vertex_config: + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: adroit-crow-413218-a956eef1a2a8.json +``` + +2. Start Proxy + +``` +$ litellm --config /path/to/config.yaml +``` + +3. Make Request use OpenAI Python SDK + +```python +import vertexai + +from vertexai.vision_models import Image, MultiModalEmbeddingModel, Video +from vertexai.vision_models import VideoSegmentConfig +from google.auth.credentials import Credentials + + +LITELLM_PROXY_API_KEY = "sk-1234" +LITELLM_PROXY_BASE = "http://0.0.0.0:4000/vertex-ai" + +import datetime + +class CredentialsWrapper(Credentials): + def __init__(self, token=None): + super().__init__() + self.token = token + self.expiry = None # or set to a future date if needed + + def refresh(self, request): + pass + + def apply(self, headers, token=None): + headers['Authorization'] = f'Bearer {self.token}' + + @property + def expired(self): + return False # Always consider the token as non-expired + + @property + def valid(self): + return True # Always consider the credentials as valid + +credentials = CredentialsWrapper(token=LITELLM_PROXY_API_KEY) + +vertexai.init( + project="adroit-crow-413218", + location="us-central1", + api_endpoint=LITELLM_PROXY_BASE, + credentials = credentials, + api_transport="rest", + +) + +model = MultiModalEmbeddingModel.from_pretrained("multimodalembedding") +image = Image.load_from_file( + "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png" +) + +embeddings = model.get_embeddings( + image=image, + contextual_text="Colosseum", + dimension=1408, +) +print(f"Image Embedding: {embeddings.image_embedding}") +print(f"Text Embedding: {embeddings.text_embedding}") +``` + + + + + +### Text + Image + Video Embeddings + + + + +Text + Image + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input=["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"] # will be sent as a gcs image +) +``` + +Text + Video + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input=["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image +) +``` + +Image + Video + +```python +response = await litellm.aembedding( + model="vertex_ai/multimodalembedding@001", + input=["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"] # will be sent as a gcs image +) +``` + + + + + +1. Add model to config.yaml +```yaml +model_list: + - model_name: multimodalembedding@001 + litellm_params: + model: vertex_ai/multimodalembedding@001 + vertex_project: "adroit-crow-413218" + vertex_location: "us-central1" + vertex_credentials: adroit-crow-413218-a956eef1a2a8.json + +litellm_settings: + drop_params: True +``` + +2. Start Proxy + +``` +$ litellm --config /path/to/config.yaml +``` + +3. Make Request use OpenAI Python SDK, Langchain Python SDK + + +Text + Image + +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = ["hey", "gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png"], +) + +print(response) +``` + +Text + Video +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = ["hey", "gs://my-bucket/embeddings/supermarket-video.mp4"], +) + +print(response) +``` + +Image + Video +```python +import openai + +client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + +# # request sent to model set on litellm proxy, `litellm --model` +response = client.embeddings.create( + model="multimodalembedding@001", + input = ["gs://cloud-samples-data/vertex-ai/llm/prompts/landmark1.png", "gs://my-bucket/embeddings/supermarket-video.mp4"], +) + +print(response) +``` + + + \ No newline at end of file diff --git a/docs/my-website/docs/providers/vertex_ocr.md b/docs/my-website/docs/providers/vertex_ocr.md new file mode 100644 index 00000000000..4e3d4b0a063 --- /dev/null +++ b/docs/my-website/docs/providers/vertex_ocr.md @@ -0,0 +1,237 @@ +# Vertex AI OCR + +## Overview + +| Property | Details | +|-------|-------| +| Description | Vertex AI OCR provides document intelligence capabilities powered by Mistral, enabling text extraction from PDFs and images | +| Provider Route on LiteLLM | `vertex_ai/` | +| Supported Operations | `/ocr` | +| Link to Provider Doc | [Vertex AI ↗](https://cloud.google.com/vertex-ai) + +Extract text from documents and images using Vertex AI's OCR models, powered by Mistral. + +## Quick Start + +### **LiteLLM SDK** + +```python showLineNumbers title="SDK Usage" +import litellm +import os + +# Set environment variables +os.environ["VERTEXAI_PROJECT"] = "your-project-id" +os.environ["VERTEXAI_LOCATION"] = "us-central1" + +# OCR with PDF URL +response = litellm.ocr( + model="vertex_ai/mistral-ocr-2505", + document={ + "type": "document_url", + "document_url": "https://example.com/document.pdf" + } +) + +# Access extracted text +for page in response.pages: + print(page.text) +``` + +### **LiteLLM PROXY** + +```yaml showLineNumbers title="proxy_config.yaml" +model_list: + - model_name: vertex-ocr + litellm_params: + model: vertex_ai/mistral-ocr-2505 + vertex_project: os.environ/VERTEXAI_PROJECT + vertex_location: os.environ/VERTEXAI_LOCATION + vertex_credentials: path/to/service-account.json # Optional + model_info: + mode: ocr +``` + +**Start Proxy** +```bash +litellm --config proxy_config.yaml +``` + +**Call OCR via Proxy** +```bash showLineNumbers title="cURL Request" +curl -X POST http://localhost:4000/ocr \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer your-api-key" \ + -d '{ + "model": "vertex-ocr", + "document": { + "type": "document_url", + "document_url": "https://arxiv.org/pdf/2201.04234" + } + }' +``` + +## Authentication + +Vertex AI OCR supports multiple authentication methods: + +### Service Account JSON + +```python showLineNumbers title="Service Account Auth" +response = litellm.ocr( + model="vertex_ai/mistral-ocr-2505", + document={"type": "document_url", "document_url": "https://..."}, + vertex_project="your-project-id", + vertex_location="us-central1", + vertex_credentials="path/to/service-account.json" +) +``` + +### Application Default Credentials + +```python showLineNumbers title="Default Credentials" +# Relies on GOOGLE_APPLICATION_CREDENTIALS environment variable +response = litellm.ocr( + model="vertex_ai/mistral-ocr-2505", + document={"type": "document_url", "document_url": "https://..."}, + vertex_project="your-project-id", + vertex_location="us-central1" +) +``` + +## Document Types + +Vertex AI OCR supports both PDFs and images. + +### PDF Documents + +```python showLineNumbers title="PDF OCR" +response = litellm.ocr( + model="vertex_ai/mistral-ocr-2505", + document={ + "type": "document_url", + "document_url": "https://example.com/document.pdf" + }, + vertex_project="your-project-id", + vertex_location="us-central1" +) +``` + +### Image Documents + +```python showLineNumbers title="Image OCR" +response = litellm.ocr( + model="vertex_ai/mistral-ocr-2505", + document={ + "type": "image_url", + "image_url": "https://example.com/image.png" + }, + vertex_project="your-project-id", + vertex_location="us-central1" +) +``` + +### Base64 Encoded Documents + +```python showLineNumbers title="Base64 PDF" +import base64 + +# Read and encode PDF +with open("document.pdf", "rb") as f: + pdf_base64 = base64.b64encode(f.read()).decode() + +response = litellm.ocr( + model="vertex_ai/mistral-ocr-2505", + document={ + "type": "document_url", + "document_url": f"data:application/pdf;base64,{pdf_base64}" + }, + vertex_project="your-project-id", + vertex_location="us-central1" +) +``` + +## Supported Parameters + +```python showLineNumbers title="All Parameters" +response = litellm.ocr( + model="vertex_ai/mistral-ocr-2505", + document={ # Required: Document to process + "type": "document_url", + "document_url": "https://..." + }, + vertex_project="your-project-id", # Required: GCP project ID + vertex_location="us-central1", # Optional: Defaults to us-central1 + vertex_credentials="path/to/key.json", # Optional: Service account key + include_image_base64=True, # Optional: Include base64 images + pages=[0, 1, 2], # Optional: Specific pages to process + image_limit=10 # Optional: Limit number of images +) +``` + +## Response Format + +```python showLineNumbers title="Response Structure" +# Response has the following structure +response.pages # List of pages with extracted text +response.model # Model used +response.object # "ocr" +response.usage_info # Token usage information + +# Access page content +for page in response.pages: + print(f"Page {page.page_number}:") + print(page.text) +``` + +## Async Support + +```python showLineNumbers title="Async Usage" +import litellm + +response = await litellm.aocr( + model="vertex_ai/mistral-ocr-2505", + document={ + "type": "document_url", + "document_url": "https://example.com/document.pdf" + }, + vertex_project="your-project-id", + vertex_location="us-central1" +) +``` + +## Cost Tracking + +LiteLLM automatically tracks costs for Vertex AI OCR: + +- **Cost per page**: $0.0005 (based on $1.50 per 1,000 pages) + +```python showLineNumbers title="View Cost" +response = litellm.ocr( + model="vertex_ai/mistral-ocr-2505", + document={"type": "document_url", "document_url": "https://..."}, + vertex_project="your-project-id" +) + +# Access cost information +print(f"Cost: ${response._hidden_params.get('response_cost', 0)}") +``` + +## Important Notes + +:::info URL Conversion +Vertex AI OCR endpoints don't have internet access. LiteLLM automatically converts public URLs to base64 data URIs before sending requests to Vertex AI. +::: + +:::tip Regional Availability +Mistral OCR is available in multiple regions. Specify `vertex_location` to use a region closer to your data: +- `us-central1` (default) +- `europe-west1` +- `asia-southeast1` +::: + +## Supported Models + +- `mistral-ocr-2505` - Latest Mistral OCR model on Vertex AI + +Use the Vertex AI provider prefix: `vertex_ai/` + diff --git a/docs/my-website/docs/proxy/access_control.md b/docs/my-website/docs/proxy/access_control.md index 4ca3eb119d6..678032be9a2 100644 --- a/docs/my-website/docs/proxy/access_control.md +++ b/docs/my-website/docs/proxy/access_control.md @@ -1,25 +1,342 @@ +import Image from '@theme/IdealImage'; + # Role-based Access Controls (RBAC) Role-based access control (RBAC) is based on Organizations, Teams and Internal User Roles + + + - `Organizations` are the top-level entities that contain Teams. - `Team` - A Team is a collection of multiple `Internal Users` - `Internal Users` - users that can create keys, make LLM API calls, view usage on LiteLLM. Users can be on multiple teams. -- `Roles` define the permissions of an `Internal User` -- `Virtual Keys` - Keys are used for authentication to the LiteLLM API. Keys are tied to a `Internal User` and `Team` +- `Virtual Keys` - Keys are used for authentication to the LiteLLM API. Each key can optionally be associated with a `user_id`, a `team_id`, or both: + - **User-only key**: Has a `user_id` but no `team_id`. Tracked individually, deleted when the user is deleted. + - **Team key (Service Account)**: Has a `team_id` but no `user_id`. Shared by the team, not deleted when users are removed. [Learn more about service account keys](https://docs.litellm.ai/docs/proxy/virtual_keys#service-account-keys). + - **User + Team key**: Has both `user_id` and `team_id`. Belongs to a specific user within a team context. -## Roles +### When to Use Each Key Type -| Role Type | Role Name | Permissions | -|-----------|-----------|-------------| -| **Admin** | `proxy_admin` | Admin over the platform | -| | `proxy_admin_viewer` | Can login, view all keys, view all spend. **Cannot** create keys/delete keys/add new users | -| **Organization** | `org_admin` | Admin over the organization. Can create teams and users within their organization | -| **Internal User** | `internal_user` | Can login, view/create/delete their own keys, view their spend. **Cannot** add new users | -| | `internal_user_viewer` | Can login, view their own keys, view their own spend. **Cannot** create/delete keys, add new users | +| Key Type | Use Case | Spend Tracking | Lifecycle | +|----------|----------|----------------|-----------| +| **User-only** | Personal API keys for individual developers | Tracked to the user | Deleted when user is deleted | +| **Team (Service Account)** | Production apps, CI/CD pipelines, shared services | Tracked to the team only | Persists even when team members leave | +| **User + Team** | User working within a team context | Tracked to both user and team | Deleted when user is deleted | + +**Example scenarios:** +- Use **user-only keys** for developers testing locally +- Use **team service account keys** for your production application that shouldn't break when employees leave +- Use **user + team keys** when you want individual accountability within a team budget + +--- + +## User Roles + +LiteLLM has two types of roles: + +1. **Global Proxy Roles** - Platform-wide roles that apply across all organizations and teams +2. **Organization/Team Specific Roles** - Roles scoped to specific organizations or teams (**Premium Feature**) + +### Global Proxy Roles + +| Role Name | Permissions | +|-----------|-------------| +| `proxy_admin` | Admin over the entire platform. Full control over all organizations, teams, and users | +| `proxy_admin_viewer` | Can login, view all keys, view all spend across the platform. **Cannot** create keys/delete keys/add new users | +| `internal_user` | Can login, view/create (when allowed by team-specific permissions)/delete their own keys, view their spend. **Cannot** add new users | +| `internal_user_viewer` | ⚠️ **DEPRECATED** - Use team/org specific roles instead. Can login, view their own keys, view their own spend. **Cannot** create/delete keys, add new users | + +### Organization/Team Specific Roles + +| Role Name | Permissions | +|-----------|-------------| +| `org_admin` | Admin over a specific organization. Can create teams and users within their organization ✨ **Premium Feature** | +| `team_admin` | Admin over a specific team. Can manage team members, update team settings, and create keys for their team. ✨ **Premium Feature** | + +## What Can Each Role Do? + +Here's what each role can actually do. Think of it like levels of access. + +--- + +## Global Proxy Roles + +These roles apply across the entire LiteLLM platform, regardless of organization or team boundaries. + +### Proxy Admin - Full Access + +The proxy admin controls everything. They're like the owner of the whole platform. + +**What they can do:** +- Create and manage all organizations +- Create and manage all teams (across all organizations) +- Create and manage all users +- View all spend and usage across the platform +- Create and delete keys for anyone +- Update team budgets, rate limits, and models +- Manage team members and assign roles + +**Who should be a proxy admin:** Only the people running the LiteLLM instance. + +--- + +### Proxy Admin Viewer - Platform-Wide Read Access + +The proxy admin viewer can see everything across the platform but cannot make changes. + +**What they can do:** +- View all organizations, teams, and users +- View all spend and usage across the platform +- View all API keys +- Login to the admin dashboard + +**What they cannot do:** +- Create or delete keys +- Add or remove users +- Modify budgets, rate limits, or settings +- Make any changes to the platform + +**Who should be a proxy admin viewer:** Finance teams, auditors, or stakeholders who need platform-wide visibility without modification rights. + +--- + +### Internal User + +An internal user can create API keys (when allowed by team-specific permissions) and make calls. They see their own stuff only. They can become a team admin or org admin if they are assigned the respective roles. + +**What they can do:** +- Create API keys for themselves +- Delete their own API keys +- View their own spend and usage +- Make API calls using their keys + + +**Who should be an internal user:** Anyone who needs UI access for team/org specific operations **OR** for developers you plan to give multiple keys to. + +--- + +### Internal User Viewer - Read-Only Access + +:::warning DEPRECATED +This role is deprecated in favor of team/org specific roles. Use `org_admin` or `team_admin` roles for better granular control over user permissions within organizations and teams. +::: + +An internal user viewer can view their own information but cannot create or delete keys. + +**What they can do:** +- View their own API keys +- View their own spend and usage +- Login to see their dashboard + +**What they cannot do:** +- Create or delete API keys +- Make changes to any settings +- Create teams or add users +- View other people's information + +**Who should be an internal user viewer (deprecated):** Consider using team/org specific roles instead for better access control. + +--- + +## Organization/Team Specific Roles + +:::info +Organization/Team specific roles are premium features. You need to be a LiteLLM Enterprise user to use them. [Get a 7 day trial here](https://www.litellm.ai/#trial). +::: + +These roles are scoped to specific organizations or teams. Users with these roles can only manage resources within their assigned organization or team. + +### Org Admin - Organization Level Access + +An org admin manages one or more organizations. They can create teams within their organization but can't touch other organizations. + +**What they can do:** +- Create teams within their organization +- Add users to teams in their organization +- View spend for their organization +- Create keys for users in their organization + +**What they cannot do:** +- Create or manage other organizations +- Modify org budgets / rate limits +- Modify org allowed models (e.g. adding a proxy-level model to the org) + +**Who should be an org admin:** Department leads or managers who need to manage multiple teams. + +--- + +### Team Admin - Team Level Access + +✨ **This is a Premium Feature** + +A team admin manages a specific team. They're like a team lead who can add people, update settings, but only for their team. + +**What they can do:** +- Add or remove team members from their team +- Update team members' budgets and rate limits within the team +- Change team settings (budget, rate limits, models) +- Create and delete keys for team members +- Onboard a [team-BYOK](./team_model_add) model to LiteLLM (e.g. onboarding a team's finetuned model) +- Configure [team member permissions](#team-member-permissions) to control what regular team members can do + +**What they cannot do:** +- Create new teams +- Modify team's budget / rate limits +- Add/remove global proxy models to their team + + +**Who should be a team admin:** Team leads who need to manage their team's API access without bothering IT. + +:::info How to create a team admin + +You need to be a LiteLLM Enterprise user to assign team admins. [Get a 7 day trial here](https://www.litellm.ai/#trial). + +```shell +curl -X POST 'http://0.0.0.0:4000/team/member_add' \ + -H 'Authorization: Bearer sk-1234' \ + -H 'Content-Type: application/json' \ + -d '{"team_id": "team-123", "member": {"role": "admin", "user_id": "user@company.com"}}' +``` + +::: + +--- + +## Team Member Permissions + +✨ **This is a Premium Feature** + +Team member permissions allow you to control what regular team members (with role=`user`) can do with API keys in their team. By default, team members can only view key information, but you can grant them additional permissions to create, update, or delete keys. + +### How It Works + +- **Applies to**: Team members with role=`user` (not team admins or org admins) +- **Scope**: Permissions only apply to keys belonging to their team +- **Configuration**: Set at the team level using `team_member_permissions` +- **Override**: Team admins and org admins always have full permissions regardless of these settings + +### Available Permissions + +| Permission | Method | Description | +|-----------|--------|-------------| +| `/key/info` | GET | View information about virtual keys in the team | +| `/key/health` | GET | Check health status of virtual keys in the team | +| `/key/list` | GET | List all virtual keys belonging to the team | +| `/key/generate` | POST | Create new virtual keys for the team | +| `/key/service-account/generate` | POST | Create service account keys (not tied to a specific user) for the team | +| `/key/update` | POST | Modify existing virtual keys in the team | +| `/key/delete` | POST | Delete virtual keys belonging to the team | +| `/key/regenerate` | POST | Regenerate virtual keys in the team | +| `/key/block` | POST | Block virtual keys in the team | +| `/key/unblock` | POST | Unblock virtual keys in the team | + +### Default Permissions + +By default, team members can only: +- `/key/info` - View key information +- `/key/health` - Check key health + +### Common Permission Scenarios + +**Read-only access** (default): +```json +["/key/info", "/key/health"] +``` + +**Allow key creation but not deletion**: +```json +["/key/info", "/key/health", "/key/generate", "/key/update"] +``` + +**Full key management**: +```json +["/key/info", "/key/health", "/key/generate", "/key/update", "/key/delete", "/key/regenerate", "/key/block", "/key/unblock", "/key/list"] +``` + +### How to Configure Team Member Permissions + +#### View Current Permissions + +```shell +curl --location 'http://0.0.0.0:4000/team/permissions_list?team_id=team-123' \ + --header 'Authorization: Bearer sk-1234' +``` + +Expected Response: +```json +{ + "team_id": "team-123", + "team_member_permissions": ["/key/info", "/key/health"], + "all_available_permissions": ["/key/generate", "/key/update", "/key/delete", ...] +} +``` + +#### Update Team Member Permissions + +```shell +curl --location 'http://0.0.0.0:4000/team/update' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "team_id": "team-123", + "team_member_permissions": ["/key/info", "/key/health", "/key/generate", "/key/update"] + }' +``` + +This allows team members to: +- View key information +- Create new keys +- Update existing keys +- But NOT delete keys + +### Who Can Configure These Permissions? + +- **Proxy Admin**: Can configure permissions for any team +- **Org Admin**: Can configure permissions for teams in their organization +- **Team Admin**: Can configure permissions for their own team + +--- + +## Quick Comparison + +Here's the quick version: + +### Global Proxy Roles + +| Action | Proxy Admin | Proxy Admin Viewer | Internal User | Internal User Viewer ⚠️ (Deprecated) | +|--------|-------------|-------------------|---------------|-------------------------------------| +| Create organizations | ✅ | ❌ | ❌ | ❌ | +| Create teams | ✅ | ❌ | ❌ | ❌ | +| Manage all teams | ✅ | ❌ | ❌ | ❌ | +| Create/delete any keys | ✅ | ❌ | ❌ | ❌ | +| Create/delete own keys | ✅ | ❌ | ✅ | ❌ | +| View all platform spend | ✅ | ✅ | ❌ | ❌ | +| View own spend | ✅ | ✅ | ✅ | ✅ | +| View all keys | ✅ | ✅ | ❌ | ❌ | +| View own keys | ✅ | ✅ | ✅ | ✅ | +| Add/remove users | ✅ | ❌ | ❌ | ❌ | + +> **Note:** The `internal_user_viewer` role is deprecated. Use team/org specific roles for better granular access control. + +### Organization/Team Specific Roles + +| Action | Org Admin | Team Admin | +|--------|-----------|------------| +| Create teams (in their org) | ✅ | ❌ | +| Manage teams in their org | ✅ | ❌ | +| Manage their specific team | ✅ | ✅ | +| Add/remove team members | ✅ (in their org) | ✅ (their team only) | +| Update team budgets | ✅ (in their org) | ✅ (their team only) | +| Create keys for team members | ✅ (in their org) | ✅ (their team only) | +| View organization spend | ✅ (their org) | ❌ | +| View team spend | ✅ (in their org) | ✅ (their team) | +| Create organizations | ❌ | ❌ | +| View all platform spend | ❌ | ❌ | ## Onboarding Organizations +✨ **This is a Premium Feature** + ### 1. Creating a new Organization Any user with role=`proxy_admin` can create a new organization @@ -124,18 +441,79 @@ Expected Response ``` -### `Organization Admin` - Add an `Internal User` +### 4. `Organization Admin` - Add a Team Admin -The organization admin will use the virtual key created in [step 2](#2-adding-an-org_admin-to-an-organization) to add an Internal User to the `engineering_team` Team. +✨ **This is a Premium Feature** -- We will assign role=`internal_user` so the user can create Virtual Keys for themselves +The organization admin can now add a team admin who will manage the `engineering_team`. + +- We assign role=`admin` to make them a team admin for this specific team - `team_id` is from [step 3](#3-organization-admin---create-a-team) ```shell curl -X POST 'http://0.0.0.0:4000/team/member_add' \ - -H 'Authorization: Bearer sk-1234' \ + -H 'Authorization: Bearer sk-7shH8TGMAofR4zQpAAo6kQ' \ -H 'Content-Type: application/json' \ - -d '{"team_id": "01044ee8-441b-45f4-be7d-c70e002722d8", "member": {"role": "internal_user", "user_id": "krrish@berri.ai"}}' - + -d '{"team_id": "01044ee8-441b-45f4-be7d-c70e002722d8", "member": {"role": "admin", "user_id": "john@company.com"}}' +``` + +Now `john@company.com` is a team admin. They can manage the `engineering_team` - add members, update budgets, create keys - but they can't touch other teams. + +Create a Virtual Key for the team admin: + +```shell +curl --location 'http://0.0.0.0:4000/key/generate' \ + --header 'Authorization: Bearer sk-7shH8TGMAofR4zQpAAo6kQ' \ + --header 'Content-Type: application/json' \ + --data '{"user_id": "john@company.com"}' +``` + +Expected Response: + +```json +{ + "models": [], + "user_id": "john@company.com", + "key": "sk-TeamAdminKey123", + "key_name": "sk-...Key123" +} +``` + +### 5. `Team Admin` - Add Team Members + +Now the team admin can use their key to add team members without needing to ask the org admin. + +```shell +curl -X POST 'http://0.0.0.0:4000/team/member_add' \ + -H 'Authorization: Bearer sk-TeamAdminKey123' \ + -H 'Content-Type: application/json' \ + -d '{"team_id": "01044ee8-441b-45f4-be7d-c70e002722d8", "member": {"role": "user", "user_id": "krrish@berri.ai"}}' +``` + +The team admin can also create keys for their team members: + +```shell +curl --location 'http://0.0.0.0:4000/key/generate' \ + --header 'Authorization: Bearer sk-TeamAdminKey123' \ + --header 'Content-Type: application/json' \ + --data '{ + "user_id": "krrish@berri.ai", + "team_id": "01044ee8-441b-45f4-be7d-c70e002722d8" + }' +``` + +### 6. `Team Admin` - Update Team Settings + +The team admin can update team budgets and rate limits: + +```shell +curl --location 'http://0.0.0.0:4000/team/update' \ + --header 'Authorization: Bearer sk-TeamAdminKey123' \ + --header 'Content-Type: application/json' \ + --data '{ + "team_id": "01044ee8-441b-45f4-be7d-c70e002722d8", + "max_budget": 100, + "rpm_limit": 1000 + }' ``` diff --git a/docs/my-website/docs/proxy/admin_ui_sso.md b/docs/my-website/docs/proxy/admin_ui_sso.md index bd18dd9c690..ae082848b6b 100644 --- a/docs/my-website/docs/proxy/admin_ui_sso.md +++ b/docs/my-website/docs/proxy/admin_ui_sso.md @@ -320,6 +320,16 @@ Okta requires the `GENERIC_CLIENT_STATE` parameter: GENERIC_CLIENT_STATE="random-string" # Required for Okta ``` +### Okta PKCE + +If your Okta application is configured to require PKCE (Proof Key for Code Exchange), enable it by setting: + +```bash +GENERIC_CLIENT_USE_PKCE="true" +``` + +This is required when your Okta app settings enforce PKCE for enhanced security. LiteLLM will automatically handle PKCE parameter generation and verification during the OAuth flow. + ### Common Configuration Issues #### Missing Protocol in Base URL diff --git a/docs/my-website/docs/proxy/caching.md b/docs/my-website/docs/proxy/caching.md index 9cfd796d90f..6da977c8b05 100644 --- a/docs/my-website/docs/proxy/caching.md +++ b/docs/my-website/docs/proxy/caching.md @@ -1018,6 +1018,21 @@ cache_params: ``` +## Provider-Specific Optional Parameters Caching + +By default, LiteLLM only includes standard OpenAI parameters in cache keys. However, some providers (like Vertex AI) use additional parameters that affect the output but aren't included in the standard cache key generation. + +### Enable Provider-Specific Parameter Caching + +Add this setting to your `config.yaml` to include provider-specific optional parameters in cache keys: + +```yaml +litellm_settings: + cache: True + cache_params: + type: "redis" + enable_caching_on_provider_specific_optional_params: True # Include provider-specific params in cache keys +``` ## Advanced - user api key cache ttl Configure how long the in-memory cache stores the key object (prevents db requests) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 4e440857261..4d02d5729bf 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -101,6 +101,7 @@ general_settings: disable_retry_on_max_parallel_request_limit_error: boolean # turn off retries when max parallel request limit is reached disable_reset_budget: boolean # turn off reset budget scheduled task disable_adding_master_key_hash_to_db: boolean # turn off storing master key hash in db, for spend tracking + disable_responses_id_security: boolean # turn off response ID security checks that prevent users from accessing other users' responses enable_jwt_auth: boolean # allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims enforce_user_param: boolean # requires all openai endpoint requests to have a 'user' param allowed_routes: ["route1", "route2"] # list of allowed proxy API routes - a user can access. (currently JWT-Auth only) @@ -197,6 +198,7 @@ router_settings: | disable_retry_on_max_parallel_request_limit_error | boolean | If true, turns off retries when max parallel request limit is reached | | disable_reset_budget | boolean | If true, turns off reset budget scheduled task | | disable_adding_master_key_hash_to_db | boolean | If true, turns off storing master key hash in db | +| disable_responses_id_security | boolean | If true, disables response ID security checks that prevent users from accessing response IDs from other users. When false (default), response IDs are encrypted with user information to ensure users can only access their own responses. Applies to /v1/responses endpoints | | enable_jwt_auth | boolean | allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims. [Doc on JWT Tokens](token_auth) | | enforce_user_param | boolean | If true, requires all OpenAI endpoint requests to have a 'user' param. [Doc on call hooks](call_hooks)| | allowed_routes | array of strings | List of allowed proxy API routes a user can access [Doc on controlling allowed routes](enterprise#control-available-public-private-routes)| @@ -230,7 +232,7 @@ router_settings: | max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. | | proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. **Default is 597 seconds** | | proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. **Default is 605 seconds** | -| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 10 seconds** | +| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 30 seconds** | | proxy_batch_polling_interval | int | Time (in seconds) to wait before polling a batch, to check if it's completed. **Default is 6000 seconds (1 hour)** | | alerting_args | dict | Args for Slack Alerting [Doc on Slack Alerting](./alerting.md) | | custom_key_generate | str | Custom function for key generation [Doc on custom key generation](./virtual_keys.md#custom--key-generate) | @@ -340,6 +342,7 @@ router_settings: | router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) | | optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Currently supported: 'router_budget_limiting', 'prompt_caching' | | ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. | +| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search.md) | ### environment variables - Reference @@ -358,6 +361,10 @@ router_settings: | AIOHTTP_TRUST_ENV | Flag to enable aiohttp trust environment. When this is set to True, aiohttp will respect HTTP(S)_PROXY env vars. **Default is False** | AIOHTTP_TTL_DNS_CACHE | DNS cache time-to-live for aiohttp in seconds. **Default is 300** | ALLOWED_EMAIL_DOMAINS | List of email domains allowed for access +| APSCHEDULER_COALESCE | Whether to combine multiple pending executions of a job into one. **Default is False** +| APSCHEDULER_MAX_INSTANCES | Maximum number of concurrent instances of each job. **Default is 1** +| APSCHEDULER_MISFIRE_GRACE_TIME | Grace time in seconds for misfired jobs. **Default is 1** +| APSCHEDULER_REPLACE_EXISTING | Whether to replace existing jobs with the same ID. **Default is False** | ARIZE_API_KEY | API key for Arize platform integration | ARIZE_SPACE_KEY | Space key for Arize platform | ARGILLA_BATCH_SIZE | Batch size for Argilla logging @@ -389,10 +396,11 @@ router_settings: | AZURE_CERTIFICATE_PASSWORD | Password for Azure OpenAI certificate | AZURE_CLIENT_ID | Client ID for Azure services | AZURE_CLIENT_SECRET | Client secret for Azure services -| AZURE_CODE_INTERPRETER_COST_PER_SESSION | Cost per session for Azure Code Interpreter service | AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS | Input cost per 1K tokens for Azure Computer Use service | AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS | Output cost per 1K tokens for Azure Computer Use service | AZURE_DEFAULT_RESPONSES_API_VERSION | Version of the Azure Default Responses API being used. Default is "preview" +| AZURE_DOCUMENT_INTELLIGENCE_API_VERSION | API version for Azure Document Intelligence service +| AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI | Default DPI (dots per inch) setting for Azure Document Intelligence service | AZURE_TENANT_ID | Tenant ID for Azure Active Directory | AZURE_USERNAME | Username for Azure services, use in conjunction with AZURE_PASSWORD for azure ad token with basic username/password workflow | AZURE_PASSWORD | Password for Azure services, use in conjunction with AZURE_USERNAME for azure ad token with basic username/password workflow @@ -423,6 +431,12 @@ router_settings: | CLOUDZERO_MAX_FETCHED_DATA_RECORDS | Maximum number of data records to fetch from CloudZero | CLOUDZERO_TIMEZONE | Timezone for date handling (default: UTC) | CONFIG_FILE_PATH | File path for configuration file +| CYBERARK_ACCOUNT | CyberArk account name for secret management +| CYBERARK_API_BASE | Base URL for CyberArk API +| CYBERARK_API_KEY | API key for CyberArk secret management service +| CYBERARK_CLIENT_CERT | Path to client certificate for CyberArk authentication +| CYBERARK_CLIENT_KEY | Path to client key for CyberArk authentication +| CYBERARK_USERNAME | Username for CyberArk authentication | CONFIDENT_API_KEY | API key for DeepEval integration | CUSTOM_TIKTOKEN_CACHE_DIR | Custom directory for Tiktoken cache | CONFIDENT_API_KEY | API key for Confident AI (Deepeval) Logging service @@ -439,9 +453,15 @@ router_settings: | DAYS_IN_A_MONTH | Days in a month for calculation purposes. Default is 28 | DAYS_IN_A_WEEK | Days in a week for calculation purposes. Default is 7 | DAYS_IN_A_YEAR | Days in a year for calculation purposes. Default is 365 +| DYNAMOAI_API_KEY | API key for DynamoAI Guardrails service +| DYNAMOAI_API_BASE | Base URL for DynamoAI API. Default is https://api.dynamo.ai +| DYNAMOAI_MODEL_ID | Model ID for DynamoAI tracking/logging purposes +| DYNAMOAI_POLICY_IDS | Comma-separated list of DynamoAI policy IDs to apply | DD_BASE_URL | Base URL for Datadog integration | DATADOG_BASE_URL | (Alternative to DD_BASE_URL) Base URL for Datadog integration | _DATADOG_BASE_URL | (Alternative to DD_BASE_URL) Base URL for Datadog integration +| DD_AGENT_HOST | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API +| DD_AGENT_PORT | Port of DataDog agent for log intake. Default is 10518 | DD_API_KEY | API key for Datadog integration | DD_SITE | Site URL for Datadog (e.g., datadoghq.com) | DD_SOURCE | Source identifier for Datadog logs @@ -456,9 +476,11 @@ router_settings: | DEFAULT_CLIENT_DISCONNECT_CHECK_TIMEOUT_SECONDS | Timeout in seconds for checking client disconnection. Default is 1 | DEFAULT_COOLDOWN_TIME_SECONDS | Duration in seconds to cooldown a model after failures. Default is 5 | DEFAULT_CRON_JOB_LOCK_TTL_SECONDS | Time-to-live for cron job locks in seconds. Default is 60 (1 minute) +| DEFAULT_DATAFORSEO_LOCATION_CODE | Default location code for DataForSEO search API. Default is 2250 (France) | DEFAULT_FAILURE_THRESHOLD_PERCENT | Threshold percentage of failures to cool down a deployment. Default is 0.5 (50%) | DEFAULT_FLUSH_INTERVAL_SECONDS | Default interval in seconds for flushing operations. Default is 5 | DEFAULT_HEALTH_CHECK_INTERVAL | Default interval in seconds for health checks. Default is 300 (5 minutes) +| DEFAULT_HEALTH_CHECK_PROMPT | Default prompt used during health checks for non-image models. Default is "test from litellm" | DEFAULT_IMAGE_HEIGHT | Default height for images. Default is 300 | DEFAULT_IMAGE_TOKEN_COUNT | Default token count for images. Default is 250 | DEFAULT_IMAGE_WIDTH | Default width for images. Default is 300 @@ -470,6 +492,7 @@ router_settings: | DEFAULT_MAX_RETRIES | Default maximum retry attempts. Default is 2 | DEFAULT_MAX_TOKENS | Default maximum tokens for LLM calls. Default is 4096 | DEFAULT_MAX_TOKENS_FOR_TRITON | Default maximum tokens for Triton models. Default is 2000 +| DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE | Default maximum size for redis batch cache. Default is 1000 | DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20 | DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10 | DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602 @@ -484,6 +507,7 @@ router_settings: | DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH | Default minimal reasoning effort thinking budget for Gemini 2.5 Flash. Default is 512 | DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE | Default minimal reasoning effort thinking budget for Gemini 2.5 Flash Lite. Default is 512 | DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO | Default minimal reasoning effort thinking budget for Gemini 2.5 Pro. Default is 512 +| DEFAULT_REDIS_MAJOR_VERSION | Default Redis major version to assume when version cannot be determined. Default is 7 | DEFAULT_REDIS_SYNC_INTERVAL | Default Redis synchronization interval in seconds. Default is 1 | DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND | Default price per second for Replicate GPU. Default is 0.001400 | DEFAULT_REPLICATE_POLLING_DELAY_SECONDS | Default delay in seconds for Replicate polling. Default is 1 @@ -495,11 +519,13 @@ router_settings: | DEFAULT_SLACK_ALERTING_THRESHOLD | Default threshold for Slack alerting. Default is 300 | DEFAULT_SOFT_BUDGET | Default soft budget for LiteLLM proxy keys. Default is 50.0 | DEFAULT_TRIM_RATIO | Default ratio of tokens to trim from prompt end. Default is 0.75 +| DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS | Default duration for video generation in seconds in google. Default is 8 | DIRECT_URL | Direct URL for service endpoint | DISABLE_ADMIN_UI | Toggle to disable the admin UI | DISABLE_AIOHTTP_TRANSPORT | Flag to disable aiohttp transport. When this is set to True, litellm will use httpx instead of aiohttp. **Default is False** | DISABLE_AIOHTTP_TRUST_ENV | Flag to disable aiohttp trust environment. When this is set to True, litellm will not trust the environment for aiohttp eg. `HTTP_PROXY` and `HTTPS_PROXY` environment variables will not be used when this is set to True. **Default is False** | DISABLE_SCHEMA_UPDATE | Toggle to disable schema updates +| DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE | Threshold for deployment failures per minute before enforcing rate limits in parallel request limiter. Default is 1 | DOCS_DESCRIPTION | Description text for documentation pages | DOCS_FILTERED | Flag indicating filtered documentation | DOCS_TITLE | Title of the documentation pages @@ -533,6 +559,7 @@ router_settings: | GENERIC_CLIENT_ID | Client ID for generic OAuth providers | GENERIC_CLIENT_SECRET | Client secret for generic OAuth providers | GENERIC_CLIENT_STATE | State parameter for generic client authentication +| GENERIC_CLIENT_USE_PKCE | Enable PKCE (Proof Key for Code Exchange) for generic OAuth providers. Set to "true" when your OAuth provider requires PKCE. **Default is false** | GENERIC_SSO_HEADERS | Comma-separated list of additional headers to add to the request - e.g. Authorization=Bearer ``, Content-Type=application/json, etc. | GENERIC_INCLUDE_CLIENT_ID | Include client ID in requests for OAuth | GENERIC_SCOPE | Scope settings for generic OAuth providers @@ -555,6 +582,8 @@ router_settings: | GITHUB_COPILOT_ACCESS_TOKEN_FILE | File to store GitHub Copilot access token for `github_copilot` llm provider | GREENSCALE_API_KEY | API key for Greenscale service | GREENSCALE_ENDPOINT | Endpoint URL for Greenscale service +| GRAYSWAN_API_BASE | Base URL for GraySwan API. Default is https://api.grayswan.ai +| GRAYSWAN_API_KEY | API key for GraySwan Cygnal service | GOOGLE_APPLICATION_CREDENTIALS | Path to Google Cloud credentials JSON file | GOOGLE_CLIENT_ID | Client ID for Google OAuth | GOOGLE_CLIENT_SECRET | Client secret for Google OAuth @@ -565,9 +594,14 @@ router_settings: | HEROKU_API_KEY | API key for Heroku services | HF_API_BASE | Base URL for Hugging Face API | HCP_VAULT_ADDR | Address for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault) +| HCP_VAULT_APPROLE_MOUNT_PATH | Mount path for AppRole authentication in [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault). Default is "approle" +| HCP_VAULT_APPROLE_ROLE_ID | Role ID for AppRole authentication in [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault) +| HCP_VAULT_APPROLE_SECRET_ID | Secret ID for AppRole authentication in [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault) | HCP_VAULT_CLIENT_CERT | Path to client certificate for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault) | HCP_VAULT_CLIENT_KEY | Path to client key for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault) +| HCP_VAULT_MOUNT_NAME | Mount name for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault) | HCP_VAULT_NAMESPACE | Namespace for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault) +| HCP_VAULT_PATH_PREFIX | Path prefix for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault) | HCP_VAULT_TOKEN | Token for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault) | HCP_VAULT_CERT_ROLE | Role for [Hashicorp Vault Secret Manager Auth](../secret.md#hashicorp-vault) | HELICONE_API_KEY | API key for Helicone service @@ -578,6 +612,8 @@ router_settings: | HUGGINGFACE_API_KEY | API key for Hugging Face API | HUMANLOOP_PROMPT_CACHE_TTL_SECONDS | Time-to-live in seconds for cached prompts in Humanloop. Default is 60 | IAM_TOKEN_DB_AUTH | IAM token for database authentication +| IBM_GUARDRAILS_API_BASE | Base URL for IBM Guardrails API +| IBM_GUARDRAILS_AUTH_TOKEN | Authorization bearer token for IBM Guardrails API | INITIAL_RETRY_DELAY | Initial delay in seconds for retrying requests. Default is 0.5 | JITTER | Jitter factor for retry delay calculations. Default is 0.75 | JSON_LOGS | Enable JSON formatted logging @@ -625,6 +661,7 @@ router_settings: | LITELLM_LICENSE | License key for LiteLLM usage | LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM | LITELLM_LOG | Enable detailed logging for LiteLLM +| LITELLM_MODEL_COST_MAP_URL | URL for fetching model cost map data. Default is https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json | LITELLM_LOG_FILE | File path to write LiteLLM logs to. When set, logs will be written to both console and the specified file | LITELLM_LOGGER_NAME | Name for OTEL logger | LITELLM_METER_NAME | Name for OTEL Meter @@ -632,6 +669,7 @@ router_settings: | LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL | LITELLM_MASTER_KEY | Master key for proxy authentication | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development) +| LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60 | LITELLM_SALT_KEY | Salt key for encryption in LiteLLM | LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections. @@ -655,7 +693,7 @@ router_settings: | MAX_TOKEN_TRIMMING_ATTEMPTS | Maximum number of attempts to trim a token message. Default is 10 | MAXIMUM_TRACEBACK_LINES_TO_LOG | Maximum number of lines to log in traceback in LiteLLM Logs UI. Default is 100 | MAX_RETRY_DELAY | Maximum delay in seconds for retrying requests. Default is 8.0 -| MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 20. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times. +| MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 50. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times. | MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001 | MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024 | MISTRAL_API_BASE | Base URL for Mistral API. Default is https://api.mistral.ai @@ -711,10 +749,11 @@ router_settings: | PROMPTLAYER_API_KEY | API key for PromptLayer integration | PROXY_ADMIN_ID | Admin identifier for proxy server | PROXY_BASE_URL | Base URL for proxy service -| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10 +| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 30 | PROXY_BATCH_POLLING_INTERVAL | Time in seconds to wait before polling a batch, to check if it's completed. Default is 6000s (1 hour) | PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605 | PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597 +| PYTHON_GC_THRESHOLD | GC thresholds ('gen0,gen1,gen2', e.g. '1000,50,50'); defaults to Python’s values. | PROXY_LOGOUT_URL | URL for logging out of the proxy service | QDRANT_API_BASE | Base URL for Qdrant API | QDRANT_API_KEY | API key for Qdrant service @@ -730,14 +769,20 @@ router_settings: | REDIS_GCP_SSL_CA_CERTS | Path to SSL CA certificate file for secure GCP Memorystore Redis connections | REDOC_URL | The path to the Redoc Fast API documentation. **By default this is "/redoc"** | REPEATED_STREAMING_CHUNK_LIMIT | Limit for repeated streaming chunks to detect looping. Default is 100 +| REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES | Maximum size in bytes for WebSocket messages in realtime connections. Default is None. | REPLICATE_MODEL_NAME_WITH_ID_LENGTH | Length of Replicate model names with ID. Default is 64 | REPLICATE_POLLING_DELAY_SECONDS | Delay in seconds for Replicate polling operations. Default is 0.5 | REQUEST_TIMEOUT | Timeout in seconds for requests. Default is 6000 | ROUTER_MAX_FALLBACKS | Maximum number of fallbacks for router. Default is 5 +| RUNWAYML_DEFAULT_API_VERSION | Default API version for RunwayML service. Default is "2024-11-06" +| RUNWAYML_POLLING_TIMEOUT | Timeout in seconds for RunwayML image generation polling. Default is 600 (10 minutes) | SECRET_MANAGER_REFRESH_INTERVAL | Refresh interval in seconds for secret manager. Default is 86400 (24 hours) | SEPARATE_HEALTH_APP | If set to '1', runs health endpoints on a separate ASGI app and port. Default: '0'. | SEPARATE_HEALTH_PORT | Port for the separate health endpoints app. Only used if SEPARATE_HEALTH_APP=1. Default: 4001. | SERVER_ROOT_PATH | Root path for the server application +| SEND_USER_API_KEY_ALIAS | Flag to send user API key alias to Zscaler AI Guard. Default is False +| SEND_USER_API_KEY_TEAM_ID | Flag to send user API key team ID to Zscaler AI Guard. Default is False +| SEND_USER_API_KEY_USER_ID | Flag to send user API key user ID to Zscaler AI Guard. Default is False | SET_VERBOSE | Flag to enable verbose logging | SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD | Minimum number of requests to consider "reasonable traffic" for single-deployment cooldown logic. Default is 1000 | SLACK_DAILY_REPORT_FREQUENCY | Frequency of daily Slack reports (e.g., daily, weekly) @@ -752,6 +797,7 @@ router_settings: | SPEND_LOGS_URL | URL for retrieving spend logs | SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000 | SSL_CERTIFICATE | Path to the SSL certificate file +| SSL_ECDH_CURVE | ECDH curve for SSL/TLS key exchange (e.g., 'X25519' to disable PQC). | SSL_SECURITY_LEVEL | [BETA] Security level for SSL/TLS connections. E.g. `DEFAULT@SECLEVEL=1` | SSL_VERIFY | Flag to enable or disable SSL certificate verification | SSL_CERT_FILE | Path to the SSL certificate file for custom CA bundle @@ -784,4 +830,7 @@ router_settings: | SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000 | COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY | Maximum size for CoroutineChecker in-memory cache. Default is 1000 | DEFAULT_SHARED_HEALTH_CHECK_TTL | Time-to-live in seconds for cached health check results in shared health check mode. Default is 300 (5 minutes) -| DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL | Time-to-live in seconds for health check lock in shared health check mode. Default is 60 (1 minute) \ No newline at end of file +| DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL | Time-to-live in seconds for health check lock in shared health check mode. Default is 60 (1 minute) +| ZSCALER_AI_GUARD_API_KEY | API key for Zscaler AI Guard service +| ZSCALER_AI_GUARD_POLICY_ID | Policy ID for Zscaler AI Guard guardrails +| ZSCALER_AI_GUARD_URL | Base URL for Zscaler AI Guard API. Default is https://api.us1.zseclipse.net/v1/detection/execute-policy \ No newline at end of file diff --git a/docs/my-website/docs/proxy/cost_tracking.md b/docs/my-website/docs/proxy/cost_tracking.md index 85147e12c66..019cd62c620 100644 --- a/docs/my-website/docs/proxy/cost_tracking.md +++ b/docs/my-website/docs/proxy/cost_tracking.md @@ -2,14 +2,14 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import Image from '@theme/IdealImage'; -# 💸 Spend Tracking +# Spend Tracking Track spend for keys, users, and teams across 100+ LLMs. LiteLLM automatically tracks spend for all known models. See our [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json) :::tip Keep Pricing Data Updated -[Sync model pricing data from GitHub](../sync_models_github.md) to ensure accurate cost tracking. +[Sync model pricing data from GitHub](./sync_models_github.md) to ensure accurate cost tracking. ::: ### How to Track Spend with LiteLLM @@ -23,7 +23,7 @@ LiteLLM automatically tracks spend for all known models. See our [model cost map -```python +```python title="Send Request with Spend Tracking" showLineNumbers import openai client = openai.OpenAI( api_key="sk-1234", @@ -55,7 +55,7 @@ print(response) Pass `metadata` as part of the request body -```shell +```shell title="Curl Request with Spend Tracking" showLineNumbers curl --location 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ --header 'Authorization: Bearer sk-1234' \ @@ -77,7 +77,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ -```python +```python title="Langchain with Spend Tracking" showLineNumbers from langchain.chat_models import ChatOpenAI from langchain.prompts.chat import ( ChatPromptTemplate, @@ -131,7 +131,7 @@ Expect to see `x-litellm-response-cost` in the response headers with calculated The following spend gets tracked in Table `LiteLLM_SpendLogs` -```json +```json title="Spend Log Entry Format" showLineNumbers { "api_key": "fe6b0cab4ff5a5a8df823196cc8a450*****", # Hash of API Key used "user": "default_user", # Internal User (LiteLLM_UserTable) that owns `api_key=sk-1234`. @@ -169,7 +169,7 @@ Schedule a [meeting with us to get your Enterprise License](https://calendly.com Create Key with with `permissions={"get_spend_routes": true}` -```shell +```shell title="Generate Key with Spend Route Permissions" showLineNumbers curl --location 'http://0.0.0.0:4000/key/generate' \ --header 'Authorization: Bearer sk-1234' \ --header 'Content-Type: application/json' \ @@ -216,7 +216,7 @@ curl -X POST \ Assuming you have been issuing keys for end users, and setting their `user_id` on the key, you can check their usage. -```shell title="Total for a user API" showLineNumbers +```shell title="Get User Spend - API Request" showLineNumbers curl -L -X GET 'http://localhost:4000/user/info?user_id=jane_smith' \ -H 'Authorization: Bearer sk-...' ``` @@ -840,14 +840,14 @@ The `/spend/logs` endpoint now supports a `summarize` parameter to control data **Get individual transaction logs:** -```bash +```bash title="Get Individual Transaction Logs" showLineNumbers curl -X GET "http://localhost:4000/spend/logs?start_date=2024-01-01&end_date=2024-01-02&summarize=false" \ -H "Authorization: Bearer sk-1234" ``` **Get summarized data (default):** -```bash +```bash title="Get Summarized Spend Data" showLineNumbers curl -X GET "http://localhost:4000/spend/logs?start_date=2024-01-01&end_date=2024-01-02" \ -H "Authorization: Bearer sk-1234" ``` diff --git a/docs/my-website/docs/proxy/custom_pricing.md b/docs/my-website/docs/proxy/custom_pricing.md index fc7312b92ac..4698889786b 100644 --- a/docs/my-website/docs/proxy/custom_pricing.md +++ b/docs/my-website/docs/proxy/custom_pricing.md @@ -2,23 +2,27 @@ import Image from '@theme/IdealImage'; # Custom LLM Pricing -Use this to register custom pricing for models. +## Overview -There's 2 ways to track cost: -- cost per token -- cost per second +LiteLLM provides flexible cost tracking and pricing customization for all LLM providers: + +- **Custom Pricing** - Override default model costs or set pricing for custom models +- **Cost Per Token** - Track costs based on input/output tokens (most common) +- **Cost Per Second** - Track costs based on runtime (e.g., Sagemaker) +- **Provider Discounts** - Apply percentage-based discounts to specific providers +- **Base Model Mapping** - Ensure accurate cost tracking for Azure deployments By default, the response cost is accessible in the logging object via `kwargs["response_cost"]` on success (sync + async). [**Learn More**](../observability/custom_callback.md) :::info -LiteLLM already has pricing for any model in our [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). +LiteLLM already has pricing for 100+ models in our [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). ::: ## Cost Per Second (e.g. Sagemaker) -### Usage with LiteLLM Proxy Server +#### Usage with LiteLLM Proxy Server **Step 1: Add pricing to config.yaml** ```yaml @@ -47,7 +51,7 @@ litellm /path/to/config.yaml ## Cost Per Token (e.g. Azure) -### Usage with LiteLLM Proxy Server +#### Usage with LiteLLM Proxy Server ```yaml model_list: @@ -62,6 +66,58 @@ model_list: output_cost_per_token: 0.000520 # 👈 ONLY to track cost per token ``` +## Provider-Specific Cost Discounts + +Apply percentage-based discounts to specific providers (e.g., negotiated enterprise pricing). + +#### Usage with LiteLLM Proxy Server + +**Step 1: Add discount config to config.yaml** + +```yaml +# Apply 5% discount to all Vertex AI and Gemini costs +cost_discount_config: + vertex_ai: 0.05 # 5% discount + gemini: 0.05 # 5% discount + openrouter: 0.05 # 5% discount + # openai: 0.10 # 10% discount (example) +``` + +**Step 2: Start proxy** + +```bash +litellm /path/to/config.yaml +``` + +The discount will be automatically applied to all cost calculations for the configured providers. + + +#### How Discounts Work + +- Discounts are applied **after** all other cost calculations (tokens, caching, tools, etc.) +- The discount is a percentage (0.05 = 5%, 0.10 = 10%, etc.) +- Discounts only apply to the configured providers +- Original cost, discount amount, and final cost are tracked in cost breakdown logs +- Discount information is returned in response headers: + - `x-litellm-response-cost` - Final cost after discount + - `x-litellm-response-cost-original` - Cost before discount + - `x-litellm-response-cost-discount-amount` - Discount amount in USD + +#### Supported Providers + +You can apply discounts to all LiteLLM supported providers. Common examples: + +- `vertex_ai` - Google Vertex AI +- `gemini` - Google Gemini +- `openai` - OpenAI +- `anthropic` - Anthropic +- `azure` - Azure OpenAI +- `bedrock` - AWS Bedrock +- `cohere` - Cohere +- `openrouter` - OpenRouter + +See the full list of providers in the [LlmProviders](https://github.com/BerriAI/litellm/blob/main/litellm/types/utils.py) enum. + ## Override Model Cost Map You can override [our model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json) with your own custom pricing for a mapped model. diff --git a/docs/my-website/docs/proxy/custom_prompt_management.md b/docs/my-website/docs/proxy/custom_prompt_management.md index 98e5228af36..f82e7fb68cb 100644 --- a/docs/my-website/docs/proxy/custom_prompt_management.md +++ b/docs/my-website/docs/proxy/custom_prompt_management.md @@ -173,6 +173,28 @@ curl -X POST http://0.0.0.0:4000/v1/chat/completions \ +### Using the LiteLLM SDK Directly + +If you call `litellm.completion()` from a Python script (without going through the proxy), register your custom prompt manager before making the request: + +```python + +import litellm +from custom_prompt import prompt_management + +litellm.callbacks = [prompt_management] +litellm.use_litellm_proxy = True + +response = litellm.completion( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + prompt_id="1234", + prompt_variables={"user_message": "hi"}, +) +``` + +> **Note:** `litellm.callbacks = [prompt_management]` (or equivalently `litellm.logging_callback_manager.add_litellm_callback(prompt_management)`) is required in SDK scripts. The proxy reads `callbacks` from `config.yaml` automatically, but standalone scripts do not. + The request will be transformed from: ```json { diff --git a/docs/my-website/docs/proxy/customers.md b/docs/my-website/docs/proxy/customers.md index ac160d26542..66142ca3d84 100644 --- a/docs/my-website/docs/proxy/customers.md +++ b/docs/my-website/docs/proxy/customers.md @@ -12,7 +12,7 @@ Track spend, set budgets for your customers. Make a /chat/completions call, pass 'user' - First call Works -```bash +```bash showLineNumbers title="Make request with customer ID" curl -X POST 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ --header 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY @@ -39,14 +39,14 @@ If the customer_id already exists, spend will be incremented. Call `/customer/info` to get a customer's all up spend -```bash +```bash showLineNumbers title="Get customer spend" curl -X GET 'http://0.0.0.0:4000/customer/info?end_user_id=ishaan3' \ # 👈 CUSTOMER ID -H 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY ``` Expected Response: -``` +```json showLineNumbers title="Response" { "user_id": "ishaan3", "blocked": false, @@ -67,20 +67,20 @@ E.g. if your server is `https://webhook.site` and your listening on `6ab090e8-c5 1. Add webhook url to your proxy environment: -```bash +```bash showLineNumbers title="Set webhook URL" export WEBHOOK_URL="https://webhook.site/6ab090e8-c55f-4a23-b075-3209f5c57906" ``` 2. Add 'webhook' to config.yaml -```yaml +```yaml showLineNumbers title="config.yaml" general_settings: alerting: ["webhook"] # 👈 KEY CHANGE ``` 3. Test it! -```bash +```bash showLineNumbers title="Test webhook" curl -X POST 'http://localhost:4000/chat/completions' \ -H 'Content-Type: application/json' \ -H 'Authorization: Bearer sk-1234' \ @@ -99,7 +99,7 @@ curl -X POST 'http://localhost:4000/chat/completions' \ Expected Response -```json +```json showLineNumbers title="Webhook event payload" { "spend": 0.0011120000000000001, # 👈 SPEND "max_budget": null, @@ -127,12 +127,51 @@ Expected Response Set customer budgets (e.g. monthly budgets, tpm/rpm limits) on LiteLLM Proxy +### Default Budget for All Customers + +Apply budget limits to all customers without explicit budgets. This is useful for rate limiting and spending controls across all end users. + +**Step 1: Create a default budget** + +```bash showLineNumbers title="Create default budget" +curl -X POST 'http://localhost:4000/budget/new' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "max_budget": 10, + "rpm_limit": 2, + "tpm_limit": 1000 +}' +``` + +**Step 2: Configure the default budget ID** + +```yaml showLineNumbers title="config.yaml" +litellm_settings: + max_end_user_budget_id: "budget_id_from_step_1" +``` + +**Step 3: Test it** + +```bash showLineNumbers title="Make request with customer ID" +curl -X POST 'http://localhost:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "user": "my-customer-id" +}' +``` + +The customer will be subject to the default budget limits (RPM, TPM, and $ budget). Customers with explicit budgets are unaffected. + ### Quick Start Create / Update a customer with budget **Create New Customer w/ budget** -```bash +```bash showLineNumbers title="Create customer with budget" curl -X POST 'http://0.0.0.0:4000/customer/new' -H 'Authorization: Bearer sk-1234' -H 'Content-Type: application/json' @@ -144,7 +183,7 @@ curl -X POST 'http://0.0.0.0:4000/customer/new' **Test it!** -```bash +```bash showLineNumbers title="Test customer budget" curl -X POST 'http://localhost:4000/chat/completions' \ -H 'Content-Type: application/json' \ -H 'Authorization: Bearer sk-1234' \ @@ -180,7 +219,7 @@ Create and assign customers to pricing tiers. Use the `/budget/new` endpoint for creating a new budget. [API Reference](https://litellm-api.up.railway.app/#/budget%20management/new_budget_budget_new_post) -```bash +```bash showLineNumbers title="Create budget via API" curl -X POST 'http://localhost:4000/budget/new' \ -H 'Content-Type: application/json' \ -H 'Authorization: Bearer sk-1234' \ @@ -200,7 +239,7 @@ In your application code, assign budget when creating a new customer. Just use the `budget_id` used when creating the budget. In our example, this is `my-free-tier`. -```bash +```bash showLineNumbers title="Assign budget to customer" curl -X POST 'http://localhost:4000/customer/new' \ -H 'Content-Type: application/json' \ -H 'Authorization: Bearer sk-1234' \ @@ -215,7 +254,7 @@ curl -X POST 'http://localhost:4000/customer/new' \ -```bash +```bash showLineNumbers title="Test with curl" curl -X POST 'http://localhost:4000/customer/new' \ -H 'Content-Type: application/json' \ -H 'Authorization: Bearer sk-1234' \ @@ -228,7 +267,7 @@ curl -X POST 'http://localhost:4000/customer/new' \ -```python +```python showLineNumbers title="Test with OpenAI SDK" from openai import OpenAI client = OpenAI( base_url="", diff --git a/docs/my-website/docs/proxy/demo.md b/docs/my-website/docs/proxy/demo.md deleted file mode 100644 index c4b8671aab9..00000000000 --- a/docs/my-website/docs/proxy/demo.md +++ /dev/null @@ -1,9 +0,0 @@ -# Demo App - -Here is a demo of the proxy. To log in pass in: - -- Username: admin -- Password: sk-1234 - - -[Demo UI](https://demo.litellm.ai/ui) diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 7d2389383d1..e40d7acc7c8 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -2,10 +2,12 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import Image from '@theme/IdealImage'; -# Docker, Deployment +# Docker, Helm, Terraform You can find the Dockerfile to build litellm proxy [here](https://github.com/BerriAI/litellm/blob/main/Dockerfile) +> Note: Production requires at least 4 CPU cores and 8 GB RAM. + ## Quick Start To start using Litellm, run the following commands in a shell: @@ -195,7 +197,7 @@ docker run \ s/o [Nicholas Cecere](https://www.linkedin.com/in/nicholas-cecere-24243549/) for his LiteLLM User Management Terraform -👉 [Go here for Terraform](https://github.com/ncecere/terraform-litellm-user-mgmt) +👉 [Go here for Terraform](https://github.com/BerriAI/terraform-provider-litellm) ### Kubernetes @@ -785,6 +787,16 @@ docker run --name litellm-proxy \ +### 6. Disable pulling live model prices + +Disable pulling the model prices from LiteLLM's [hosted model prices file](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json), if you're seeing long cold start times or network security issues. + +```env +export LITELLM_LOCAL_MODEL_COST_MAP="True" +``` + +This will use the local model prices file instead. + ## Platform-specific Guide diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md index f3da18065ec..d82a0b01d1d 100644 --- a/docs/my-website/docs/proxy/docker_quick_start.md +++ b/docs/my-website/docs/proxy/docker_quick_start.md @@ -2,7 +2,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# E2E Tutorial +# Getting Started Tutorial End-to-End tutorial for LiteLLM Proxy to: - Add an Azure OpenAI model @@ -82,6 +82,8 @@ model_list: ### Model List Specification +You can read more about how model resolution works in the [Model Configuration](#understanding-model-configuration) section. + - **`model_name`** (`str`) - This field should contain the name of the model as received. - **`litellm_params`** (`dict`) [See All LiteLLM Params](https://github.com/BerriAI/litellm/blob/559a6ad826b5daef41565f54f06c739c8c068b28/litellm/types/router.py#L222) - **`model`** (`str`) - Specifies the model name to be sent to `litellm.acompletion` / `litellm.aembedding`, etc. This is the identifier used by LiteLLM to route to the correct model + provider logic on the backend. @@ -89,6 +91,10 @@ model_list: - **`api_base`** (`str`) - The API base for your azure deployment. - **`api_version`** (`str`) - The API Version to use when calling Azure's OpenAI API. Get the latest Inference API version [here](https://learn.microsoft.com/en-us/azure/ai-services/openai/api-version-deprecation?source=recommendations#latest-preview-api-releases). +--- + + +--- ### Useful Links - [**All Supported LLM API Providers (OpenAI/Bedrock/Vertex/etc.)**](../providers/) @@ -407,6 +413,138 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ - [Set Budgets / Rate Limits per key/user/teams](./users.md) - [Dynamic TPM/RPM Limits for keys](./team_budgets.md#dynamic-tpmrpm-allocation) +## Key Concepts + +This section explains key concepts on LiteLLM AI Gateway. + +### Understanding Model Configuration + +For this config.yaml example: + +```yaml +model_list: + - model_name: gpt-4o + litellm_params: + model: azure/my_azure_deployment + api_base: os.environ/AZURE_API_BASE + api_key: "os.environ/AZURE_API_KEY" + api_version: "2025-01-01-preview" # [OPTIONAL] litellm uses the latest azure api_version by default +``` + +**How Model Resolution Works:** + +``` +Client Request LiteLLM Proxy Provider API +────────────── ──────────────── ───────────── + +POST /chat/completions +{ 1. Looks up model_name + "model": "gpt-4o" ──────────▶ in config.yaml + ... +} 2. Finds matching entry: + model_name: gpt-4o + + 3. Extracts litellm_params: + model: azure/my_azure_deployment + api_base: https://... + api_key: sk-... + + 4. Routes to provider ──▶ Azure OpenAI API + POST /deployments/my_azure_deployment/... +``` + +**Breaking Down the `model` Parameter under `litellm_params`:** + +```yaml +model_list: + - model_name: gpt-4o # What the client calls + litellm_params: + model: azure/my_azure_deployment # / + ───── ─────────────────── + │ │ + │ └─────▶ Model name sent to the provider API + │ + └─────────────────▶ Provider that LiteLLM routes to +``` + +**Visual Breakdown:** + +``` +model: azure/my_azure_deployment + └─┬─┘ └─────────┬─────────┘ + │ │ + │ └────▶ The actual model identifier that gets sent to Azure + │ (e.g., your deployment name, or the model name) + │ + └──────────────────▶ Tells LiteLLM which provider to use + (azure, openai, anthropic, bedrock, etc.) +``` + +**Key Concepts:** + +- **`model_name`**: The alias your client uses to call the model. This is what you send in your API requests (e.g., `gpt-4o`). + +- **`model` (in litellm_params)**: Format is `/` + - **Provider** (before `/`): Routes to the correct LLM provider (e.g., `azure`, `openai`, `anthropic`, `bedrock`) + - **Model identifier** (after `/`): The actual model/deployment name sent to that provider's API + +**Advanced Configuration Examples:** + +For custom OpenAI-compatible endpoints (e.g., vLLM, Ollama, custom deployments): + +```yaml +model_list: + - model_name: my-custom-model + litellm_params: + model: openai/nvidia/llama-3.2-nv-embedqa-1b-v2 + api_base: http://my-service.svc.cluster.local:8000/v1 + api_key: "sk-1234" +``` + +**Breaking down complex model paths:** + +``` +model: openai/nvidia/llama-3.2-nv-embedqa-1b-v2 + └─┬──┘ └────────────┬────────────────┘ + │ │ + │ └────▶ Full model string sent to the provider API + │ (in this case: "nvidia/llama-3.2-nv-embedqa-1b-v2") + │ + └──────────────────────▶ Provider (openai = OpenAI-compatible API) +``` + +The key point: Everything after the first `/` is passed as-is to the provider's API. + +**Common Patterns:** + +```yaml +model_list: + # Azure deployment + - model_name: gpt-4 + litellm_params: + model: azure/gpt-4-deployment + api_base: https://my-azure.openai.azure.com + + # OpenAI + - model_name: gpt-4 + litellm_params: + model: openai/gpt-4 + api_key: os.environ/OPENAI_API_KEY + + # Custom OpenAI-compatible endpoint + - model_name: my-llama-model + litellm_params: + model: openai/meta/llama-3-8b + api_base: http://my-vllm-server:8000/v1 + api_key: "optional-key" + + # Bedrock + - model_name: claude-3 + litellm_params: + model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0 + aws_region_name: us-east-1 +``` + ## Troubleshooting @@ -504,7 +642,7 @@ LiteLLM Proxy uses the [LiteLLM Python SDK](https://docs.litellm.ai/docs/routing - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- [Community Slack 💭](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3) +- [Community Slack 💭](https://www.litellm.ai/support) - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai diff --git a/docs/my-website/docs/proxy/dynamic_rate_limit.md b/docs/my-website/docs/proxy/dynamic_rate_limit.md index 06d49dfaf0f..9c875a51eba 100644 --- a/docs/my-website/docs/proxy/dynamic_rate_limit.md +++ b/docs/my-website/docs/proxy/dynamic_rate_limit.md @@ -136,9 +136,16 @@ model_list: litellm_settings: callbacks: ["dynamic_rate_limiter_v3"] - priority_reservation: - "prod": 0.9 # 90% reserved for production (9 RPM) - "dev": 0.1 # 10% reserved for development (1 RPM) + priority_reservation: + "prod": 0.9 # 90% reserved for production (9 RPM) + "dev": 0.1 # 10% reserved for development (1 RPM) + # Alternative format: + # "prod": + # type: "rpm" # Reserve based on requests per minute + # value: 9 # 9 RPM = 90% of 10 RPM capacity + # "dev": + # type: "tpm" # Reserve based on tokens per minute + # value: 100 # 100 TPM priority_reservation_settings: default_priority: 0 # Weight (0%) assigned to keys without explicit priority metadata saturation_threshold: 0.50 # A model is saturated if it has hit 50% of its RPM limit @@ -150,10 +157,12 @@ general_settings: **Configuration Details:** -`priority_reservation`: Dict[str, float] +`priority_reservation`: Dict[str, Union[float, PriorityReservationDict]] - **Key (str)**: Priority level name (can be any string like "prod", "dev", "critical", etc.) -- **Value (float)**: Percentage of total TPM/RPM to reserve (0.0 to 1.0) -- **Note**: Values should sum to 1.0 or less +- **Value**: Either a float (0.0-1.0) or dict with `type` and `value` + - Float: `0.9` = 90% of capacity + - Dict: `{"type": "rpm", "value": 9}` = 9 requests/min + - Supported types: `"percent"`, `"rpm"`, `"tpm"` `priority_reservation_settings`: Object (Optional) - **default_priority (float)**: Weight/percentage (0.0 to 1.0) assigned to API keys that have no priority metadata set (defaults to 0.5) diff --git a/docs/my-website/docs/proxy/email.md b/docs/my-website/docs/proxy/email.md index 9cd027da7f6..da8fc57deea 100644 --- a/docs/my-website/docs/proxy/email.md +++ b/docs/my-website/docs/proxy/email.md @@ -18,7 +18,7 @@ Send LiteLLM Proxy users emails for specific events. | Category | Details | |----------|---------| -| Supported Events | • User added as a user on LiteLLM Proxy
• Proxy API Key created for user | +| Supported Events | • User added as a user on LiteLLM Proxy
• Proxy API Key created for user
• Proxy API Key rotated for user | | Supported Email Integrations | • Resend API
• SMTP | ## Usage @@ -123,6 +123,35 @@ On the Create Key Modal, Select Advanced Settings > Set Send Email to True. style={{width: '70%', display: 'block', margin: '0 0 2rem 0'}} /> +### 3. Proxy API Key Rotated for User + +This email is sent when you rotate an API key for a user on LiteLLM Proxy. + + + +**How to trigger this event** + +On the LiteLLM Proxy UI, go to Virtual Keys > Click on a key > Click "Regenerate Key" + +:::info + +Ensure there is a `user_id` attached to the key. This would have been set when creating the key. + +::: + + + +After regenerating the key, the user will receive an email notification with: +- Security-focused messaging about the rotation +- The new API key (or a placeholder if `EMAIL_INCLUDE_API_KEY=false`) +- Instructions to update their applications +- Security best practices ## Email Customization @@ -141,6 +170,9 @@ LiteLLM allows you to customize various aspects of your email notifications. Bel | Email Signature | `EMAIL_SIGNATURE` | string (HTML) | Standard LiteLLM footer | `"

Best regards,
Your Team

Visit us

"` | HTML-formatted footer for all emails | | Invitation Subject | `EMAIL_SUBJECT_INVITATION` | string | "LiteLLM: New User Invitation" | `"Welcome to Your Company!"` | Subject line for invitation emails | | Key Creation Subject | `EMAIL_SUBJECT_KEY_CREATED` | string | "LiteLLM: API Key Created" | `"Your New API Key is Ready"` | Subject line for key creation emails | +| Key Rotation Subject | `EMAIL_SUBJECT_KEY_ROTATED` | string | "LiteLLM: API Key Rotated" | `"Your API Key Has Been Rotated"` | Subject line for key rotation emails | +| Include API Key | `EMAIL_INCLUDE_API_KEY` | boolean | true | `"false"` | Whether to include the actual API key in emails (set to false for enhanced security) | +| Proxy Base URL | `PROXY_BASE_URL` | string | http://0.0.0.0:4000 | `"https://proxy.your-company.com"` | Base URL for the LiteLLM Proxy (used in email links) | ## HTML Support in Email Signature @@ -180,8 +212,44 @@ EMAIL_SIGNATURE="

Best regards,
Your Company Team

View Documentation + +
+ +

Need Help?

+

If you have any questions or need assistance updating your systems, please contact us at {email_support_contact}.

+
+ {email_footer} +
+ + +""" + diff --git a/litellm/integrations/gitlab/__init__.py b/litellm/integrations/gitlab/__init__.py index cd22afc2ba0..c73a23b6874 100644 --- a/litellm/integrations/gitlab/__init__.py +++ b/litellm/integrations/gitlab/__init__.py @@ -8,7 +8,7 @@ if TYPE_CHECKING: from litellm.types.prompts.init_prompts import SupportedPromptIntegrations from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.types.prompts.init_prompts import PromptSpec, PromptLiteLLMParams -from .gitlab_prompt_manager import GitLabPromptManager +from .gitlab_prompt_manager import GitLabPromptManager, GitLabPromptCache # Global instances global_gitlab_config: Optional[dict] = None @@ -16,13 +16,13 @@ global_gitlab_config: Optional[dict] = None def set_global_gitlab_config(config: dict) -> None: """ - Set the global BitBucket configuration for prompt management. + Set the global gitlab configuration for prompt management. Args: - config: Dictionary containing BitBucket configuration - - workspace: BitBucket workspace name + config: Dictionary containing gitlab configuration + - workspace: gitlab workspace name - repository: Repository name - - access_token: BitBucket access token + - access_token: gitlab access token - branch: Branch to fetch prompts from (default: main) """ import litellm @@ -34,7 +34,7 @@ def prompt_initializer( litellm_params: "PromptLiteLLMParams", prompt_spec: "PromptSpec" ) -> "CustomPromptManagement": """ - Initialize a prompt from a BitBucket repository. + Initialize a prompt from a Gitlab repository. """ gitlab_config = getattr(litellm_params, "gitlab_config", None) prompt_id = getattr(litellm_params, "prompt_id", None) @@ -42,16 +42,16 @@ def prompt_initializer( if not gitlab_config: raise ValueError( - "bitbucket_config is required for BitBucket prompt integration" + "gitlab_config is required for gitlab prompt integration" ) try: - bitbucket_prompt_manager = GitLabPromptManager( + gitlab_prompt_manager = GitLabPromptManager( gitlab_config=gitlab_config, prompt_id=prompt_id, ) - return bitbucket_prompt_manager + return gitlab_prompt_manager except Exception as e: raise e @@ -90,6 +90,7 @@ prompt_initializer_registry = { # Export public API __all__ = [ "GitLabPromptManager", + "GitLabPromptCache", "set_global_gitlab_config", "global_gitlab_config", ] diff --git a/litellm/integrations/gitlab/gitlab_prompt_manager.py b/litellm/integrations/gitlab/gitlab_prompt_manager.py index b782f10ccc5..37013273cb0 100644 --- a/litellm/integrations/gitlab/gitlab_prompt_manager.py +++ b/litellm/integrations/gitlab/gitlab_prompt_manager.py @@ -12,10 +12,24 @@ from litellm.integrations.prompt_management_base import ( ) from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import StandardCallbackDynamicParams - from litellm.integrations.gitlab.gitlab_client import GitLabClient +GITLAB_PREFIX = "gitlab::" + +def encode_prompt_id(raw_id: str) -> str: + """Convert GitLab path IDs like 'invoice/extract' → 'gitlab::invoice::extract'""" + if raw_id.startswith(GITLAB_PREFIX): + return raw_id # already encoded + return f"{GITLAB_PREFIX}{raw_id.replace('/', '::')}" + +def decode_prompt_id(encoded_id: str) -> str: + """Convert 'gitlab::invoice::extract' → 'invoice/extract'""" + if not encoded_id.startswith(GITLAB_PREFIX): + return encoded_id + return encoded_id[len(GITLAB_PREFIX):].replace("::", "/") + + class GitLabPromptTemplate: def __init__( self, @@ -87,6 +101,7 @@ class GitLabTemplateManager: def _id_to_repo_path(self, prompt_id: str) -> str: """Map a prompt_id to a repo path (respects prompts_path and adds .prompt).""" + prompt_id = decode_prompt_id(prompt_id) if self.prompts_path: return f"{self.prompts_path}/{prompt_id}.prompt" return f"{prompt_id}.prompt" @@ -101,26 +116,27 @@ class GitLabTemplateManager: path = path[len(self.prompts_path.strip("/")) + 1 :] if path.endswith(".prompt"): path = path[: -len(".prompt")] - return path + return encode_prompt_id(path) # ---------- loading ---------- def _load_prompt_from_gitlab(self, prompt_id: str, *, ref: Optional[str] = None) -> None: """Load a specific .prompt file from GitLab (scoped under prompts_path if set).""" try: + # prompt_id = decode_prompt_id(prompt_id) file_path = self._id_to_repo_path(prompt_id) prompt_content = self.gitlab_client.get_file_content(file_path, ref=ref) if prompt_content: template = self._parse_prompt_file(prompt_content, prompt_id) self.prompts[prompt_id] = template except Exception as e: - raise Exception(f"Failed to load prompt '{prompt_id}' from GitLab: {e}") + raise Exception(f"Failed to load prompt '{encode_prompt_id(prompt_id)}' from GitLab: {e}") def load_all_prompts(self, *, recursive: bool = True) -> List[str]: """ Eagerly load all .prompt files from prompts_path. Returns loaded IDs. """ - files = self.list_templates(recursive=recursive) # reuse logic + files = self.list_templates(recursive=recursive) loaded: List[str] = [] for pid in files: if pid not in self.prompts: @@ -195,9 +211,6 @@ class GitLabTemplateManager: return self.prompts.get(template_id) def list_templates(self, *, recursive: bool = True) -> List[str]: - """ - List available prompt IDs discovered under prompts_path (no extension, relative to prompts_path). - """ """ List available prompt IDs under prompts_path (no extension). Compatible with both list_files signatures: @@ -248,7 +261,7 @@ class GitLabPromptManager(CustomPromptManagement): "access_token": "glpat_***", "tag": "v1.2.3", # optional; takes precedence "branch": "main", # default fallback - "prompts_path": "prompts/chat" # <--- NEW + "prompts_path": "prompts/chat" } """ @@ -438,9 +451,11 @@ class GitLabPromptManager(CustomPromptManagement): prompt_version: Optional[int] = None, ) -> PromptManagementClient: try: - if prompt_id not in self.prompt_manager.prompts: + decoded_id = decode_prompt_id(prompt_id) + if decoded_id not in self.prompt_manager.prompts: git_ref = getattr(dynamic_callback_params, "extra", {}).get("git_ref") if hasattr(dynamic_callback_params, "extra") else None - self.prompt_manager._load_prompt_from_gitlab(prompt_id, ref=git_ref) + self.prompt_manager._load_prompt_from_gitlab(decoded_id, ref=git_ref) + rendered_prompt, prompt_metadata = self.get_prompt_template( prompt_id, prompt_variables @@ -486,3 +501,148 @@ class GitLabPromptManager(CustomPromptManagement): prompt_label, prompt_version, ) + + +class GitLabPromptCache: + """ + Cache all .prompt files from a GitLab repo into memory. + + - Keys are the *repo file paths* (e.g. "prompts/chat/greet/hi.prompt") + mapped to JSON-like dicts containing content + metadata. + - Also exposes a by-ID view (ID == path relative to prompts_path without ".prompt", + e.g. "greet/hi"). + + Usage: + + cfg = { + "project": "group/subgroup/repo", + "access_token": "glpat_***", + "prompts_path": "prompts/chat", # optional, can be empty for repo root + # "branch": "main", # default is "main" + # "tag": "v1.2.3", # takes precedence over branch + # "base_url": "https://gitlab.com/api/v4" # default + } + + cache = GitLabPromptCache(cfg) + cache.load_all() # fetch + parse all .prompt files + + print(cache.list_files()) # repo file paths + print(cache.list_ids()) # template IDs relative to prompts_path + + prompt_json = cache.get_by_file("prompts/chat/greet/hi.prompt") + prompt_json2 = cache.get_by_id("greet/hi") + + # If GitLab content changes and you want to refresh: + cache.reload() # re-scan and refresh all + """ + + def __init__( + self, + gitlab_config: Dict[str, Any], + *, + ref: Optional[str] = None, + gitlab_client: Optional[GitLabClient] = None, + ) -> None: + # Build a PromptManager (which internally builds TemplateManager + Client) + self.prompt_manager = GitLabPromptManager( + gitlab_config=gitlab_config, + prompt_id=None, + ref=ref, + gitlab_client=gitlab_client, + ) + self.template_manager: GitLabTemplateManager = self.prompt_manager.prompt_manager + + # In-memory stores + self._by_file: Dict[str, Dict[str, Any]] = {} + self._by_id: Dict[str, Dict[str, Any]] = {} + + # ------------------------- + # Public API + # ------------------------- + + def load_all(self, *, recursive: bool = True) -> Dict[str, Dict[str, Any]]: + """ + Scan GitLab for all .prompt files under prompts_path, load and parse each, + and return the mapping of repo file path -> JSON-like dict. + """ + ids = self.template_manager.list_templates(recursive=recursive) # IDs relative to prompts_path + for pid in ids: + # Ensure template is loaded into TemplateManager + if pid not in self.template_manager.prompts: + self.template_manager._load_prompt_from_gitlab(pid) + + tmpl = self.template_manager.get_template(pid) + if tmpl is None: + # If something raced/failed, try once more + self.template_manager._load_prompt_from_gitlab(pid) + tmpl = self.template_manager.get_template(pid) + if tmpl is None: + continue + + file_path = self.template_manager._id_to_repo_path(pid) # "prompts/chat/..../file.prompt" + entry = self._template_to_json(pid, tmpl) + + self._by_file[file_path] = entry + # prefixed_id = pid if pid.startswith("gitlab::") else f"gitlab::{pid}" + encoded_id = encode_prompt_id(pid) + self._by_id[encoded_id] = entry + # self._by_id[pid] = entry + + return self._by_id + + def reload(self, *, recursive: bool = True) -> Dict[str, Dict[str, Any]]: + """Clear the cache and re-load from GitLab.""" + self._by_file.clear() + self._by_id.clear() + return self.load_all(recursive=recursive) + + def list_files(self) -> List[str]: + """Return the repo file paths currently cached.""" + return list(self._by_file.keys()) + + def list_ids(self) -> List[str]: + """Return the template IDs (relative to prompts_path, without extension) currently cached.""" + return list(self._by_id.keys()) + + def get_by_file(self, file_path: str) -> Optional[Dict[str, Any]]: + """Get a cached prompt JSON by repo file path.""" + return self._by_file.get(file_path) + + def get_by_id(self, prompt_id: str) -> Optional[Dict[str, Any]]: + """Get a cached prompt JSON by prompt ID (relative to prompts_path).""" + if prompt_id in self._by_id: + return self._by_id[prompt_id] + + # Try normalized forms + decoded = decode_prompt_id(prompt_id) + encoded = encode_prompt_id(decoded) + + return self._by_id.get(encoded) or self._by_id.get(decoded) + + # ------------------------- + # Internals + # ------------------------- + + def _template_to_json(self, prompt_id: str, tmpl: GitLabPromptTemplate) -> Dict[str, Any]: + """ + Normalize a GitLabPromptTemplate into a JSON-like dict that is easy to serialize. + """ + # Safer copy of metadata (avoid accidental mutation) + md = dict(tmpl.metadata or {}) + + # Pull standard fields (also present in metadata sometimes) + model = tmpl.model + temperature = tmpl.temperature + max_tokens = tmpl.max_tokens + optional_params = dict(tmpl.optional_params or {}) + + return { + "id": prompt_id, # e.g. "greet/hi" + "path": self.template_manager._id_to_repo_path(prompt_id), # e.g. "prompts/chat/greet/hi.prompt" + "content": tmpl.content, # rendered content (without frontmatter) + "metadata": md, # parsed frontmatter + "model": model, + "temperature": temperature, + "max_tokens": max_tokens, + "optional_params": optional_params, + } \ No newline at end of file diff --git a/litellm/integrations/helicone.py b/litellm/integrations/helicone.py index 79585a412b3..198cbaf4058 100644 --- a/litellm/integrations/helicone.py +++ b/litellm/integrations/helicone.py @@ -100,6 +100,11 @@ class HeliconeLogger: for header_key in proxy_headers: if header_key.startswith("helicone_"): metadata[header_key] = proxy_headers.get(header_key) + + # Remove OpenTelemetry span from metadata as it's not JSON serializable + # The span is used internally for tracing but shouldn't be logged to external services + if "litellm_parent_otel_span" in metadata: + metadata.pop("litellm_parent_otel_span") return metadata diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 7f807bb8b0c..c2a2cc77950 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -683,16 +683,35 @@ class LangFuseLogger: _usage_obj = getattr(response_obj, "usage", None) if _usage_obj: + # Safely get usage values, defaulting None to 0 for Langfuse compatibility. + # Some providers may return null for token counts. + prompt_tokens = getattr(_usage_obj, "prompt_tokens", None) or 0 + completion_tokens = ( + getattr(_usage_obj, "completion_tokens", None) or 0 + ) + total_tokens = getattr(_usage_obj, "total_tokens", None) or 0 + + cache_creation_input_tokens = ( + _usage_obj.get("cache_creation_input_tokens") or 0 + ) + cache_read_input_tokens = ( + _usage_obj.get("cache_read_input_tokens") or 0 + ) + usage = { - "prompt_tokens": _usage_obj.prompt_tokens, - "completion_tokens": _usage_obj.completion_tokens, + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, "total_cost": cost if self._supports_costs() else None, } - usage_details = LangfuseUsageDetails(input=_usage_obj.prompt_tokens, - output=_usage_obj.completion_tokens, - total=_usage_obj.total_tokens, - cache_creation_input_tokens=_usage_obj.get('cache_creation_input_tokens', 0), - cache_read_input_tokens=_usage_obj.get('cache_read_input_tokens', 0)) + # According to langfuse documentation: "the input value must be reduced by the number of cache_read_input_tokens" + input_tokens = prompt_tokens - cache_read_input_tokens + usage_details = LangfuseUsageDetails( + input=input_tokens, + output=completion_tokens, + total=total_tokens, + cache_creation_input_tokens=cache_creation_input_tokens, + cache_read_input_tokens=cache_read_input_tokens, + ) generation_name = clean_metadata.pop("generation_name", None) if generation_name is None: @@ -790,7 +809,7 @@ class LangFuseLogger: """ Get the responses API content for Langfuse logging """ - if hasattr(response_obj, 'output') and response_obj.output: + if hasattr(response_obj, "output") and response_obj.output: # ResponsesAPIResponse.output is a list of strings return response_obj.output else: @@ -880,29 +899,44 @@ class LangFuseLogger: guardrail_information = standard_logging_object.get( "guardrail_information", None ) - if guardrail_information is None: + if not guardrail_information: verbose_logger.debug( - "Not logging guardrail information as span because guardrail_information is None" + "Not logging guardrail information as span because guardrail_information is empty" ) return - span = trace.span( - name="guardrail", - input=guardrail_information.get("guardrail_request", None), - output=guardrail_information.get("guardrail_response", None), - metadata={ - "guardrail_name": guardrail_information.get("guardrail_name", None), - "guardrail_mode": guardrail_information.get("guardrail_mode", None), - "guardrail_masked_entity_count": guardrail_information.get( - "masked_entity_count", None - ), - }, - start_time=guardrail_information.get("start_time", None), # type: ignore - end_time=guardrail_information.get("end_time", None), # type: ignore - ) + if not isinstance(guardrail_information, list): + verbose_logger.debug( + "Not logging guardrail information as span because guardrail_information is not a list: %s", + type(guardrail_information), + ) + return - verbose_logger.debug(f"Logged guardrail information as span: {span}") - span.end() + for guardrail_entry in guardrail_information: + if not isinstance(guardrail_entry, dict): + verbose_logger.debug( + "Skipping guardrail entry with unexpected type: %s", + type(guardrail_entry), + ) + continue + + span = trace.span( + name="guardrail", + input=guardrail_entry.get("guardrail_request", None), + output=guardrail_entry.get("guardrail_response", None), + metadata={ + "guardrail_name": guardrail_entry.get("guardrail_name", None), + "guardrail_mode": guardrail_entry.get("guardrail_mode", None), + "guardrail_masked_entity_count": guardrail_entry.get( + "masked_entity_count", None + ), + }, + start_time=guardrail_entry.get("start_time", None), # type: ignore + end_time=guardrail_entry.get("end_time", None), # type: ignore + ) + + verbose_logger.debug(f"Logged guardrail information as span: {span}") + span.end() def _add_prompt_to_generation_params( diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index fbe480be95f..6992ea17cc8 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -5,6 +5,9 @@ from typing import TYPE_CHECKING, Any, Optional, Union from litellm._logging import verbose_logger from litellm.integrations.arize import _utils +from litellm.integrations.langfuse.langfuse_otel_attributes import ( + LangfuseLLMObsOTELAttributes, +) from litellm.integrations.opentelemetry import OpenTelemetry from litellm.types.integrations.langfuse_otel import ( LangfuseOtelConfig, @@ -33,26 +36,24 @@ LANGFUSE_CLOUD_EU_ENDPOINT = "https://cloud.langfuse.com/api/public/otel" LANGFUSE_CLOUD_US_ENDPOINT = "https://us.cloud.langfuse.com/api/public/otel" - class LangfuseOtelLogger(OpenTelemetry): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - @staticmethod def set_langfuse_otel_attributes(span: Span, kwargs, response_obj): """ Sets OpenTelemetry span attributes for Langfuse observability. Uses the same attribute setting logic as Arize Phoenix for consistency. """ - _utils.set_attributes(span, kwargs, response_obj) + + _utils.set_attributes(span, kwargs, response_obj, LangfuseLLMObsOTELAttributes) ######################################################### - # Set Langfuse specific attributes eg Langfuse Environment + # Set Langfuse specific attributes ######################################################### LangfuseOtelLogger._set_langfuse_specific_attributes( - span=span, - kwargs=kwargs + span=span, kwargs=kwargs, response_obj=response_obj ) return @@ -86,30 +87,10 @@ class LangfuseOtelLogger(OpenTelemetry): return metadata @staticmethod - def _set_langfuse_specific_attributes(span: Span, kwargs): - """ - Sets Langfuse specific metadata attributes onto the OTEL span. - - All keys supported by the vanilla Langfuse integration are mapped to - OTEL-safe attribute names defined in LangfuseSpanAttributes. Complex - values (lists/dicts) are serialised to JSON strings for OTEL - compatibility. - """ + def _set_metadata_attributes(span: Span, metadata: dict): + """Helper to set metadata attributes from mapping.""" from litellm.integrations.arize._utils import safe_set_attribute - # 1) Environment variable override - langfuse_environment = os.environ.get("LANGFUSE_TRACING_ENVIRONMENT") - if langfuse_environment: - safe_set_attribute( - span, - LangfuseSpanAttributes.LANGFUSE_ENVIRONMENT.value, - langfuse_environment, - ) - - # 2) Dynamic metadata from kwargs / headers - metadata = LangfuseOtelLogger._extract_langfuse_metadata(kwargs) - - # Mapping from metadata key -> OTEL attribute enum mapping = { "generation_name": LangfuseSpanAttributes.GENERATION_NAME, "generation_id": LangfuseSpanAttributes.GENERATION_ID, @@ -133,7 +114,6 @@ class LangfuseOtelLogger(OpenTelemetry): for key, enum_attr in mapping.items(): if key in metadata and metadata[key] is not None: value = metadata[key] - # Lists / dicts must be stringified for OTEL if isinstance(value, (list, dict)): try: value = json.dumps(value) @@ -141,6 +121,106 @@ class LangfuseOtelLogger(OpenTelemetry): value = str(value) safe_set_attribute(span, enum_attr.value, value) + @staticmethod + def _set_observation_output(span: Span, response_obj): + """Helper to set observation output attributes.""" + from litellm.integrations.arize._utils import safe_set_attribute + from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + + if not response_obj or not hasattr(response_obj, "get"): + return + + choices = response_obj.get("choices", []) + if choices: + first_choice = choices[0] + message = first_choice.get("message", {}) + tool_calls = message.get("tool_calls") + if tool_calls: + transformed_tool_calls = [] + for tool_call in tool_calls: + function = tool_call.get("function", {}) + arguments_str = function.get("arguments", "{}") + try: + arguments_obj = ( + json.loads(arguments_str) + if isinstance(arguments_str, str) + else arguments_str + ) + except json.JSONDecodeError: + arguments_obj = {} + langfuse_tool_call = { + "id": response_obj.get("id", ""), + "name": function.get("name", ""), + "call_id": tool_call.get("id", ""), + "type": "function_call", + "arguments": arguments_obj, + } + transformed_tool_calls.append(langfuse_tool_call) + safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, safe_dumps(transformed_tool_calls)) + else: + output_data = {} + if message.get("role"): + output_data["role"] = message.get("role") + if message.get("content") is not None: + output_data["content"] = message.get("content") + if output_data: + safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, safe_dumps(output_data)) + + output = response_obj.get("output", []) + if output: + output_items_data: list[dict] = [] + for item in output: + if hasattr(item, "type"): + item_type = item.type + if item_type == "reasoning" and hasattr(item, "summary"): + for summary in item.summary: + if hasattr(summary, "text"): + output_items_data.append({"role": "reasoning_summary", "content": summary.text}) + elif item_type == "message": + output_items_data.append({ + "role": getattr(item, "role", "assistant"), + "content": getattr(getattr(item, "content", [{}])[0], "text", "") + }) + elif item_type == "function_call": + arguments_str = getattr(item, "arguments", "{}") + arguments_obj = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str + langfuse_tool_call = { + "id": getattr(item, "id", ""), + "name": getattr(item, "name", ""), + "call_id": getattr(item, "call_id", ""), + "type": "function_call", + "arguments": arguments_obj, + } + output_items_data.append(langfuse_tool_call) + if output_items_data: + safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, safe_dumps(output_items_data)) + + @staticmethod + def _set_langfuse_specific_attributes(span: Span, kwargs, response_obj): + """ + Sets Langfuse specific metadata attributes onto the OTEL span. + + All keys supported by the vanilla Langfuse integration are mapped to + OTEL-safe attribute names defined in LangfuseSpanAttributes. Complex + values (lists/dicts) are serialised to JSON strings for OTEL + compatibility. + """ + from litellm.integrations.arize._utils import safe_set_attribute + from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + + langfuse_environment = os.environ.get("LANGFUSE_TRACING_ENVIRONMENT") + if langfuse_environment: + safe_set_attribute(span, LangfuseSpanAttributes.LANGFUSE_ENVIRONMENT.value, langfuse_environment) + + metadata = LangfuseOtelLogger._extract_langfuse_metadata(kwargs) + LangfuseOtelLogger._set_metadata_attributes(span=span, metadata=metadata) + + messages = kwargs.get("messages") + if messages: + safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_INPUT.value, safe_dumps(messages)) + + LangfuseOtelLogger._set_observation_output(span=span, response_obj=response_obj) + @staticmethod def _get_langfuse_otel_host() -> Optional[str]: """ @@ -191,8 +271,7 @@ class LangfuseOtelLogger(OpenTelemetry): verbose_logger.debug(f"Using Langfuse US cloud endpoint: {endpoint}") auth_header = LangfuseOtelLogger._get_langfuse_authorization_header( - public_key=public_key, - secret_key=secret_key + public_key=public_key, secret_key=secret_key ) otlp_auth_headers = f"Authorization={auth_header}" @@ -203,7 +282,7 @@ class LangfuseOtelLogger(OpenTelemetry): return LangfuseOtelConfig( otlp_auth_headers=otlp_auth_headers, protocol="otlp_http" ) - + @staticmethod def _get_langfuse_authorization_header(public_key: str, secret_key: str) -> str: """ @@ -211,11 +290,10 @@ class LangfuseOtelLogger(OpenTelemetry): """ auth_string = f"{public_key}:{secret_key}" auth_header = base64.b64encode(auth_string.encode()).decode() - return f'Basic {auth_header}' - + return f"Basic {auth_header}" + def construct_dynamic_otel_headers( - self, - standard_callback_dynamic_params: StandardCallbackDynamicParams + self, standard_callback_dynamic_params: StandardCallbackDynamicParams ) -> Optional[dict]: """ Construct dynamic Langfuse headers from standard callback dynamic params @@ -227,13 +305,17 @@ class LangfuseOtelLogger(OpenTelemetry): """ dynamic_headers = {} - dynamic_langfuse_public_key = standard_callback_dynamic_params.get("langfuse_public_key") - dynamic_langfuse_secret_key = standard_callback_dynamic_params.get("langfuse_secret_key") + dynamic_langfuse_public_key = standard_callback_dynamic_params.get( + "langfuse_public_key" + ) + dynamic_langfuse_secret_key = standard_callback_dynamic_params.get( + "langfuse_secret_key" + ) if dynamic_langfuse_public_key and dynamic_langfuse_secret_key: auth_header = LangfuseOtelLogger._get_langfuse_authorization_header( public_key=dynamic_langfuse_public_key, - secret_key=dynamic_langfuse_secret_key + secret_key=dynamic_langfuse_secret_key, ) dynamic_headers["Authorization"] = auth_header - + return dynamic_headers diff --git a/litellm/integrations/langfuse/langfuse_otel_attributes.py b/litellm/integrations/langfuse/langfuse_otel_attributes.py new file mode 100644 index 00000000000..fb4a0a6a36c --- /dev/null +++ b/litellm/integrations/langfuse/langfuse_otel_attributes.py @@ -0,0 +1,108 @@ +""" +If the LLM Obs has any specific attributes to log request or response, we can add them here. + +Relevant Issue: https://github.com/BerriAI/litellm/issues/13764 +""" + +import json +from typing import TYPE_CHECKING, Any, Dict, Optional, Union + +from pydantic import BaseModel +from typing_extensions import override + +from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes import ( + BaseLLMObsOTELAttributes, + safe_set_attribute, +) +from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse +from litellm.types.utils import ( + EmbeddingResponse, + ImageResponse, + ModelResponse, + RerankResponse, + TextCompletionResponse, + TranscriptionResponse, +) + +if TYPE_CHECKING: + from opentelemetry.trace import Span + + +def get_output_content_by_type( + response_obj: Union[ + None, + dict, + EmbeddingResponse, + ModelResponse, + TextCompletionResponse, + ImageResponse, + TranscriptionResponse, + RerankResponse, + HttpxBinaryResponseContent, + ResponsesAPIResponse, + list, + ], + kwargs: Optional[Dict[str, Any]] = None, +) -> str: + """ + Extract output content from response objects based on their type. + + This utility function handles the type-specific logic for converting + various response objects into appropriate output formats for Langfuse logging. + + Args: + response_obj: The response object returned by the function + kwargs: Optional keyword arguments containing call_type and other metadata + + Returns: + The formatted output content suitable for Langfuse logging, or None + """ + if response_obj is None: + return "" + + kwargs = kwargs or {} + call_type = kwargs.get("call_type", None) + + # Embedding responses - no output content + if call_type == "embedding" or isinstance(response_obj, EmbeddingResponse): + return "embedding-output" + + # Binary/Speech responses + if isinstance(response_obj, HttpxBinaryResponseContent): + return "speech-output" + + if isinstance(response_obj, BaseModel): + return response_obj.model_dump_json() + + if response_obj and ( + isinstance(response_obj, dict) or isinstance(response_obj, list) + ): + return json.dumps(response_obj) + else: + return "" + + +class LangfuseLLMObsOTELAttributes(BaseLLMObsOTELAttributes): + @staticmethod + @override + def set_messages(span: "Span", kwargs: Dict[str, Any]): + prompt = {"messages": kwargs.get("messages")} + optional_params = kwargs.get("optional_params", {}) + functions = optional_params.get("functions") + tools = optional_params.get("tools") + if functions is not None: + prompt["functions"] = functions + if tools is not None: + prompt["tools"] = tools + + input = prompt + safe_set_attribute(span, "langfuse.observation.input", json.dumps(input)) + + @staticmethod + @override + def set_response_output_messages(span: "Span", response_obj): + safe_set_attribute( + span, + "langfuse.observation.output", + get_output_content_by_type(response_obj), + ) diff --git a/litellm/integrations/mlflow.py b/litellm/integrations/mlflow.py index 86af800d732..b348737868d 100644 --- a/litellm/integrations/mlflow.py +++ b/litellm/integrations/mlflow.py @@ -60,7 +60,10 @@ class MlflowLogger(CustomLogger): inputs = self._construct_input(kwargs) input_messages = inputs.get("messages", []) - output_messages = [c.message.model_dump(exclude_none=True) for c in getattr(response_obj, "choices", [])] + output_messages = [ + c.message.model_dump(exclude_none=True) + for c in getattr(response_obj, "choices", []) + ] if messages := [*input_messages, *output_messages]: set_span_chat_messages(span, messages) if tools := inputs.get("tools"): @@ -184,7 +187,9 @@ class MlflowLogger(CustomLogger): "call_type": kwargs.get("call_type"), "model": kwargs.get("model"), } - standard_obj: Optional[StandardLoggingPayload] = kwargs.get("standard_logging_object") + standard_obj: Optional[StandardLoggingPayload] = kwargs.get( + "standard_logging_object" + ) if standard_obj: attributes.update( { @@ -257,12 +262,25 @@ class MlflowLogger(CustomLogger): span_type=span_type, inputs=inputs, attributes=attributes, - tags=self._transform_tag_list_to_dict(attributes.get("request_tags", [])), + tags=self._transform_tag_list_to_dict( + attributes.get("request_tags", []) + ), start_time_ns=start_time_ns, ) def _transform_tag_list_to_dict(self, tag_list: list) -> dict: - return {tag: "" for tag in tag_list} + """ + Transform a list of colon-separated tags into a dictionary. + Tags without colons are stored with empty string as the value. + """ + tags = {} + for tag in tag_list: + if ":" in tag: + k, v = tag.split(":", 1) + tags[k.strip()] = v.strip() + else: + tags[tag.strip()] = "" + return tags def _end_span_or_trace(self, span, outputs, end_time_ns, status): """End an MLflow span or a trace.""" diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index e825f89f56e..53b7825b3d3 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -10,6 +10,7 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.types.services import ServiceLoggerPayload from litellm.types.utils import ( ChatCompletionMessageToolCall, + CostBreakdown, Function, StandardCallbackDynamicParams, StandardLoggingPayload, @@ -141,7 +142,6 @@ class OpenTelemetry(CustomLogger): meter_provider: Optional[Any] = None, **kwargs, ): - if config is None: config = OpenTelemetryConfig.from_env() @@ -186,9 +186,11 @@ class OpenTelemetry(CustomLogger): ) return - # Add Otel as a service callback - if "otel" not in litellm.service_callback: - litellm.service_callback.append("otel") + # Add self as a service callback + if "otel" not in litellm.service_callback and all( + not isinstance(cb, OpenTelemetry) for cb in litellm.service_callback + ): + litellm.service_callback.append(self) setattr(proxy_server, "open_telemetry_logger", self) def _init_tracing(self, tracer_provider): @@ -198,12 +200,44 @@ class OpenTelemetry(CustomLogger): # use provided tracer or create a new one if tracer_provider is None: - tracer_provider = TracerProvider(resource=_get_litellm_resource()) - # Only add OTLP span processor if we created the tracer provider ourselves - tracer_provider.add_span_processor(self._get_span_processor()) + # Check if a TracerProvider is already set globally (e.g., by Langfuse SDK) + try: + from opentelemetry.trace import ProxyTracerProvider - # register global provider and grab our tracer - trace.set_tracer_provider(tracer_provider) + existing_provider = trace.get_tracer_provider() + + # If an actual provider exists (not the default proxy), use it + if not isinstance(existing_provider, ProxyTracerProvider): + verbose_logger.debug( + "OpenTelemetry: Using existing TracerProvider: %s", + type(existing_provider).__name__, + ) + tracer_provider = existing_provider + # Don't call set_tracer_provider to preserve existing context + else: + # No real provider exists yet, create our own + verbose_logger.debug("OpenTelemetry: Creating new TracerProvider") + tracer_provider = TracerProvider(resource=_get_litellm_resource()) + tracer_provider.add_span_processor(self._get_span_processor()) + trace.set_tracer_provider(tracer_provider) + except Exception as e: + # Fallback: create a new provider if something goes wrong + verbose_logger.debug( + "OpenTelemetry: Exception checking existing provider, creating new one: %s", + str(e), + ) + tracer_provider = TracerProvider(resource=_get_litellm_resource()) + tracer_provider.add_span_processor(self._get_span_processor()) + trace.set_tracer_provider(tracer_provider) + else: + # Tracer provider explicitly provided (e.g., for testing) + verbose_logger.debug( + "OpenTelemetry: Using provided TracerProvider: %s", + type(tracer_provider).__name__, + ) + trace.set_tracer_provider(tracer_provider) + + # grab our tracer self.tracer = trace.get_tracer(LITELLM_TRACER_NAME) self.span_kind = SpanKind @@ -227,8 +261,11 @@ class OpenTelemetry(CustomLogger): PeriodicExportingMetricReader, ) + normalized_endpoint = self._normalize_otel_endpoint( + self.config.endpoint, "metrics" + ) _metric_exporter = OTLPMetricExporter( - endpoint=self.config.endpoint, + endpoint=normalized_endpoint, headers=OpenTelemetry._get_headers_dictionary(self.config.headers), preferred_temporality={Histogram: AggregationTemporality.DELTA}, ) @@ -247,12 +284,12 @@ class OpenTelemetry(CustomLogger): metrics.set_meter_provider(meter_provider) self._operation_duration_histogram = meter.create_histogram( - name="gen_ai.client.operation.duration", # Replace with semconv constant in otel 1.38 + name="gen_ai.client.operation.duration", # Replace with semconv constant in otel 1.38 description="GenAI operation duration", unit="s", ) self._token_usage_histogram = meter.create_histogram( - name="gen_ai.client.token.usage", # Replace with semconv constant in otel 1.38 + name="gen_ai.client.token.usage", # Replace with semconv constant in otel 1.38 description="GenAI token usage", unit="{token}", ) @@ -268,22 +305,20 @@ class OpenTelemetry(CustomLogger): return from opentelemetry._logs import set_logger_provider - from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider from opentelemetry.sdk._logs.export import BatchLogRecordProcessor # set up log pipeline if logger_provider is None: - logger_provider = OTLoggerProvider() + litellm_resource = _get_litellm_resource() + logger_provider = OTLoggerProvider(resource=litellm_resource) # Only add OTLP exporter if we created the logger provider ourselves - logger_provider.add_log_record_processor( - BatchLogRecordProcessor( - OTLPLogExporter( - endpoint=self.config.endpoint, - headers=self._get_headers_dictionary(self.config.headers), - ) + log_exporter = self._get_log_exporter() + if log_exporter: + logger_provider.add_log_record_processor( + BatchLogRecordProcessor(log_exporter) # type: ignore[arg-type] ) - ) + set_logger_provider(logger_provider) def log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -523,7 +558,6 @@ class OpenTelemetry(CustomLogger): ######################################################### def _handle_success(self, kwargs, response_obj, start_time, end_time): - verbose_logger.debug( "OpenTelemetry Logger: Logging kwargs: %s, OTEL config settings=%s", kwargs, @@ -543,7 +577,7 @@ class OpenTelemetry(CustomLogger): # 4. Metrics & cost recording self._record_metrics(kwargs, response_obj, start_time, end_time) - # 5. Semantic logs. + # 5. Semantic logs. if self.config.enable_events: self._emit_semantic_logs(kwargs, response_obj, span) @@ -581,7 +615,6 @@ class OpenTelemetry(CustomLogger): raw_span_name = generation_name if generation_name else RAW_REQUEST_SPAN_NAME - otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs) raw_span = otel_tracer.start_span( name=raw_span_name, @@ -626,6 +659,13 @@ class OpenTelemetry(CustomLogger): if md.get(key) is not None: common_attrs[f"metadata.{key}"] = str(md[key]) + # get hidden params + hidden_params = getattr(std_log, "hidden_params", None) or (std_log or {}).get( + "hidden_params", {} + ) + if hidden_params: + common_attrs["hidden_params"] = safe_dumps(hidden_params) + if self._operation_duration_histogram: self._operation_duration_histogram.record( duration_s, attributes=common_attrs @@ -652,9 +692,17 @@ class OpenTelemetry(CustomLogger): if not self.config.enable_events: return - from opentelemetry._logs import LogRecord, get_logger + from opentelemetry._logs import SeverityNumber, get_logger, get_logger_provider + from opentelemetry.sdk._logs import LogRecord as SdkLogRecord + otel_logger = get_logger(LITELLM_LOGGER_NAME) + # Get the resource from the logger provider + logger_provider = get_logger_provider() + resource = ( + getattr(logger_provider, "_resource", None) or _get_litellm_resource() + ) + parent_ctx = span.get_span_context() provider = (kwargs.get("litellm_params") or {}).get( "custom_llm_provider", "Unknown" @@ -669,15 +717,18 @@ class OpenTelemetry(CustomLogger): if self.message_logging and msg.get("content"): attrs["gen_ai.prompt"] = msg["content"] - otel_logger.emit( - LogRecord( - attributes=attrs, - body=msg.copy(), - trace_id=parent_ctx.trace_id, - span_id=parent_ctx.span_id, - trace_flags=parent_ctx.trace_flags, - ) + log_record = SdkLogRecord( + timestamp=self._to_ns(datetime.now()), + trace_id=parent_ctx.trace_id, + span_id=parent_ctx.span_id, + trace_flags=parent_ctx.trace_flags, + severity_number=SeverityNumber.INFO, + severity_text="INFO", + body=msg.copy(), + resource=resource, + attributes=attrs, ) + otel_logger.emit(log_record) # per-choice events for idx, choice in enumerate(response_obj.get("choices", [])): @@ -698,16 +749,18 @@ class OpenTelemetry(CustomLogger): if self.message_logging and body_msg.get("content"): body["message"]["content"] = body_msg["content"] - otel_logger.emit( - LogRecord( - attributes=attrs, - body=body, - trace_id=parent_ctx.trace_id, - span_id=parent_ctx.span_id, - trace_flags=parent_ctx.trace_flags, - ) + log_record = SdkLogRecord( + timestamp=self._to_ns(datetime.now()), + trace_id=parent_ctx.trace_id, + span_id=parent_ctx.span_id, + trace_flags=parent_ctx.trace_flags, + severity_number=SeverityNumber.INFO, + severity_text="INFO", + body=body, + resource=resource, + attributes=attrs, ) - + otel_logger.emit(log_record) def _create_guardrail_span( self, kwargs: Optional[dict], context: Optional[Context] @@ -723,52 +776,63 @@ class OpenTelemetry(CustomLogger): if standard_logging_payload is None: return - guardrail_information = standard_logging_payload.get("guardrail_information") - if guardrail_information is None: + guardrail_information_data = standard_logging_payload.get( + "guardrail_information" + ) + if not guardrail_information_data: return - start_time_float = guardrail_information.get("start_time") - end_time_float = guardrail_information.get("end_time") - start_time_datetime = datetime.now() - if start_time_float is not None: - start_time_datetime = datetime.fromtimestamp(start_time_float) - end_time_datetime = datetime.now() - if end_time_float is not None: - end_time_datetime = datetime.fromtimestamp(end_time_float) + guardrail_information_list = [ + information + for information in guardrail_information_data + if isinstance(information, dict) + ] + + if not guardrail_information_list: + return otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs) - guardrail_span = otel_tracer.start_span( - name="guardrail", - start_time=self._to_ns(start_time_datetime), - context=context, - ) + for guardrail_information in guardrail_information_list: + start_time_float = guardrail_information.get("start_time") + end_time_float = guardrail_information.get("end_time") + start_time_datetime = datetime.now() + if start_time_float is not None: + start_time_datetime = datetime.fromtimestamp(start_time_float) + end_time_datetime = datetime.now() + if end_time_float is not None: + end_time_datetime = datetime.fromtimestamp(end_time_float) - self.safe_set_attribute( - span=guardrail_span, - key="guardrail_name", - value=guardrail_information.get("guardrail_name"), - ) - - self.safe_set_attribute( - span=guardrail_span, - key="guardrail_mode", - value=guardrail_information.get("guardrail_mode"), - ) - - # Set masked_entity_count directly without conversion - masked_entity_count = guardrail_information.get("masked_entity_count") - if masked_entity_count is not None: - guardrail_span.set_attribute( - "masked_entity_count", safe_dumps(masked_entity_count) + guardrail_span = otel_tracer.start_span( + name="guardrail", + start_time=self._to_ns(start_time_datetime), + context=context, ) - self.safe_set_attribute( - span=guardrail_span, - key="guardrail_response", - value=guardrail_information.get("guardrail_response"), - ) + self.safe_set_attribute( + span=guardrail_span, + key="guardrail_name", + value=guardrail_information.get("guardrail_name"), + ) - guardrail_span.end(end_time=self._to_ns(end_time_datetime)) + self.safe_set_attribute( + span=guardrail_span, + key="guardrail_mode", + value=guardrail_information.get("guardrail_mode"), + ) + + masked_entity_count = guardrail_information.get("masked_entity_count") + if masked_entity_count is not None: + guardrail_span.set_attribute( + "masked_entity_count", safe_dumps(masked_entity_count) + ) + + self.safe_set_attribute( + span=guardrail_span, + key="guardrail_response", + value=guardrail_information.get("guardrail_response"), + ) + + guardrail_span.end(end_time=self._to_ns(end_time_datetime)) def _handle_failure(self, kwargs, response_obj, start_time, end_time): from opentelemetry.trace import Status, StatusCode @@ -789,6 +853,10 @@ class OpenTelemetry(CustomLogger): ) span.set_status(Status(StatusCode.ERROR)) self.set_attributes(span, kwargs, response_obj) + + # Record exception information using OTEL standard method + self._record_exception_on_span(span=span, kwargs=kwargs) + span.end(end_time=self._to_ns(end_time)) # Create span for guardrail information @@ -797,6 +865,87 @@ class OpenTelemetry(CustomLogger): if parent_otel_span is not None: parent_otel_span.end(end_time=self._to_ns(datetime.now())) + def _record_exception_on_span(self, span: Span, kwargs: dict): + """ + Record exception information on the span using OTEL standard methods. + + This extracts error information from StandardLoggingPayload and: + 1. Uses span.record_exception() for the actual exception object (OTEL standard) + 2. Sets structured error attributes from StandardLoggingPayloadErrorInformation + """ + try: + from litellm.integrations._types.open_inference import ErrorAttributes + + # Get the exception object if available + exception = kwargs.get("exception") + + # Record the exception using OTEL's standard method + if exception is not None: + span.record_exception(exception) + + # Get StandardLoggingPayload for structured error information + standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get( + "standard_logging_object" + ) + + if standard_logging_payload is None: + return + + # Extract error_information from StandardLoggingPayload + error_information = standard_logging_payload.get("error_information") + + if error_information is None: + # Fallback to error_str if error_information is not available + error_str = standard_logging_payload.get("error_str") + if error_str: + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_MESSAGE, + value=error_str, + ) + return + + # Set structured error attributes from StandardLoggingPayloadErrorInformation + if error_information.get("error_code"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_CODE, + value=error_information["error_code"], + ) + + if error_information.get("error_class"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_TYPE, + value=error_information["error_class"], + ) + + if error_information.get("error_message"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_MESSAGE, + value=error_information["error_message"], + ) + + if error_information.get("llm_provider"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_LLM_PROVIDER, + value=error_information["llm_provider"], + ) + + if error_information.get("traceback"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_STACK_TRACE, + value=error_information["traceback"], + ) + + except Exception as e: + verbose_logger.exception( + "OpenTelemetry: Error recording exception on span: %s", str(e) + ) + def set_tools_attributes(self, span: Span, tools): import json @@ -920,6 +1069,24 @@ class OpenTelemetry(CustomLogger): span=span, key="metadata.{}".format(key), value=value ) + # get hidden params + hidden_params = getattr( + standard_logging_payload, "hidden_params", None + ) or (standard_logging_payload or {}).get("hidden_params", {}) + if hidden_params: + self.safe_set_attribute( + span=span, key="hidden_params", value=safe_dumps(hidden_params) + ) + # Cost breakdown tracking + cost_breakdown: Optional[CostBreakdown] = standard_logging_payload.get("cost_breakdown") + if cost_breakdown: + for key, value in cost_breakdown.items(): + if value is not None: + self.safe_set_attribute( + span=span, + key=f"gen_ai.cost.{key}", + value=value, + ) ############################################# ########## LLM Request Attributes ########### ############################################# @@ -1204,7 +1371,7 @@ class OpenTelemetry(CustomLogger): return _parent_context def _get_span_context(self, kwargs): - from opentelemetry import trace + from opentelemetry import context, trace from opentelemetry.trace.propagation.tracecontext import ( TraceContextTextMapPropagator, ) @@ -1216,20 +1383,46 @@ class OpenTelemetry(CustomLogger): _metadata = litellm_params.get("metadata", {}) or {} parent_otel_span = _metadata.get("litellm_parent_otel_span", None) - """ - Two way to use parents in opentelemetry - - using the traceparent header - - using the parent_otel_span in the [metadata][parent_otel_span] - """ + # Priority 1: Explicit parent span from metadata if parent_otel_span is not None: + verbose_logger.debug( + "OpenTelemetry: Using explicit parent span from metadata" + ) return trace.set_span_in_context(parent_otel_span), parent_otel_span - if traceparent is None: - return None, None - else: + # Priority 2: HTTP traceparent header + if traceparent is not None: + verbose_logger.debug( + "OpenTelemetry: Using traceparent header for context propagation" + ) carrier = {"traceparent": traceparent} return TraceContextTextMapPropagator().extract(carrier=carrier), None + # Priority 3: Active span from global context (auto-detection) + try: + current_span = trace.get_current_span() + if current_span is not None: + span_context = current_span.get_span_context() + if span_context.is_valid: + verbose_logger.debug( + "OpenTelemetry: Using active span from global context: %s (trace_id=%s, span_id=%s, is_recording=%s)", + current_span, + format(span_context.trace_id, "032x"), + format(span_context.span_id, "016x"), + current_span.is_recording(), + ) + return context.get_current(), current_span + except Exception as e: + verbose_logger.debug( + "OpenTelemetry: Error getting current span: %s", str(e) + ) + + # Priority 4: No parent context + verbose_logger.debug( + "OpenTelemetry: No parent context found, creating root span" + ) + return None, None + def _get_span_processor(self, dynamic_headers: Optional[dict] = None): from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( OTLPSpanExporter as OTLPSpanExporterGRPC, @@ -1278,9 +1471,12 @@ class OpenTelemetry(CustomLogger): "OpenTelemetry: intiializing http exporter. Value of OTEL_EXPORTER: %s", self.OTEL_EXPORTER, ) + normalized_endpoint = self._normalize_otel_endpoint( + self.OTEL_ENDPOINT, "traces" + ) return BatchSpanProcessor( OTLPSpanExporterHTTP( - endpoint=self.OTEL_ENDPOINT, headers=_split_otel_headers + endpoint=normalized_endpoint, headers=_split_otel_headers ), ) elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc": @@ -1288,9 +1484,12 @@ class OpenTelemetry(CustomLogger): "OpenTelemetry: intiializing grpc exporter. Value of OTEL_EXPORTER: %s", self.OTEL_EXPORTER, ) + normalized_endpoint = self._normalize_otel_endpoint( + self.OTEL_ENDPOINT, "traces" + ) return BatchSpanProcessor( OTLPSpanExporterGRPC( - endpoint=self.OTEL_ENDPOINT, headers=_split_otel_headers + endpoint=normalized_endpoint, headers=_split_otel_headers ), ) else: @@ -1300,6 +1499,151 @@ class OpenTelemetry(CustomLogger): ) return BatchSpanProcessor(ConsoleSpanExporter()) + def _get_log_exporter(self): + """ + Get the appropriate log exporter based on the configuration. + """ + verbose_logger.debug( + "OpenTelemetry Logger, initializing log exporter \nself.OTEL_EXPORTER: %s\nself.OTEL_ENDPOINT: %s\nself.OTEL_HEADERS: %s", + self.OTEL_EXPORTER, + self.OTEL_ENDPOINT, + self.OTEL_HEADERS, + ) + + _split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS) + + # Normalize endpoint for logs - ensure it points to /v1/logs instead of /v1/traces + normalized_endpoint = self._normalize_otel_endpoint(self.OTEL_ENDPOINT, "logs") + + verbose_logger.debug( + "OpenTelemetry: Log endpoint normalized from %s to %s", + self.OTEL_ENDPOINT, + normalized_endpoint, + ) + + if hasattr(self.OTEL_EXPORTER, "export"): + # Custom exporter provided + verbose_logger.debug( + "OpenTelemetry: Using custom log exporter. Value of OTEL_EXPORTER: %s", + self.OTEL_EXPORTER, + ) + return self.OTEL_EXPORTER + + if self.OTEL_EXPORTER == "console": + from opentelemetry.sdk._logs.export import ConsoleLogExporter + + verbose_logger.debug( + "OpenTelemetry: Using console log exporter. Value of OTEL_EXPORTER: %s", + self.OTEL_EXPORTER, + ) + return ConsoleLogExporter() + elif ( + self.OTEL_EXPORTER == "otlp_http" + or self.OTEL_EXPORTER == "http/protobuf" + or self.OTEL_EXPORTER == "http/json" + ): + from opentelemetry.exporter.otlp.proto.http._log_exporter import ( + OTLPLogExporter, + ) + + verbose_logger.debug( + "OpenTelemetry: Using HTTP log exporter. Value of OTEL_EXPORTER: %s, endpoint: %s", + self.OTEL_EXPORTER, + normalized_endpoint, + ) + return OTLPLogExporter( + endpoint=normalized_endpoint, headers=_split_otel_headers + ) + elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc": + from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter, + ) + + verbose_logger.debug( + "OpenTelemetry: Using gRPC log exporter. Value of OTEL_EXPORTER: %s, endpoint: %s", + self.OTEL_EXPORTER, + normalized_endpoint, + ) + return OTLPLogExporter( + endpoint=normalized_endpoint, headers=_split_otel_headers + ) + else: + verbose_logger.warning( + "OpenTelemetry: Unknown log exporter '%s', defaulting to console. Supported: console, otlp_http, otlp_grpc", + self.OTEL_EXPORTER, + ) + from opentelemetry.sdk._logs.export import ConsoleLogExporter + + return ConsoleLogExporter() + + def _normalize_otel_endpoint( + self, endpoint: Optional[str], signal_type: str + ) -> Optional[str]: + """ + Normalize the endpoint URL for a specific OpenTelemetry signal type. + + The OTLP exporters expect endpoints to use signal-specific paths: + - traces: /v1/traces + - metrics: /v1/metrics + - logs: /v1/logs + + This method ensures the endpoint has the correct path for the given signal type. + + Args: + endpoint: The endpoint URL to normalize + signal_type: The telemetry signal type ('traces', 'metrics', or 'logs') + + Returns: + Normalized endpoint URL with the correct signal path + + Examples: + _normalize_otel_endpoint("http://collector:4318/v1/traces", "logs") + -> "http://collector:4318/v1/logs" + + _normalize_otel_endpoint("http://collector:4318", "traces") + -> "http://collector:4318/v1/traces" + + _normalize_otel_endpoint("http://collector:4318/v1/logs", "metrics") + -> "http://collector:4318/v1/metrics" + """ + if not endpoint: + return endpoint + + # Validate signal_type + valid_signals = {"traces", "metrics", "logs"} + if signal_type not in valid_signals: + verbose_logger.warning( + "Invalid signal_type '%s' provided to _normalize_otel_endpoint. " + "Valid values: %s. Returning endpoint unchanged.", + signal_type, + valid_signals, + ) + return endpoint + + # Remove trailing slash + endpoint = endpoint.rstrip("/") + + # Check if endpoint already ends with the correct signal path + target_path = f"/v1/{signal_type}" + if endpoint.endswith(target_path): + return endpoint + + # Replace existing signal path with the target signal path + other_signals = valid_signals - {signal_type} + for other_signal in other_signals: + other_path = f"/v1/{other_signal}" + if endpoint.endswith(other_path): + endpoint = endpoint.rsplit("/", 1)[0] + f"/{signal_type}" + return endpoint + + # No existing signal path found, append the target path + if not endpoint.endswith("/v1"): + endpoint = endpoint + target_path + else: + endpoint = endpoint + f"/{signal_type}" + + return endpoint + @staticmethod def _get_headers_dictionary(headers: Optional[Union[str, dict]]) -> Dict[str, str]: """ @@ -1310,11 +1654,10 @@ class OpenTelemetry(CustomLogger): if isinstance(headers, str): # when passed HEADERS="x-honeycomb-team=B85YgLm96******" # Split only on first '=' occurrence - parts = headers.split("=", 1) - if len(parts) == 2: - _split_otel_headers = {parts[0]: parts[1]} - else: - _split_otel_headers = {} + parts = headers.split(",") + for part in parts: + key, value = part.split("=", 1) + _split_otel_headers[key] = value elif isinstance(headers, dict): _split_otel_headers = headers return _split_otel_headers diff --git a/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py b/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py new file mode 100644 index 00000000000..f74da8231f3 --- /dev/null +++ b/litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py @@ -0,0 +1,37 @@ +from abc import ABC +from typing import TYPE_CHECKING, Any, Dict, Union + +if TYPE_CHECKING: + from opentelemetry.trace import Span + + +class BaseLLMObsOTELAttributes(ABC): + @staticmethod + def set_messages(span: "Span", kwargs: Dict[str, Any]): + pass + + @staticmethod + def set_response_output_messages(span: "Span", response_obj): + pass + + +def cast_as_primitive_value_type(value) -> Union[str, bool, int, float]: + """ + Converts a value to an OTEL-supported primitive for Arize/Phoenix observability. + """ + if value is None: + return "" + if isinstance(value, (str, bool, int, float)): + return value + try: + return str(value) + except Exception: + return "" + + +def safe_set_attribute(span: "Span", key: str, value: Any): + """ + Sets a span attribute safely with OTEL-compliant primitive typing for Arize/Phoenix. + """ + primitive_value = cast_as_primitive_value_type(value) + span.set_attribute(key, primitive_value) diff --git a/litellm/integrations/opik/opik.py b/litellm/integrations/opik/opik.py index 9fa3482f663..7b687d34d1c 100644 --- a/litellm/integrations/opik/opik.py +++ b/litellm/integrations/opik/opik.py @@ -3,10 +3,9 @@ Opik Logger that logs LLM events to an Opik server """ import asyncio -from datetime import timezone -import json import traceback -from typing import Dict, List +from datetime import datetime +from typing import Any, Dict, Optional from litellm._logging import verbose_logger from litellm.integrations.custom_batch_logger import CustomBatchLogger @@ -16,12 +15,22 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) -from .utils import ( - create_usage_object, - create_uuid7, - get_opik_config_variable, - get_traces_and_spans_from_payload, -) +from . import opik_payload_builder, utils + +try: + from opik.api_objects import opik_client +except Exception: + opik_client = None + + +def _should_skip_event(kwargs: Dict[str, Any]) -> bool: + """Check if event should be skipped due to missing standard_logging_object.""" + if kwargs.get("standard_logging_object") is None: + verbose_logger.debug( + "OpikLogger skipping event; no standard_logging_object found" + ) + return True + return False class OpikLogger(CustomBatchLogger): @@ -29,76 +38,140 @@ class OpikLogger(CustomBatchLogger): Opik Logger for logging events to an Opik Server """ - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any) -> None: self.async_httpx_client = get_async_httpx_client( llm_provider=httpxSpecialProvider.LoggingCallback ) self.sync_httpx_client = _get_httpx_client() - self.opik_project_name = get_opik_config_variable( - "project_name", - user_value=kwargs.get("project_name", None), - default_value="Default Project", + self.opik_project_name: str = ( + utils.get_opik_config_variable( + "project_name", + user_value=kwargs.get("project_name", None), + default_value="Default Project", + ) + or "Default Project" ) - opik_base_url = get_opik_config_variable( - "url_override", - user_value=kwargs.get("url", None), - default_value="https://www.comet.com/opik/api", + opik_base_url: str = ( + utils.get_opik_config_variable( + "url_override", + user_value=kwargs.get("url", None), + default_value="https://www.comet.com/opik/api", + ) + or "https://www.comet.com/opik/api" ) - opik_api_key = get_opik_config_variable( + opik_api_key: Optional[str] = utils.get_opik_config_variable( "api_key", user_value=kwargs.get("api_key", None), default_value=None ) - opik_workspace = get_opik_config_variable( + opik_workspace: Optional[str] = utils.get_opik_config_variable( "workspace", user_value=kwargs.get("workspace", None), default_value=None ) - self.trace_url = f"{opik_base_url}/v1/private/traces/batch" - self.span_url = f"{opik_base_url}/v1/private/spans/batch" + self.trace_url: str = f"{opik_base_url}/v1/private/traces/batch" + self.span_url: str = f"{opik_base_url}/v1/private/spans/batch" - self.headers = {} + self.headers: Dict[str, str] = {} if opik_workspace: self.headers["Comet-Workspace"] = opik_workspace if opik_api_key: self.headers["authorization"] = opik_api_key - self.opik_workspace = opik_workspace - self.opik_api_key = opik_api_key + self.opik_workspace: Optional[str] = opik_workspace + self.opik_api_key: Optional[str] = opik_api_key try: asyncio.create_task(self.periodic_flush()) - self.flush_lock = asyncio.Lock() + self.flush_lock: Optional[asyncio.Lock] = asyncio.Lock() except Exception as e: verbose_logger.exception( f"OpikLogger - Asynchronous processing not initialized as we are not running in an async context {str(e)}" ) self.flush_lock = None + # Initialize _opik_client attribute + if opik_client is not None: + self._opik_client = opik_client.get_client_cached() + else: + self._opik_client = None + super().__init__(**kwargs, flush_lock=self.flush_lock) - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_success_event( + self, + kwargs: Dict[str, Any], + response_obj: Any, + start_time: datetime, + end_time: datetime, + ) -> None: try: - opik_payload = self._create_opik_payload( + if _should_skip_event(kwargs): + return + + # Build payload using the payload builder + trace_payload, span_payload = opik_payload_builder.build_opik_payload( kwargs=kwargs, response_obj=response_obj, start_time=start_time, end_time=end_time, + project_name=self.opik_project_name, ) - self.log_queue.extend(opik_payload) - verbose_logger.debug( - f"OpikLogger added event to log_queue - Will flush in {self.flush_interval} seconds..." - ) + if self._opik_client is not None: + # Opik native client is available, use it to send data + if trace_payload is not None: + self._opik_client.trace( + id=trace_payload.id, + name=trace_payload.name, + start_time=datetime.fromisoformat(trace_payload.start_time), + end_time=datetime.fromisoformat(trace_payload.end_time), + input=trace_payload.input, + output=trace_payload.output, + metadata=trace_payload.metadata, + tags=trace_payload.tags, + thread_id=trace_payload.thread_id, + project_name=trace_payload.project_name, + ) - if len(self.log_queue) >= self.batch_size: - verbose_logger.debug("OpikLogger - Flushing batch") - await self.flush_queue() + self._opik_client.span( + id=span_payload.id, + trace_id=span_payload.trace_id, + parent_span_id=span_payload.parent_span_id, + name=span_payload.name, + type=span_payload.type, + model=span_payload.model, + start_time=datetime.fromisoformat(span_payload.start_time), + end_time=datetime.fromisoformat(span_payload.end_time), + input=span_payload.input, + output=span_payload.output, + metadata=span_payload.metadata, + tags=span_payload.tags, + usage=span_payload.usage, + project_name=span_payload.project_name, + provider=span_payload.provider, + total_cost=span_payload.total_cost, + ) + else: + # Add payloads to LiteLLM queue + if trace_payload is not None: + self.log_queue.append(trace_payload.__dict__) + self.log_queue.append(span_payload.__dict__) + + verbose_logger.debug( + f"OpikLogger added event to log_queue - Will flush in {self.flush_interval} seconds..." + ) + + if len(self.log_queue) >= self.batch_size: + verbose_logger.debug("OpikLogger - Flushing batch") + await self.flush_queue() except Exception as e: verbose_logger.exception( f"OpikLogger failed to log success event - {str(e)}\n{traceback.format_exc()}" ) - def _sync_send(self, url: str, headers: Dict[str, str], batch: Dict): + def _sync_send( + self, url: str, headers: Dict[str, str], batch: Dict[str, Any] + ) -> None: try: response = self.sync_httpx_client.post( url=url, headers=headers, json=batch # type: ignore @@ -113,30 +186,82 @@ class OpikLogger(CustomBatchLogger): f"OpikLogger failed to send batch - {str(e)}\n{traceback.format_exc()}" ) - def log_success_event(self, kwargs, response_obj, start_time, end_time): + def log_success_event( + self, + kwargs: Dict[str, Any], + response_obj: Any, + start_time: datetime, + end_time: datetime, + ) -> None: try: - opik_payload = self._create_opik_payload( + if _should_skip_event(kwargs): + return + + # Build payload using the payload builder + trace_payload, span_payload = opik_payload_builder.build_opik_payload( kwargs=kwargs, response_obj=response_obj, start_time=start_time, end_time=end_time, + project_name=self.opik_project_name, ) + if self._opik_client is not None: + # Opik native client is available, use it to send data + if trace_payload is not None: + self._opik_client.trace( + id=trace_payload.id, + name=trace_payload.name, + start_time=datetime.fromisoformat(trace_payload.start_time), + end_time=datetime.fromisoformat(trace_payload.end_time), + input=trace_payload.input, + output=trace_payload.output, + metadata=trace_payload.metadata, + tags=trace_payload.tags, + thread_id=trace_payload.thread_id, + project_name=trace_payload.project_name, + ) - traces, spans = get_traces_and_spans_from_payload(opik_payload) - if len(traces) > 0: - self._sync_send( - url=self.trace_url, headers=self.headers, batch={"traces": traces} + self._opik_client.span( + id=span_payload.id, + trace_id=span_payload.trace_id, + parent_span_id=span_payload.parent_span_id, + name=span_payload.name, + type=span_payload.type, + model=span_payload.model, + start_time=datetime.fromisoformat(span_payload.start_time), + end_time=datetime.fromisoformat(span_payload.end_time), + input=span_payload.input, + output=span_payload.output, + metadata=span_payload.metadata, + tags=span_payload.tags, + usage=span_payload.usage, + project_name=span_payload.project_name, + provider=span_payload.provider, + total_cost=span_payload.total_cost, ) - if len(spans) > 0: + else: + # Opik native client is not available, use LiteLLM queue to send data + if trace_payload is not None: + self._sync_send( + url=self.trace_url, + headers=self.headers, + batch={"traces": [trace_payload.__dict__]}, + ) + + # Always send span self._sync_send( - url=self.span_url, headers=self.headers, batch={"spans": spans} + url=self.span_url, + headers=self.headers, + batch={"spans": [span_payload.__dict__]}, ) except Exception as e: verbose_logger.exception( f"OpikLogger failed to log success event - {str(e)}\n{traceback.format_exc()}" ) - async def _submit_batch(self, url: str, headers: Dict[str, str], batch: Dict): + async def _submit_batch( + self, url: str, headers: Dict[str, str], batch: Dict[str, Any] + ) -> None: try: response = await self.async_httpx_client.post( url=url, headers=headers, json=batch # type: ignore @@ -154,8 +279,8 @@ class OpikLogger(CustomBatchLogger): except Exception as e: verbose_logger.exception(f"OpikLogger failed to send batch - {str(e)}") - def _create_opik_headers(self): - headers = {} + def _create_opik_headers(self) -> Dict[str, str]: + headers: Dict[str, str] = {} if self.opik_workspace: headers["Comet-Workspace"] = self.opik_workspace @@ -163,13 +288,13 @@ class OpikLogger(CustomBatchLogger): headers["authorization"] = self.opik_api_key return headers - async def async_send_batch(self): + async def async_send_batch(self) -> None: verbose_logger.info("Calling async_send_batch") if not self.log_queue: return # Split the log_queue into traces and spans - traces, spans = get_traces_and_spans_from_payload(self.log_queue) + traces, spans = utils.get_traces_and_spans_from_payload(self.log_queue) # Send trace batch if len(traces) > 0: @@ -182,176 +307,3 @@ class OpikLogger(CustomBatchLogger): url=self.span_url, headers=self.headers, batch={"spans": spans} ) verbose_logger.info(f"Sent {len(spans)} spans") - - def _create_opik_payload( # noqa: PLR0915 - self, kwargs, response_obj, start_time, end_time - ) -> List[Dict]: - # Get metadata - _litellm_params = kwargs.get("litellm_params", {}) or {} - litellm_params_metadata = _litellm_params.get("metadata", {}) or {} - - # Extract opik metadata - litellm_opik_metadata = litellm_params_metadata.get("opik", {}) - - # Use standard_logging_object to create metadata and input/output data - standard_logging_object = kwargs.get("standard_logging_object", None) - if standard_logging_object is None: - verbose_logger.debug( - "OpikLogger skipping event; no standard_logging_object found" - ) - return [] - - # Update litellm_opik_metadata with opik metadata from requester - standard_logging_metadata = standard_logging_object.get("metadata", {}) or {} - requester_metadata = standard_logging_metadata.get("requester_metadata", {}) or {} - requester_opik_metadata = requester_metadata.get("opik", {}) or {} - litellm_opik_metadata.update(requester_opik_metadata) - - verbose_logger.debug( - f"litellm_opik_metadata - {json.dumps(litellm_opik_metadata, default=str)}" - ) - - project_name = litellm_opik_metadata.get("project_name", self.opik_project_name) - - # Extract trace_id and parent_span_id - current_span_data = litellm_opik_metadata.get("current_span_data", None) - if isinstance(current_span_data, dict): - trace_id = current_span_data.get("trace_id", None) - parent_span_id = current_span_data.get("id", None) - elif current_span_data: - trace_id = current_span_data.trace_id - parent_span_id = current_span_data.id - else: - trace_id = None - parent_span_id = None - - # Create Opik tags - opik_tags = litellm_opik_metadata.get("tags", []) - if kwargs.get("custom_llm_provider"): - opik_tags.append(kwargs["custom_llm_provider"]) - - # Get thread_id if present - thread_id = litellm_opik_metadata.get("thread_id", None) - - # Override with any opik_ headers from proxy request - proxy_server_request = _litellm_params.get("proxy_server_request", {}) or {} - proxy_headers = proxy_server_request.get("headers", {}) or {} - for key, value in proxy_headers.items(): - if key.startswith("opik_"): - param_key = key.replace("opik_", "", 1) - if param_key == "project_name" and value: - project_name = value - elif param_key == "thread_id" and value: - thread_id = value - elif param_key == "tags" and value: - try: - parsed_tags = json.loads(value) - if isinstance(parsed_tags, list): - opik_tags.extend(parsed_tags) - except (json.JSONDecodeError, TypeError): - pass - - # Create input and output data - input_data = standard_logging_object.get("messages", {}) - output_data = standard_logging_object.get("response", {}) - - # Create usage object - usage = create_usage_object(response_obj["usage"]) - - # Define span and trace names - span_name = "%s_%s_%s" % ( - response_obj.get("model", "unknown-model"), - response_obj.get("object", "unknown-object"), - response_obj.get("created", 0), - ) - trace_name = response_obj.get("object", "unknown type") - - # Create metadata object, we add the opik metadata first and then - # update it with the standard_logging_object metadata - metadata = litellm_opik_metadata - if "current_span_data" in metadata: - del metadata["current_span_data"] - metadata["created_from"] = "litellm" - - metadata.update(standard_logging_metadata) - if "call_type" in standard_logging_object: - metadata["type"] = standard_logging_object["call_type"] - if "status" in standard_logging_object: - metadata["status"] = standard_logging_object["status"] - if "response_cost" in kwargs: - metadata["cost"] = { - "total_tokens": kwargs["response_cost"], - "currency": "USD", - } - if "response_cost_failure_debug_info" in kwargs: - metadata["response_cost_failure_debug_info"] = kwargs[ - "response_cost_failure_debug_info" - ] - if "model_map_information" in standard_logging_object: - metadata["model_map_information"] = standard_logging_object[ - "model_map_information" - ] - if "model" in standard_logging_object: - metadata["model"] = standard_logging_object["model"] - if "model_id" in standard_logging_object: - metadata["model_id"] = standard_logging_object["model_id"] - if "model_group" in standard_logging_object: - metadata["model_group"] = standard_logging_object["model_group"] - if "api_base" in standard_logging_object: - metadata["api_base"] = standard_logging_object["api_base"] - if "cache_hit" in standard_logging_object: - metadata["cache_hit"] = standard_logging_object["cache_hit"] - if "saved_cache_cost" in standard_logging_object: - metadata["saved_cache_cost"] = standard_logging_object["saved_cache_cost"] - if "error_str" in standard_logging_object: - metadata["error_str"] = standard_logging_object["error_str"] - if "model_parameters" in standard_logging_object: - metadata["model_parameters"] = standard_logging_object["model_parameters"] - if "hidden_params" in standard_logging_object: - metadata["hidden_params"] = standard_logging_object["hidden_params"] - - payload = [] - if trace_id is None: - trace_id = create_uuid7() - verbose_logger.debug( - f"OpikLogger creating payload for trace with id {trace_id}" - ) - payload.append( - { - "project_name": project_name, - "id": trace_id, - "name": trace_name, - "start_time": start_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), - "end_time": end_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), - "input": input_data, - "output": output_data, - "metadata": metadata, - "tags": opik_tags, - "thread_id": thread_id, - } - ) - - span_id = create_uuid7() - verbose_logger.debug( - f"OpikLogger creating payload for trace with id {trace_id} and span with id {span_id}" - ) - payload.append( - { - "id": span_id, - "project_name": project_name, - "trace_id": trace_id, - "parent_span_id": parent_span_id, - "name": span_name, - "type": "llm", - "start_time": start_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), - "end_time": end_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), - "input": input_data, - "output": output_data, - "metadata": metadata, - "tags": opik_tags, - "thread_id": thread_id, - "usage": usage, - } - ) - verbose_logger.debug(f"Payload: {payload}") - return payload diff --git a/litellm/integrations/opik/opik_payload_builder/__init__.py b/litellm/integrations/opik/opik_payload_builder/__init__.py new file mode 100644 index 00000000000..c57fceaa110 --- /dev/null +++ b/litellm/integrations/opik/opik_payload_builder/__init__.py @@ -0,0 +1,10 @@ +""" +Opik payload builder namespace. + +Public API: + build_opik_payload - Main function to create Opik trace and span payloads +""" + +from .api import build_opik_payload + +__all__ = ["build_opik_payload"] diff --git a/litellm/integrations/opik/opik_payload_builder/api.py b/litellm/integrations/opik/opik_payload_builder/api.py new file mode 100644 index 00000000000..99dbea165e9 --- /dev/null +++ b/litellm/integrations/opik/opik_payload_builder/api.py @@ -0,0 +1,121 @@ +"""Public API for Opik payload building.""" + +from datetime import datetime +from typing import Any, Dict, Optional, Tuple + +from litellm.integrations.opik import utils + +from . import extractors, payload_builders, types + + +def build_opik_payload( + kwargs: Dict[str, Any], + response_obj: Dict[str, Any], + start_time: datetime, + end_time: datetime, + project_name: str, +) -> Tuple[Optional[types.TracePayload], types.SpanPayload]: + """ + Build Opik trace and span payloads from LiteLLM completion data. + + This is the main public API for creating Opik payloads. It: + 1. Extracts all necessary data from LiteLLM kwargs and response + 2. Decides whether to create a new trace or attach to existing + 3. Builds trace payload (if new trace) + 4. Builds span payload (always) + + Args: + kwargs: LiteLLM kwargs containing request metadata and logging data + response_obj: LiteLLM response object containing model response + start_time: Request start time + end_time: Request end time + project_name: Default Opik project name + + Returns: + Tuple of (optional trace payload, span payload) + - First element is TracePayload if creating a new trace, None if attaching to existing + - Second element is always SpanPayload + """ + standard_logging_object = kwargs["standard_logging_object"] + + # Extract litellm params and metadata + litellm_params = kwargs.get("litellm_params", {}) or {} + litellm_metadata = litellm_params.get("metadata", {}) or {} + standard_logging_metadata = standard_logging_object.get("metadata", {}) or {} + + # Extract and merge Opik metadata + opik_metadata = extractors.extract_opik_metadata( + litellm_metadata, standard_logging_metadata + ) + + # Extract project name + current_project_name = opik_metadata.get("project_name", project_name) + + # Extract trace identifiers + current_span_data = opik_metadata.get("current_span_data") + trace_id, parent_span_id = extractors.extract_span_identifiers(current_span_data) + + # Extract tags and thread_id + tags = extractors.extract_tags(opik_metadata, kwargs.get("custom_llm_provider")) + thread_id = opik_metadata.get("thread_id") + + # Apply proxy header overrides + proxy_request = litellm_params.get("proxy_server_request", {}) or {} + proxy_headers = proxy_request.get("headers", {}) or {} + current_project_name, tags, thread_id = extractors.apply_proxy_header_overrides( + current_project_name, tags, thread_id, proxy_headers + ) + + # Build shared metadata + metadata = extractors.extract_and_build_metadata( + opik_metadata=opik_metadata, + standard_logging_metadata=standard_logging_metadata, + standard_logging_object=standard_logging_object, + litellm_kwargs=kwargs, + ) + + # Get input/output data + input_data = standard_logging_object.get("messages", {}) + output_data = standard_logging_object.get("response", {}) + + # Decide whether to create a new trace or attach to existing + trace_payload: Optional[types.TracePayload] = None + if trace_id is None: + trace_id = utils.create_uuid7() + trace_payload = payload_builders.build_trace_payload( + project_name=current_project_name, + trace_id=trace_id, + response_obj=response_obj, + start_time=start_time, + end_time=end_time, + input_data=input_data, + output_data=output_data, + metadata=metadata, + tags=tags, + thread_id=thread_id, + ) + + # Always create a span + usage = utils.create_usage_object(response_obj["usage"]) + + # Extract provider and cost + provider = extractors.normalize_provider_name(kwargs.get("custom_llm_provider")) + cost = kwargs.get("response_cost") + + span_payload = payload_builders.build_span_payload( + project_name=current_project_name, + trace_id=trace_id, + parent_span_id=parent_span_id, + response_obj=response_obj, + start_time=start_time, + end_time=end_time, + input_data=input_data, + output_data=output_data, + metadata=metadata, + tags=tags, + usage=usage, + provider=provider, + cost=cost, + ) + + return trace_payload, span_payload diff --git a/litellm/integrations/opik/opik_payload_builder/extractors.py b/litellm/integrations/opik/opik_payload_builder/extractors.py new file mode 100644 index 00000000000..e4ff021778a --- /dev/null +++ b/litellm/integrations/opik/opik_payload_builder/extractors.py @@ -0,0 +1,221 @@ +"""Data extraction functions for Opik payload building.""" + +import json +from typing import Any, Dict, List, Optional, Tuple + +from litellm import _logging + + +def normalize_provider_name(provider: Optional[str]) -> Optional[str]: + """ + Normalize LiteLLM provider names to standardized string names. + + Args: + provider: LiteLLM internal provider name + + Returns: + Normalized provider name or the original if no mapping exists + """ + if provider is None: + return None + + # Provider mapping to names used in Opik + provider_mapping = { + "openai": "openai", + "vertex_ai-language-models": "google_vertexai", + "gemini": "google_ai", + "anthropic": "anthropic", + "vertex_ai-anthropic_models": "anthropic_vertexai", + "bedrock": "bedrock", + "bedrock_converse": "bedrock", + "groq": "groq", + } + + return provider_mapping.get(provider, provider) + + +def extract_opik_metadata( + litellm_metadata: Dict[str, Any], + standard_logging_metadata: Dict[str, Any], +) -> Dict[str, Any]: + """ + Extract and merge Opik metadata from request and requester. + + Args: + litellm_metadata: Metadata from litellm_params + standard_logging_metadata: Metadata from standard_logging_object + + Returns: + Merged Opik metadata dictionary + """ + opik_meta = litellm_metadata.get("opik", {}).copy() + + requester_metadata = standard_logging_metadata.get("requester_metadata", {}) or {} + requester_opik = requester_metadata.get("opik", {}) or {} + opik_meta.update(requester_opik) + + _logging.verbose_logger.debug( + f"litellm_opik_metadata - {json.dumps(opik_meta, default=str)}" + ) + + return opik_meta + + +def extract_span_identifiers( + current_span_data: Any, +) -> Tuple[Optional[str], Optional[str]]: + """ + Extract trace_id and parent_span_id from current_span_data. + + Args: + current_span_data: Either dict with trace_id/id keys or Opik object + + Returns: + Tuple of (trace_id, parent_span_id), both optional + """ + if current_span_data is None: + return None, None + + if isinstance(current_span_data, dict): + return (current_span_data.get("trace_id"), current_span_data.get("id")) + + try: + return current_span_data.trace_id, current_span_data.id + except AttributeError: + _logging.verbose_logger.warning( + f"Unexpected current_span_data format: {type(current_span_data)}" + ) + return None, None + + +def extract_tags( + opik_metadata: Dict[str, Any], + custom_llm_provider: Optional[str], +) -> List[str]: + """ + Extract and build list of tags. + + Args: + opik_metadata: Opik metadata dictionary + custom_llm_provider: LLM provider name to add as tag + + Returns: + List of tags + """ + tags = list(opik_metadata.get("tags", [])) + + if custom_llm_provider: + tags.append(custom_llm_provider) + + return tags + + +def apply_proxy_header_overrides( + project_name: str, + tags: List[str], + thread_id: Optional[str], + proxy_headers: Dict[str, Any], +) -> Tuple[str, List[str], Optional[str]]: + """ + Apply overrides from proxy request headers (opik_* prefix). + + Args: + project_name: Current project name + tags: Current tags list + thread_id: Current thread ID + proxy_headers: HTTP headers from proxy request + + Returns: + Tuple of (project_name, tags, thread_id) with overrides applied + """ + for key, value in proxy_headers.items(): + if not key.startswith("opik_") or not value: + continue + + param_key = key.replace("opik_", "", 1) + + if param_key == "project_name": + project_name = value + elif param_key == "thread_id": + thread_id = value + elif param_key == "tags": + try: + parsed_tags = json.loads(value) + if isinstance(parsed_tags, list): + tags.extend(parsed_tags) + except (json.JSONDecodeError, TypeError): + _logging.verbose_logger.warning( + f"Failed to parse tags from header: {value}" + ) + + return project_name, tags, thread_id + + +def extract_and_build_metadata( + opik_metadata: Dict[str, Any], + standard_logging_metadata: Dict[str, Any], + standard_logging_object: Dict[str, Any], + litellm_kwargs: Dict[str, Any], +) -> Dict[str, Any]: + """ + Build the complete metadata dictionary from all available sources. + + This combines: + - Opik-specific metadata (tags, etc.) + - Standard logging metadata + - Fields from standard_logging_object (model info, status, etc.) + - Cost information from litellm_kwargs (calculated after completion) + + Args: + opik_metadata: Opik-specific metadata from request + standard_logging_metadata: Standard logging metadata + standard_logging_object: Full standard logging object with call details + litellm_kwargs: Original LiteLLM kwargs (includes response_cost) + + Returns: + Complete metadata dictionary for trace/span + """ + # Start with opik metadata (excluding current_span_data which is used for trace linking) + metadata = {k: v for k, v in opik_metadata.items() if k != "current_span_data"} + metadata["created_from"] = "litellm" + + # Merge with standard logging metadata + metadata.update(standard_logging_metadata) + + # Add fields from standard_logging_object + # These come from the LiteLLM logging infrastructure + field_mappings = { + "call_type": "type", + "status": "status", + "model": "model", + "model_id": "model_id", + "model_group": "model_group", + "api_base": "api_base", + "cache_hit": "cache_hit", + "saved_cache_cost": "saved_cache_cost", + "error_str": "error_str", + "model_parameters": "model_parameters", + "hidden_params": "hidden_params", + "model_map_information": "model_map_information", + } + + for source_key, dest_key in field_mappings.items(): + if source_key in standard_logging_object: + metadata[dest_key] = standard_logging_object[source_key] + + # Add cost information + # response_cost is calculated by LiteLLM after completion and added to kwargs + # See: litellm/litellm_core_utils/llm_response_utils/response_metadata.py + if "response_cost" in litellm_kwargs: + metadata["cost"] = { + "total_tokens": litellm_kwargs["response_cost"], + "currency": "USD", + } + + # Add debug info if cost calculation failed + if "response_cost_failure_debug_info" in litellm_kwargs: + metadata["response_cost_failure_debug_info"] = litellm_kwargs[ + "response_cost_failure_debug_info" + ] + + return metadata diff --git a/litellm/integrations/opik/opik_payload_builder/payload_builders.py b/litellm/integrations/opik/opik_payload_builder/payload_builders.py new file mode 100644 index 00000000000..4656924fdb5 --- /dev/null +++ b/litellm/integrations/opik/opik_payload_builder/payload_builders.py @@ -0,0 +1,89 @@ +"""Payload builders for Opik traces and spans.""" + +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from litellm import _logging +from litellm.integrations.opik import utils + +from . import types + + +def build_trace_payload( + project_name: str, + trace_id: str, + response_obj: Dict[str, Any], + start_time: datetime, + end_time: datetime, + input_data: Any, + output_data: Any, + metadata: Dict[str, Any], + tags: List[str], + thread_id: Optional[str], +) -> types.TracePayload: + """Build a complete trace payload.""" + trace_name = response_obj.get("object", "unknown type") + + return types.TracePayload( + project_name=project_name, + id=trace_id, + name=trace_name, + start_time=( + start_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + ), + end_time=end_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), + input=input_data, + output=output_data, + metadata=metadata, + tags=tags, + thread_id=thread_id, + ) + + +def build_span_payload( + project_name: str, + trace_id: str, + parent_span_id: Optional[str], + response_obj: Dict[str, Any], + start_time: datetime, + end_time: datetime, + input_data: Any, + output_data: Any, + metadata: Dict[str, Any], + tags: List[str], + usage: Dict[str, int], + provider: Optional[str] = None, + cost: Optional[float] = None, +) -> types.SpanPayload: + """Build a complete span payload.""" + span_id = utils.create_uuid7() + + model = response_obj.get("model", "unknown-model") + obj_type = response_obj.get("object", "unknown-object") + created = response_obj.get("created", 0) + span_name = f"{model}_{obj_type}_{created}" + + _logging.verbose_logger.debug( + f"OpikLogger creating span with id {span_id} for trace {trace_id}" + ) + + return types.SpanPayload( + id=span_id, + project_name=project_name, + trace_id=trace_id, + parent_span_id=parent_span_id, + name=span_name, + type="llm", + model=model, + start_time=( + start_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + ), + end_time=end_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), + input=input_data, + output=output_data, + metadata=metadata, + tags=tags, + usage=usage, + provider=provider, + total_cost=cost, + ) diff --git a/litellm/integrations/opik/opik_payload_builder/types.py b/litellm/integrations/opik/opik_payload_builder/types.py new file mode 100644 index 00000000000..070cb11489a --- /dev/null +++ b/litellm/integrations/opik/opik_payload_builder/types.py @@ -0,0 +1,46 @@ +"""Type definitions for Opik payload building.""" + +from dataclasses import dataclass +from typing import Any, Dict, List, Literal, Optional, Tuple, Union + + +@dataclass +class TracePayload: + """Opik trace payload structure""" + + project_name: str + id: str + name: str + start_time: str + end_time: str + input: Any + output: Any + metadata: Dict[str, Any] + tags: List[str] + thread_id: Optional[str] = None + + +@dataclass +class SpanPayload: + """Opik span payload structure""" + + id: str + project_name: str + trace_id: str + name: str + type: Literal["llm"] + model: str + start_time: str + end_time: str + input: Any + output: Any + metadata: Dict[str, Any] + tags: List[str] + usage: Dict[str, int] + parent_span_id: Optional[str] = None + provider: Optional[str] = None + total_cost: Optional[float] = None + + +PayloadItem = Union[TracePayload, SpanPayload] +TraceSpanPayloadTuple = Tuple[Optional[TracePayload], SpanPayload] diff --git a/litellm/integrations/opik/utils.py b/litellm/integrations/opik/utils.py index 7b3b64dcf38..b0ab5991c91 100644 --- a/litellm/integrations/opik/utils.py +++ b/litellm/integrations/opik/utils.py @@ -1,7 +1,7 @@ import configparser import os import time -from typing import Dict, Final, List, Optional +from typing import Any, Dict, Final, List, Optional, Tuple CONFIG_FILE_PATH_DEFAULT: Final[str] = "~/.opik.config" @@ -99,12 +99,26 @@ def create_usage_object(usage): return usage_dict -def _remove_nulls(x): - x_ = {k: v for k, v in x.items() if v is not None} - return x_ +def _remove_nulls(x: Dict[str, Any]) -> Dict[str, Any]: + """Remove None values from dict.""" + return {k: v for k, v in x.items() if v is not None} -def get_traces_and_spans_from_payload(payload: List): +def get_traces_and_spans_from_payload( + payload: List[Dict[str, Any]] +) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + """ + Separate traces and spans from payload. + + Traces are identified by not having a "type" field. + Spans are identified by having a "type" field. + + Args: + payload: List of dicts containing trace and span data + + Returns: + Tuple of (traces, spans) where both are lists of dicts with null values removed + """ traces = [_remove_nulls(x) for x in payload if "type" not in x] spans = [_remove_nulls(x) for x in payload if "type" in x] return traces, spans diff --git a/litellm/integrations/posthog.py b/litellm/integrations/posthog.py index c609d30ccff..468b1a441fb 100644 --- a/litellm/integrations/posthog.py +++ b/litellm/integrations/posthog.py @@ -10,6 +10,7 @@ For batching specific details see CustomBatchLogger class """ import asyncio +import atexit import os from typing import Any, Dict, Optional, Tuple @@ -55,7 +56,10 @@ class PostHogLogger(CustomBatchLogger): self._async_initialized = False self.flush_lock = None self.log_queue = [] - + + # Register cleanup handler to flush internal queue on exit + atexit.register(self._flush_on_exit) + super().__init__( **kwargs, flush_lock=None, batch_size=POSTHOG_MAX_BATCH_SIZE ) @@ -377,3 +381,58 @@ class PostHogLogger(CustomBatchLogger): if obj is None or not hasattr(obj, 'get'): return default return obj.get(key, default) + + def _flush_on_exit(self): + """ + Flush remaining events from internal log_queue before process exit. + Called automatically via atexit handler. + + This works in conjunction with GLOBAL_LOGGING_WORKER's atexit handler: + 1. GLOBAL_LOGGING_WORKER atexit invokes pending callbacks + 2. Callbacks add events to this logger's internal log_queue + 3. This atexit handler flushes the internal queue to PostHog + """ + if not self.log_queue: + return + + verbose_logger.debug( + f"PostHog: Flushing {len(self.log_queue)} remaining events on exit" + ) + + try: + # Group events by credentials (same logic as async_send_batch) + batches_by_credentials: Dict[Tuple[str, str], list] = {} + for item in self.log_queue: + key = (item["api_key"], item["api_url"]) + if key not in batches_by_credentials: + batches_by_credentials[key] = [] + batches_by_credentials[key].append(item["event"]) + + # Send each batch synchronously using sync_client + for (api_key, api_url), events in batches_by_credentials.items(): + headers = { + "Content-Type": "application/json", + } + + payload = self._create_posthog_payload(events, api_key) + capture_url = f"{api_url.rstrip('/')}/batch/" + + response = self.sync_client.post( + url=capture_url, + json=payload, + headers=headers, + ) + response.raise_for_status() + + if response.status_code != 200: + verbose_logger.error( + f"PostHog: Failed to flush on exit - status {response.status_code}" + ) + + verbose_logger.debug( + f"PostHog: Successfully flushed {len(self.log_queue)} events on exit" + ) + self.log_queue.clear() + + except Exception as e: + verbose_logger.error(f"PostHog: Error flushing events on exit: {str(e)}") diff --git a/litellm/integrations/s3.py b/litellm/integrations/s3.py index 53caeb0d198..2e70b1d6519 100644 --- a/litellm/integrations/s3.py +++ b/litellm/integrations/s3.py @@ -181,13 +181,13 @@ class S3Logger: def get_s3_object_key( s3_path: str, - team_alias_prefix: str, + prefix: str, start_time: datetime, s3_file_name: str, ) -> str: s3_object_key = ( (s3_path.rstrip("/") + "/" if s3_path else "") - + team_alias_prefix + + prefix + start_time.strftime("%Y-%m-%d") + "/" + s3_file_name diff --git a/litellm/integrations/s3_v2.py b/litellm/integrations/s3_v2.py index a65500c80dc..534b85e4752 100644 --- a/litellm/integrations/s3_v2.py +++ b/litellm/integrations/s3_v2.py @@ -49,6 +49,8 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): s3_batch_size: Optional[int] = DEFAULT_S3_BATCH_SIZE, s3_config=None, s3_use_team_prefix: bool = False, + s3_strip_base64_files: bool = False, + s3_use_key_prefix: bool = False, **kwargs, ): try: @@ -56,12 +58,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): f"in init s3 logger - s3_callback_params {litellm.s3_callback_params}" ) - # IMPORTANT: We use a concurrent limit of 1 to upload to s3 - # Files should get uploaded BUT they should not impact latency of LLM calling logic - self.async_httpx_client = get_async_httpx_client( - llm_provider=httpxSpecialProvider.LoggingCallback, - ) - + # Initialize S3 params first to get the correct s3_verify value self._init_s3_params( s3_bucket_name=s3_bucket_name, s3_region_name=s3_region_name, @@ -80,9 +77,21 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): s3_config=s3_config, s3_path=s3_path, s3_use_team_prefix=s3_use_team_prefix, + s3_strip_base64_files=s3_strip_base64_files, + s3_use_key_prefix=s3_use_key_prefix ) verbose_logger.debug(f"s3 logger using endpoint url {s3_endpoint_url}") + # IMPORTANT + # Create httpx client AFTER _init_s3_params so we have the correct s3_verify value + verbose_logger.debug( + f"s3_v2 logger creating async httpx client with s3_verify={self.s3_verify}" + ) + self.async_httpx_client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.LoggingCallback, + params={"ssl_verify": self.s3_verify} + ) + asyncio.create_task(self.periodic_flush()) self.flush_lock = asyncio.Lock() @@ -124,6 +133,8 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): s3_config=None, s3_path: Optional[str] = None, s3_use_team_prefix: bool = False, + s3_strip_base64_files: bool = False, + s3_use_key_prefix: bool = False, ): """ Initialize the s3 params for this logging callback @@ -144,9 +155,11 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): litellm.s3_callback_params.get("s3_api_version") or s3_api_version ) self.s3_use_ssl = ( - litellm.s3_callback_params.get("s3_use_ssl", True) or s3_use_ssl + litellm.s3_callback_params.get("s3_use_ssl", True) if litellm.s3_callback_params.get("s3_use_ssl") is not None else s3_use_ssl + ) + self.s3_verify = ( + litellm.s3_callback_params.get("s3_verify") if litellm.s3_callback_params.get("s3_verify") is not None else s3_verify ) - self.s3_verify = litellm.s3_callback_params.get("s3_verify") or s3_verify self.s3_endpoint_url = ( litellm.s3_callback_params.get("s3_endpoint_url") or s3_endpoint_url ) @@ -194,6 +207,16 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): or s3_use_team_prefix ) + self.s3_use_key_prefix = ( + bool(litellm.s3_callback_params.get("s3_use_key_prefix", False)) + or s3_use_key_prefix + ) + + self.s3_strip_base64_files = ( + bool(litellm.s3_callback_params.get("s3_strip_base64_files", False)) + or s3_strip_base64_files + ) + return async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -239,7 +262,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): ) except Exception as e: verbose_logger.exception(f"s3 Layer Error - {str(e)}") - pass + self.handle_callback_failure(callback_name="S3Logger") async def async_upload_data_to_s3( self, batch_logging_element: s3BatchLoggingElement @@ -271,6 +294,9 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): verbose_logger.debug( f"s3_v2 logger - uploading data to s3 - {batch_logging_element.s3_object_key}" ) + verbose_logger.debug( + f"s3_v2 logger - s3_verify setting: {self.s3_verify}" + ) # Prepare the URL url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}" @@ -323,6 +349,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): response.raise_for_status() except Exception as e: verbose_logger.exception(f"Error uploading to s3: {str(e)}") + self.handle_callback_failure(callback_name="S3Logger") async def async_send_batch(self): """ @@ -364,33 +391,37 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): if standard_logging_payload is None: return None - team_alias = standard_logging_payload["metadata"].get("user_api_key_team_alias") + if self.s3_strip_base64_files: + standard_logging_payload = self._strip_base64_from_messages_sync(standard_logging_payload) - team_alias_prefix = "" - if ( - litellm.enable_preview_features - and self.s3_use_team_prefix - and team_alias is not None - ): - team_alias_prefix = f"{team_alias}/" + # Base prefix (default empty) + prefix_components = [] + if self.s3_use_team_prefix: + team_alias = standard_logging_payload.get("metadata", {}).get("user_api_key_team_alias", None) + if team_alias: + prefix_components.append(team_alias) + if self.s3_use_key_prefix: + user_api_key_alias = standard_logging_payload.get("metadata", {}).get("user_api_key_alias", None) + if user_api_key_alias: + prefix_components.append(user_api_key_alias) + + + # Construct full prefix path + prefix_path = "/".join(prefix_components) + if prefix_path: + prefix_path += "/" s3_file_name = ( litellm.utils.get_logging_id(start_time, standard_logging_payload) or "" ) + verbose_logger.debug(f"Creating s3 file with prefix_components={prefix_components},prefix_path={prefix_path} and {s3_file_name}") s3_object_key = get_s3_object_key( s3_path=cast(Optional[str], self.s3_path) or "", - team_alias_prefix=team_alias_prefix, + prefix=prefix_path, start_time=start_time, s3_file_name=s3_file_name, ) - - s3_object_download_filename = ( - "time-" - + start_time.strftime("%Y-%m-%dT%H-%M-%S-%f") - + "_" - + standard_logging_payload["id"] - + ".json" - ) + verbose_logger.debug(f"s3_object_key={s3_object_key}") s3_object_download_filename = f"time-{start_time.strftime('%Y-%m-%dT%H-%M-%S-%f')}_{standard_logging_payload['id']}.json" @@ -465,12 +496,15 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM): # Prepare the signed headers signed_headers = dict(aws_request.headers.items()) - httpx_client = _get_httpx_client() + httpx_client = _get_httpx_client( + params={"ssl_verify": self.s3_verify} if self.s3_verify is not None else None + ) # Make the request response = httpx_client.put(url, data=json_string, headers=signed_headers) response.raise_for_status() except Exception as e: verbose_logger.exception(f"Error uploading to s3: {str(e)}") + self.handle_callback_failure(callback_name="S3Logger") async def _download_object_from_s3(self, s3_object_key: str) -> Optional[dict]: """ diff --git a/litellm/integrations/sqs.py b/litellm/integrations/sqs.py index 8a2ebf8d344..97a4c5723d8 100644 --- a/litellm/integrations/sqs.py +++ b/litellm/integrations/sqs.py @@ -7,6 +7,9 @@ This logger sends ``StandardLoggingPayload`` entries to an AWS SQS queue. from __future__ import annotations import asyncio +import base64 +import json +import re import traceback from typing import List, Optional @@ -27,31 +30,43 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.types.utils import StandardLoggingPayload from .custom_batch_logger import CustomBatchLogger +from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus + +_BASE64_INLINE_PATTERN = re.compile( + r"data:(?:application|image|audio|video)/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=\s]+", + re.MULTILINE, +) class SQSLogger(CustomBatchLogger, BaseAWSLLM): - """Batching logger that writes logs to an AWS SQS queue.""" + """Batching logger that writes logs to an AWS SQS queue, optionally encrypting the payload.""" def __init__( - self, - sqs_queue_url: Optional[str] = None, - sqs_region_name: Optional[str] = None, - sqs_api_version: Optional[str] = None, - sqs_use_ssl: bool = True, - sqs_verify: Optional[bool] = None, - sqs_endpoint_url: Optional[str] = None, - sqs_aws_access_key_id: Optional[str] = None, - sqs_aws_secret_access_key: Optional[str] = None, - sqs_aws_session_token: Optional[str] = None, - sqs_aws_session_name: Optional[str] = None, - sqs_aws_profile_name: Optional[str] = None, - sqs_aws_role_name: Optional[str] = None, - sqs_aws_web_identity_token: Optional[str] = None, - sqs_aws_sts_endpoint: Optional[str] = None, - sqs_flush_interval: Optional[int] = DEFAULT_SQS_FLUSH_INTERVAL_SECONDS, - sqs_batch_size: Optional[int] = DEFAULT_SQS_BATCH_SIZE, - sqs_config=None, - **kwargs, + self, + # --- Standard SQS params --- + sqs_queue_url: Optional[str] = None, + sqs_region_name: Optional[str] = None, + sqs_api_version: Optional[str] = None, + sqs_use_ssl: bool = True, + sqs_verify: Optional[bool] = None, + sqs_endpoint_url: Optional[str] = None, + sqs_aws_access_key_id: Optional[str] = None, + sqs_aws_secret_access_key: Optional[str] = None, + sqs_aws_session_token: Optional[str] = None, + sqs_aws_session_name: Optional[str] = None, + sqs_aws_profile_name: Optional[str] = None, + sqs_aws_role_name: Optional[str] = None, + sqs_aws_web_identity_token: Optional[str] = None, + sqs_aws_sts_endpoint: Optional[str] = None, + sqs_flush_interval: Optional[int] = DEFAULT_SQS_FLUSH_INTERVAL_SECONDS, + sqs_batch_size: Optional[int] = DEFAULT_SQS_BATCH_SIZE, + sqs_config=None, + sqs_strip_base64_files: bool = False, + # --- 🔐 Application-level encryption params --- + sqs_aws_use_application_level_encryption: bool = False, + sqs_app_encryption_key_b64: Optional[str] = None, + sqs_app_encryption_aad: Optional[str] = None, + **kwargs, ) -> None: try: verbose_logger.debug( @@ -77,7 +92,12 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): sqs_aws_role_name=sqs_aws_role_name, sqs_aws_web_identity_token=sqs_aws_web_identity_token, sqs_aws_sts_endpoint=sqs_aws_sts_endpoint, + sqs_strip_base64_files=sqs_strip_base64_files, + sqs_aws_use_application_level_encryption=sqs_aws_use_application_level_encryption, + sqs_app_encryption_key_b64=sqs_app_encryption_key_b64, + sqs_app_encryption_aad=sqs_app_encryption_aad, sqs_config=sqs_config, + **kwargs, ) asyncio.create_task(self.periodic_flush()) @@ -95,7 +115,6 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): ) self.log_queue: List[StandardLoggingPayload] = [] - BaseAWSLLM.__init__(self) except Exception as e: @@ -103,22 +122,26 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): raise e def _init_sqs_params( - self, - sqs_queue_url: Optional[str] = None, - sqs_region_name: Optional[str] = None, - sqs_api_version: Optional[str] = None, - sqs_use_ssl: bool = True, - sqs_verify: Optional[bool] = None, - sqs_endpoint_url: Optional[str] = None, - sqs_aws_access_key_id: Optional[str] = None, - sqs_aws_secret_access_key: Optional[str] = None, - sqs_aws_session_token: Optional[str] = None, - sqs_aws_session_name: Optional[str] = None, - sqs_aws_profile_name: Optional[str] = None, - sqs_aws_role_name: Optional[str] = None, - sqs_aws_web_identity_token: Optional[str] = None, - sqs_aws_sts_endpoint: Optional[str] = None, - sqs_config=None, + self, + sqs_queue_url: Optional[str] = None, + sqs_region_name: Optional[str] = None, + sqs_api_version: Optional[str] = None, + sqs_use_ssl: bool = True, + sqs_verify: Optional[bool] = None, + sqs_endpoint_url: Optional[str] = None, + sqs_aws_access_key_id: Optional[str] = None, + sqs_aws_secret_access_key: Optional[str] = None, + sqs_aws_session_token: Optional[str] = None, + sqs_aws_session_name: Optional[str] = None, + sqs_aws_profile_name: Optional[str] = None, + sqs_aws_role_name: Optional[str] = None, + sqs_aws_web_identity_token: Optional[str] = None, + sqs_aws_sts_endpoint: Optional[str] = None, + sqs_strip_base64_files: bool = False, + sqs_aws_use_application_level_encryption: bool = False, + sqs_app_encryption_key_b64: Optional[str] = None, + sqs_app_encryption_aad: Optional[str] = None, + sqs_config=None, ) -> None: litellm.aws_sqs_callback_params = litellm.aws_sqs_callback_params or {} @@ -128,67 +151,95 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): litellm.aws_sqs_callback_params[key] = litellm.get_secret(value) self.sqs_queue_url = ( - litellm.aws_sqs_callback_params.get("sqs_queue_url") or sqs_queue_url + litellm.aws_sqs_callback_params.get("sqs_queue_url") or sqs_queue_url ) self.sqs_region_name = ( - litellm.aws_sqs_callback_params.get("sqs_region_name") or sqs_region_name + litellm.aws_sqs_callback_params.get("sqs_region_name") or sqs_region_name ) self.sqs_api_version = ( - litellm.aws_sqs_callback_params.get("sqs_api_version") or sqs_api_version + litellm.aws_sqs_callback_params.get("sqs_api_version") or sqs_api_version ) self.sqs_use_ssl = ( - litellm.aws_sqs_callback_params.get("sqs_use_ssl", True) or sqs_use_ssl + litellm.aws_sqs_callback_params.get("sqs_use_ssl", True) or sqs_use_ssl ) self.sqs_verify = litellm.aws_sqs_callback_params.get("sqs_verify") or sqs_verify self.sqs_endpoint_url = ( - litellm.aws_sqs_callback_params.get("sqs_endpoint_url") or sqs_endpoint_url + litellm.aws_sqs_callback_params.get("sqs_endpoint_url") or sqs_endpoint_url ) self.sqs_aws_access_key_id = ( - litellm.aws_sqs_callback_params.get("sqs_aws_access_key_id") - or sqs_aws_access_key_id + litellm.aws_sqs_callback_params.get("sqs_aws_access_key_id") + or sqs_aws_access_key_id ) self.sqs_aws_secret_access_key = ( - litellm.aws_sqs_callback_params.get("sqs_aws_secret_access_key") - or sqs_aws_secret_access_key + litellm.aws_sqs_callback_params.get("sqs_aws_secret_access_key") + or sqs_aws_secret_access_key ) self.sqs_aws_session_token = ( - litellm.aws_sqs_callback_params.get("sqs_aws_session_token") - or sqs_aws_session_token + litellm.aws_sqs_callback_params.get("sqs_aws_session_token") + or sqs_aws_session_token ) self.sqs_aws_session_name = ( - litellm.aws_sqs_callback_params.get("sqs_aws_session_name") or sqs_aws_session_name + litellm.aws_sqs_callback_params.get("sqs_aws_session_name") or sqs_aws_session_name ) self.sqs_aws_profile_name = ( - litellm.aws_sqs_callback_params.get("sqs_aws_profile_name") or sqs_aws_profile_name + litellm.aws_sqs_callback_params.get("sqs_aws_profile_name") or sqs_aws_profile_name ) self.sqs_aws_role_name = ( - litellm.aws_sqs_callback_params.get("sqs_aws_role_name") or sqs_aws_role_name + litellm.aws_sqs_callback_params.get("sqs_aws_role_name") or sqs_aws_role_name ) self.sqs_aws_web_identity_token = ( - litellm.aws_sqs_callback_params.get("sqs_aws_web_identity_token") - or sqs_aws_web_identity_token + litellm.aws_sqs_callback_params.get("sqs_aws_web_identity_token") + or sqs_aws_web_identity_token ) self.sqs_aws_sts_endpoint = ( - litellm.aws_sqs_callback_params.get("sqs_aws_sts_endpoint") or sqs_aws_sts_endpoint + litellm.aws_sqs_callback_params.get("sqs_aws_sts_endpoint") or sqs_aws_sts_endpoint + ) + self.sqs_strip_base64_files = ( + litellm.aws_sqs_callback_params.get("sqs_strip_base64_files", False) + or sqs_strip_base64_files ) + self.sqs_aws_use_application_level_encryption = ( + litellm.aws_sqs_callback_params.get("sqs_aws_use_application_level_encryption", False) + or sqs_aws_use_application_level_encryption + ) + self.sqs_app_encryption_key_b64 = ( + litellm.aws_sqs_callback_params.get("sqs_app_encryption_key_b64") + or sqs_app_encryption_key_b64 + ) + self.sqs_app_encryption_aad = ( + litellm.aws_sqs_callback_params.get("sqs_app_encryption_aad") + or sqs_app_encryption_aad + ) + self.app_crypto: Optional["AppCrypto"] = None + if self.sqs_aws_use_application_level_encryption: + from litellm.litellm_core_utils.app_crypto import AppCrypto + if not self.sqs_app_encryption_key_b64: + raise ValueError("sqs_app_encryption_key_b64 is required when encryption is enabled.") + key = base64.b64decode(self.sqs_app_encryption_key_b64) + self.app_crypto = AppCrypto(key) + verbose_logger.debug( + "SQSLogger: Application-level encryption enabled." + ) self.sqs_config = litellm.aws_sqs_callback_params.get("sqs_config") or sqs_config async def async_log_success_event( - self, kwargs, response_obj, start_time, end_time + self, kwargs, response_obj, start_time, end_time ) -> None: try: verbose_logger.debug( "SQS Logging - Enters logging function for model %s", kwargs ) standard_logging_payload = kwargs.get("standard_logging_object") + if self.sqs_strip_base64_files: + standard_logging_payload = await self._strip_base64_from_messages(standard_logging_payload) if standard_logging_payload is None: raise ValueError("standard_logging_payload is None") @@ -206,6 +257,8 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): standard_logging_payload = kwargs.get("standard_logging_object") if standard_logging_payload is None: raise ValueError("standard_logging_payload is None") + if self.sqs_strip_base64_files: + standard_logging_payload = await self._strip_base64_from_messages(standard_logging_payload) self.log_queue.append(standard_logging_payload) verbose_logger.debug( @@ -256,11 +309,21 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): if self.sqs_queue_url is None: raise ValueError("sqs_queue_url not set") - json_string = safe_dumps(payload) + json_data = json.loads(safe_dumps(payload)) + if self.app_crypto: + aad_bytes = ( + self.sqs_app_encryption_aad.encode("utf-8") + if self.sqs_app_encryption_aad + else None + ) + encrypted = self.app_crypto.encrypt_json(json_data, aad=aad_bytes) + json_string = json.dumps({"__encrypted__": True, "payload": encrypted}) + else: + json_string = safe_dumps(payload) body = ( - f"Action={SQS_SEND_MESSAGE_ACTION}&Version={SQS_API_VERSION}&MessageBody=" - + quote(json_string, safe="") + f"Action={SQS_SEND_MESSAGE_ACTION}&Version={SQS_API_VERSION}&MessageBody=" + + quote(json_string, safe="") ) headers = { @@ -293,3 +356,18 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM): except Exception as e: verbose_logger.exception(f"Error sending to SQS: {str(e)}") + async def async_health_check(self) -> IntegrationHealthCheckStatus: + """ + Health check for SQS by sending a small test message to the configured queue. + """ + try: + from litellm.litellm_core_utils.litellm_logging import ( + create_dummy_standard_logging_payload, + ) + # Create a minimal standard logging payload + standard_logging_object: StandardLoggingPayload = create_dummy_standard_logging_payload() + # Attempt to send a single message + await self.async_send_message(standard_logging_object) + return IntegrationHealthCheckStatus(status="healthy", error_message=None) + except Exception as e: + return IntegrationHealthCheckStatus(status="unhealthy", error_message=str(e)) diff --git a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py index 8ef160dd783..236935778d6 100644 --- a/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py +++ b/litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py @@ -5,7 +5,7 @@ This hook is called before making an LLM request when a vector store is configur It searches the vector store for relevant context and appends it to the messages. """ -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, cast +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast import litellm import litellm.vector_stores @@ -25,6 +25,7 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = None + class VectorStorePreCallHook(CustomLogger): CONTENT_PREFIX_STRING = "Context:\n\n" """ @@ -54,7 +55,7 @@ class VectorStorePreCallHook(CustomLogger): ) -> Tuple[str, List[AllMessageValues], dict]: """ Perform vector store search and append results as context to messages. - + Args: model: The model name messages: List of messages @@ -64,7 +65,7 @@ class VectorStorePreCallHook(CustomLogger): dynamic_callback_params: Optional dynamic callback parameters prompt_label: Optional prompt label prompt_version: Optional prompt version - + Returns: Tuple of (model, modified_messages, non_default_params) """ @@ -73,124 +74,283 @@ class VectorStorePreCallHook(CustomLogger): if litellm.vector_store_registry is None: return model, messages, non_default_params - vector_stores_to_run: List[LiteLLM_ManagedVectorStore] = litellm.vector_store_registry.pop_vector_stores_to_run( - non_default_params=non_default_params, tools=tools + vector_stores_to_run: List[LiteLLM_ManagedVectorStore] = ( + litellm.vector_store_registry.pop_vector_stores_to_run( + non_default_params=non_default_params, tools=tools + ) ) - + if not vector_stores_to_run: return model, messages, non_default_params - + # Extract the query from the last user message query = self._extract_query_from_messages(messages) - + if not query: - verbose_logger.debug("No query found in messages for vector store search") + verbose_logger.debug( + "No query found in messages for vector store search" + ) return model, messages, non_default_params - + modified_messages: List[AllMessageValues] = messages.copy() + all_search_results: List[VectorStoreSearchResponse] = [] + for vector_store_to_run in vector_stores_to_run: - + # Get vector store id from the vector store config vector_store_id = vector_store_to_run.get("vector_store_id", "") custom_llm_provider = vector_store_to_run.get("custom_llm_provider") - litellm_params_for_vector_store = vector_store_to_run.get("litellm_params", {}) or {} + litellm_params_for_vector_store = ( + vector_store_to_run.get("litellm_params", {}) or {} + ) # Call litellm.vector_stores.search() with the required parameters search_response = await litellm.vector_stores.asearch( - vector_store_id=vector_store_id, - query=query, - custom_llm_provider=custom_llm_provider, - **litellm_params_for_vector_store + **{ + "vector_store_id": vector_store_id, + "query": query, + "custom_llm_provider": custom_llm_provider, + **litellm_params_for_vector_store, + }, ) verbose_logger.debug(f"search_response: {search_response}") - - + + # Store search results for later use in citations + all_search_results.append(search_response) + # Process search results and append as context modified_messages = self._append_search_results_to_messages( - messages=messages, - search_response=search_response + messages=messages, search_response=search_response ) - + # Get the number of results for logging num_results = 0 num_results = len(search_response.get("data", []) or []) - verbose_logger.debug(f"Vector store search completed. Added context from {num_results} results") - + verbose_logger.debug( + f"Vector store search completed. Added context from {num_results} results" + ) + + # Store search results as-is (already in OpenAI-compatible format) + if litellm_logging_obj and all_search_results: + litellm_logging_obj.model_call_details["search_results"] = ( + all_search_results + ) + return model, modified_messages, non_default_params - + except Exception as e: verbose_logger.exception(f"Error in VectorStorePreCallHook: {str(e)}") # Return original parameters on error return model, messages, non_default_params - def _extract_query_from_messages(self, messages: List[AllMessageValues]) -> Optional[str]: + def _extract_query_from_messages( + self, messages: List[AllMessageValues] + ) -> Optional[str]: """ Extract the query from the last user message. - + Args: messages: List of messages - + Returns: The extracted query string or None if not found """ if not messages or len(messages) == 0: return None - + last_message = messages[-1] if not isinstance(last_message, dict) or "content" not in last_message: return None - + content = last_message["content"] - + if isinstance(content, str): return content elif isinstance(content, list) and len(content) > 0: # Handle list of content items, extract text from first text item for item in content: - if isinstance(item, dict) and item.get("type") == "text" and "text" in item: + if ( + isinstance(item, dict) + and item.get("type") == "text" + and "text" in item + ): return item["text"] - + return None def _append_search_results_to_messages( - self, - messages: List[AllMessageValues], - search_response: VectorStoreSearchResponse + self, + messages: List[AllMessageValues], + search_response: VectorStoreSearchResponse, ) -> List[AllMessageValues]: """ Append search results as context to the messages. - + Args: messages: Original list of messages search_response: Response from vector store search - + Returns: Modified list of messages with context appended """ - search_response_data: Optional[List[VectorStoreSearchResult]] = search_response.get("data") + search_response_data: Optional[List[VectorStoreSearchResult]] = ( + search_response.get("data") + ) if not search_response_data: return messages - + context_content = self.CONTENT_PREFIX_STRING - + for result in search_response_data: - result_content: Optional[List[VectorStoreResultContent]] = result.get("content") + result_content: Optional[List[VectorStoreResultContent]] = result.get( + "content" + ) if result_content: for content_item in result_content: content_text: Optional[str] = content_item.get("text") if content_text: context_content += content_text + "\n\n" - + # Only add context if we found any content if context_content != "Context:\n\n": # Create a copy of messages to avoid modifying the original modified_messages = messages.copy() # Add context as a new message before the last user message context_message: ChatCompletionUserMessage = { - "role": "user", - "content": context_content + "role": "user", + "content": context_content, } modified_messages.insert(-1, cast(AllMessageValues, context_message)) return modified_messages - + return messages + + async def async_post_call_success_deployment_hook( + self, + request_data: dict, + response: Any, + call_type: Optional[Any], + ) -> Optional[Any]: + """ + Add search results to the response after successful LLM call. + + This hook adds the vector store search results (already in OpenAI-compatible format) + to the response's provider_specific_fields. + """ + try: + verbose_logger.debug( + "VectorStorePreCallHook.async_post_call_success_deployment_hook called" + ) + + # Get logging object from request_data + litellm_logging_obj = request_data.get("litellm_logging_obj") + if not litellm_logging_obj: + verbose_logger.debug("No litellm_logging_obj in request_data") + return None + + verbose_logger.debug( + f"model_call_details keys: {list(litellm_logging_obj.model_call_details.keys())}" + ) + + # Get search results from model_call_details (already in OpenAI format) + search_results: Optional[List[VectorStoreSearchResponse]] = ( + litellm_logging_obj.model_call_details.get("search_results") + ) + + verbose_logger.debug(f"Search results found: {search_results is not None}") + + if not search_results: + verbose_logger.debug("No search results found") + return None + + # Add search results to response object + if hasattr(response, "choices") and response.choices: + for choice in response.choices: + if hasattr(choice, "message") and choice.message: + # Get existing provider_specific_fields or create new dict + provider_fields = ( + getattr(choice.message, "provider_specific_fields", None) + or {} + ) + + # Add search results (already in OpenAI-compatible format) + provider_fields["search_results"] = search_results + + # Set the provider_specific_fields + setattr( + choice.message, "provider_specific_fields", provider_fields + ) + + verbose_logger.debug( + f"Added {len(search_results)} search results to response" + ) + + # Return modified response + return response + + except Exception as e: + verbose_logger.exception( + f"Error adding search results to response: {str(e)}" + ) + # Don't fail the request if search results fail to be added + return None + + async def async_post_call_streaming_deployment_hook( + self, + request_data: dict, + response_chunk: Any, + call_type: Optional[Any], + ) -> Optional[Any]: + """ + Add search results to the final streaming chunk. + + This hook is called for the final streaming chunk, allowing us to add + search results to the stream before it's returned to the user. + """ + try: + verbose_logger.debug( + "VectorStorePreCallHook.async_post_call_streaming_deployment_hook called" + ) + + # Get search results from model_call_details (already in OpenAI format) + search_results: Optional[List[VectorStoreSearchResponse]] = ( + request_data.get("search_results") + ) + + verbose_logger.debug( + f"Search results found for streaming chunk: {search_results is not None}" + ) + + if not search_results: + verbose_logger.debug("No search results found for streaming chunk") + return response_chunk + + # Add search results to streaming chunk + if hasattr(response_chunk, "choices") and response_chunk.choices: + for choice in response_chunk.choices: + if hasattr(choice, "delta") and choice.delta: + # Get existing provider_specific_fields or create new dict + provider_fields = ( + getattr(choice.delta, "provider_specific_fields", None) + or {} + ) + + # Add search results (already in OpenAI-compatible format) + provider_fields["search_results"] = search_results + + # Set the provider_specific_fields + choice.delta.provider_specific_fields = provider_fields + + verbose_logger.debug( + f"Added {len(search_results)} search results to streaming chunk" + ) + + # Return modified chunk + return response_chunk + + except Exception as e: + verbose_logger.exception( + f"Error adding search results to streaming chunk: {str(e)}" + ) + # Don't fail the request if search results fail to be added + return response_chunk diff --git a/litellm/litellm_core_utils/app_crypto.py b/litellm/litellm_core_utils/app_crypto.py new file mode 100644 index 00000000000..5ce6d8d77f9 --- /dev/null +++ b/litellm/litellm_core_utils/app_crypto.py @@ -0,0 +1,33 @@ +import base64 +import json +import os +from typing import Optional + +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + +class AppCrypto: + def __init__(self, master_key: bytes): + if len(master_key) != 32: + raise ValueError("Master key must be 32 bytes for AES-256-GCM") + self.key = master_key + + def encrypt_json(self, data: dict, aad: Optional[bytes] = None) -> dict: + aes = AESGCM(self.key) + nonce = os.urandom(12) + plaintext = json.dumps(data).encode("utf-8") + ct = aes.encrypt(nonce, plaintext, aad) + ciphertext, tag = ct[:-16], ct[-16:] + return { + "nonce": base64.b64encode(nonce).decode(), + "ciphertext": base64.b64encode(ciphertext).decode(), + "tag": base64.b64encode(tag).decode(), + } + + def decrypt_json(self, enc: dict, aad: Optional[bytes] = None) -> dict: + aes = AESGCM(self.key) + nonce = base64.b64decode(enc["nonce"]) + ct = base64.b64decode(enc["ciphertext"]) + tag = base64.b64decode(enc["tag"]) + data = aes.decrypt(nonce, ct + tag, aad) + return json.loads(data.decode()) \ No newline at end of file diff --git a/litellm/litellm_core_utils/audio_utils/utils.py b/litellm/litellm_core_utils/audio_utils/utils.py index fc0c8aca842..2f0db4978ff 100644 --- a/litellm/litellm_core_utils/audio_utils/utils.py +++ b/litellm/litellm_core_utils/audio_utils/utils.py @@ -4,6 +4,7 @@ Utils used for litellm.transcription() and litellm.atranscription() import os from dataclasses import dataclass +from typing import Optional from litellm.types.files import get_file_mime_type_from_extension from litellm.types.utils import FileTypes @@ -13,12 +14,13 @@ from litellm.types.utils import FileTypes class ProcessedAudioFile: """ Processed audio file data. - + Attributes: file_content: The binary content of the audio file filename: The filename (extracted or generated) content_type: The MIME type of the audio file """ + file_content: bytes filename: str content_type: str @@ -27,61 +29,63 @@ class ProcessedAudioFile: def process_audio_file(audio_file: FileTypes) -> ProcessedAudioFile: """ Common utility function to process audio files for audio transcription APIs. - + Handles various input types: - File paths (str, os.PathLike) - Raw bytes/bytearray - Tuples (filename, content, optional content_type) - File-like objects with read() method - + Args: audio_file: The audio file input in various formats - + Returns: ProcessedAudioFile: Structured data with file content, filename, and content type - + Raises: ValueError: If audio_file type is unsupported or content cannot be extracted """ file_content = None filename = None - + if isinstance(audio_file, (bytes, bytearray)): # Raw bytes - filename = 'audio.wav' + filename = "audio.wav" file_content = bytes(audio_file) elif isinstance(audio_file, (str, os.PathLike)): # File path or PathLike file_path = str(audio_file) - with open(file_path, 'rb') as f: + with open(file_path, "rb") as f: file_content = f.read() - filename = file_path.split('/')[-1] + filename = file_path.split("/")[-1] elif isinstance(audio_file, tuple): # Tuple format: (filename, content, content_type) or (filename, content) if len(audio_file) >= 2: - filename = audio_file[0] or 'audio.wav' + filename = audio_file[0] or "audio.wav" content = audio_file[1] if isinstance(content, (bytes, bytearray)): file_content = bytes(content) elif isinstance(content, (str, os.PathLike)): # File path or PathLike - with open(str(content), 'rb') as f: + with open(str(content), "rb") as f: file_content = f.read() - elif hasattr(content, 'read'): + elif hasattr(content, "read"): # File-like object file_content = content.read() - if hasattr(content, 'seek'): + if hasattr(content, "seek"): content.seek(0) else: raise ValueError(f"Unsupported content type in tuple: {type(content)}") else: raise ValueError("Tuple must have at least 2 elements: (filename, content)") - elif hasattr(audio_file, 'read') and not isinstance(audio_file, (str, bytes, bytearray, tuple, os.PathLike)): + elif hasattr(audio_file, "read") and not isinstance( + audio_file, (str, bytes, bytearray, tuple, os.PathLike) + ): # File-like object (IO) - check this after all other types - filename = getattr(audio_file, 'name', 'audio.wav') + filename = getattr(audio_file, "name", "audio.wav") file_content = audio_file.read() # type: ignore # Reset file pointer if possible - if hasattr(audio_file, 'seek'): + if hasattr(audio_file, "seek"): audio_file.seek(0) # type: ignore else: raise ValueError(f"Unsupported audio_file type: {type(audio_file)}") @@ -90,20 +94,18 @@ def process_audio_file(audio_file: FileTypes) -> ProcessedAudioFile: raise ValueError("Could not extract file content from audio_file") # Determine content type using LiteLLM's file type utilities - content_type = 'audio/wav' # Default fallback + content_type = "audio/wav" # Default fallback if filename: try: # Extract extension from filename - extension = filename.split('.')[-1].lower() if '.' in filename else 'wav' + extension = filename.split(".")[-1].lower() if "." in filename else "wav" content_type = get_file_mime_type_from_extension(extension) except ValueError: # If extension is not recognized, fallback to audio/wav - content_type = 'audio/wav' - + content_type = "audio/wav" + return ProcessedAudioFile( - file_content=file_content, - filename=filename, - content_type=content_type + file_content=file_content, filename=filename, content_type=content_type ) @@ -134,3 +136,74 @@ def get_audio_file_for_health_check() -> FileTypes: pwd = os.path.dirname(os.path.realpath(__file__)) file_path = os.path.join(pwd, "audio_health_check.wav") return open(file_path, "rb") + + +def calculate_request_duration(file: FileTypes) -> Optional[float]: + """ + Calculate audio duration from file content. + + Args: + file: The audio file (can be file path, bytes, or file-like object) + + Returns: + Duration in seconds, or None if extraction fails or soundfile is not available + """ + try: + import soundfile as sf + except ImportError: + # soundfile not available, cannot extract duration + return None + + try: + import io + + # Handle different file input types + file_content: Optional[bytes] = None + + if isinstance(file, (bytes, bytearray)): + # Raw bytes + file_content = bytes(file) + elif isinstance(file, (str, os.PathLike)): + # File path + with open(str(file), "rb") as f: + file_content = f.read() + elif isinstance(file, tuple): + # Tuple format: (filename, content, optional content_type) + if len(file) >= 2: + content = file[1] + if isinstance(content, bytes): + file_content = content + elif hasattr(content, "read") and not isinstance( + content, (str, os.PathLike) + ): + # File-like object in tuple + current_pos = getattr(content, "tell", lambda: None)() + # Seek to start to ensure we read the entire content + if hasattr(content, "seek"): + content.seek(0) + file_content = content.read() + if current_pos is not None and hasattr(content, "seek"): + content.seek(current_pos) + elif hasattr(file, "read") and not isinstance(file, tuple): + # File-like object (including BytesIO) + current_position = file.tell() if hasattr(file, "tell") else None + # Seek to start to ensure we read the entire content + if hasattr(file, "seek"): + file.seek(0) + file_content = file.read() + # Reset file position if possible + if current_position is not None and hasattr(file, "seek"): + file.seek(current_position) + + if file_content is None or not isinstance(file_content, bytes): + return None + + # Extract duration using soundfile + file_object = io.BytesIO(file_content) + with sf.SoundFile(file_object) as audio: + duration = len(audio) / audio.samplerate + return duration + + except Exception: + # Silently fail if duration extraction fails + return None diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 7423e55b626..47034c3a5c3 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -1,6 +1,6 @@ # What is this? ## Helper utilities -from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union +from typing import TYPE_CHECKING, Any, Iterable, List, Literal, Optional, Union import httpx @@ -138,6 +138,22 @@ def add_missing_spend_metadata_to_litellm_metadata( return litellm_metadata +def get_metadata_variable_name_from_kwargs( + kwargs: dict, +) -> Literal["metadata", "litellm_metadata"]: + """ + Helper to return what the "metadata" field should be called in the request data + + - New endpoints return `litellm_metadata` + - Old endpoints return `metadata` + + Context: + - LiteLLM used `metadata` as an internal field for storing metadata + - OpenAI then started using this field for their metadata + - LiteLLM is now moving to using `litellm_metadata` for our metadata + """ + return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" + def get_litellm_metadata_from_kwargs(kwargs: dict): """ Helper to get litellm metadata from all litellm request kwargs @@ -218,7 +234,8 @@ def preserve_upstream_non_openai_attributes( """ Preserve non-OpenAI attributes from the original chunk. """ - expected_keys = set(model_response.model_fields.keys()).union({"usage"}) + # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings + expected_keys = set(type(model_response).model_fields.keys()).union({"usage"}) for key, value in original_chunk.model_dump().items(): if key not in expected_keys: setattr(model_response, key, value) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index c6d3637ffcb..f8c786daef3 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -3,6 +3,7 @@ import traceback from typing import Any, Optional import httpx +import re import litellm from litellm._logging import verbose_logger @@ -12,6 +13,7 @@ from ..exceptions import ( APIConnectionError, APIError, AuthenticationError, + BadGatewayError, BadRequestError, ContentPolicyViolationError, ContextWindowExceededError, @@ -43,16 +45,23 @@ class ExceptionCheckers: """ if not isinstance(error_str, str): return False - - if "429" in error_str or "rate limit" in error_str.lower(): + + # Only treat 429 as a rate limit signal when it appears as a standalone token + if re.search(r"\b429\b", error_str): return True - + + _error_str_lower = error_str.lower() + + # Match "rate limit" (including variations like rate-limit / rate_limit) + if re.search(r"rate[\s_\-]*limit", _error_str_lower): + return True + ####################################### # Mistral API returns this error string ######################################### - if "service tier capacity exceeded" in error_str.lower(): + if "service tier capacity exceeded" in _error_str_lower: return True - + return False @staticmethod @@ -67,11 +76,30 @@ class ExceptionCheckers: "string too long. expected a string with maximum length", "model's maximum context limit", "is longer than the model's context length", + "input tokens exceed the configured limit", ] for substring in known_exception_substrings: if substring in _error_str_lowercase: return True return False + + @staticmethod + def is_azure_content_policy_violation_error(error_str: str) -> bool: + """ + Check if an error string indicates a content policy violation error. + """ + known_exception_substrings = [ + "invalid_request_error", + "content_policy_violation", + "the response was filtered due to the prompt triggering azure openai's content management", + "your task failed as a result of our safety system", + "the model produced invalid content", + "content_filter_policy", + ] + for substring in known_exception_substrings: + if substring in error_str.lower(): + return True + return False def get_error_message(error_obj) -> Optional[str]: @@ -135,9 +163,6 @@ def _get_response_headers(original_exception: Exception) -> Optional[httpx.Heade return _response_headers -import re - - def extract_and_raise_litellm_exception( response: Optional[Any], error_str: str, @@ -506,6 +531,15 @@ def exception_type( # type: ignore # noqa: PLR0915 response=getattr(original_exception, "response", None), litellm_debug_info=extra_information, ) + elif original_exception.status_code == 502: + exception_mapping_worked = True + raise BadGatewayError( + message=f"BadGatewayError: {exception_provider} - {message}", + model=model, + llm_provider=custom_llm_provider, + response=getattr(original_exception, "response", None), + litellm_debug_info=extra_information, + ) elif original_exception.status_code == 503: exception_mapping_worked = True raise ServiceUnavailableError( @@ -636,6 +670,15 @@ def exception_type( # type: ignore # noqa: PLR0915 message=f"AnthropicException - {error_str}. Handle with `litellm.InternalServerError`.", llm_provider="anthropic", model=model, + response=getattr(original_exception, "response", None), + ) + elif original_exception.status_code == 502: + exception_mapping_worked = True + raise BadGatewayError( + message=f"AnthropicException BadGatewayError - {error_str}", + llm_provider="anthropic", + model=model, + response=getattr(original_exception, "response", None), ) elif original_exception.status_code == 503: exception_mapping_worked = True @@ -643,6 +686,15 @@ def exception_type( # type: ignore # noqa: PLR0915 message=f"AnthropicException - {error_str}. Handle with `litellm.ServiceUnavailableError`.", llm_provider="anthropic", model=model, + response=getattr(original_exception, "response", None), + ) + elif original_exception.status_code == 504: # gateway timeout error + exception_mapping_worked = True + raise Timeout( + message=f"AnthropicException Timeout - {error_str}", + model=model, + llm_provider="anthropic", + exception_status_code=original_exception.status_code, ) elif custom_llm_provider == "replicate": if "Incorrect authentication token" in error_str: @@ -1259,6 +1311,7 @@ def exception_type( # type: ignore # noqa: PLR0915 elif ( "429 Quota exceeded" in error_str or "Quota exceeded for" in error_str + or "Resource exhausted" in error_str or "IndexError: list index out of range" in error_str or "429 Unable to submit request because the service is temporarily out of capacity." in error_str @@ -1991,26 +2044,19 @@ def exception_type( # type: ignore # noqa: PLR0915 response=getattr(original_exception, "response", None), ) elif ( - ( - "invalid_request_error" in error_str - and "content_policy_violation" in error_str - ) - or ( - "The response was filtered due to the prompt triggering Azure OpenAI's content management" - in error_str - ) - or "Your task failed as a result of our safety system" in error_str - or "The model produced invalid content" in error_str - or "content_filter_policy" in error_str + ExceptionCheckers.is_azure_content_policy_violation_error(error_str) ): exception_mapping_worked = True - raise ContentPolicyViolationError( - message=f"litellm.ContentPolicyViolationError: AzureException - {message}", - llm_provider="azure", - model=model, - litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + from litellm.llms.azure.exception_mapping import ( + AzureOpenAIExceptionMapping, ) + raise AzureOpenAIExceptionMapping.create_content_policy_violation_error( + message=message, + model=model, + extra_information=extra_information, + original_exception=original_exception, + ) + elif "invalid_request_error" in error_str: exception_mapping_worked = True raise BadRequestError( @@ -2088,6 +2134,15 @@ def exception_type( # type: ignore # noqa: PLR0915 litellm_debug_info=extra_information, response=getattr(original_exception, "response", None), ) + elif original_exception.status_code == 502: + exception_mapping_worked = True + raise BadGatewayError( + message=f"AzureException BadGatewayError - {message}", + model=model, + llm_provider="azure", + litellm_debug_info=extra_information, + response=getattr(original_exception, "response", None), + ) elif original_exception.status_code == 503: exception_mapping_worked = True raise ServiceUnavailableError( diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index c167c202e5d..d5675a2ac51 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -120,5 +120,6 @@ def get_litellm_params( "vertex_project": kwargs.get("vertex_project"), "use_litellm_proxy": use_litellm_proxy, "litellm_request_debug": litellm_request_debug, + "aws_region_name": kwargs.get("aws_region_name"), } return litellm_params diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index f209aed483c..fb25c5ed840 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -279,6 +279,7 @@ def get_llm_provider( # noqa: PLR0915 or "ft:gpt-3.5-turbo" in model or "ft:gpt-4" in model # catches ft:gpt-4-0613, ft:gpt-4o or model in litellm.openai_image_generation_models + or model in litellm.openai_video_generation_models ): custom_llm_provider = "openai" elif model in litellm.open_ai_text_completion_models: @@ -383,6 +384,8 @@ def get_llm_provider( # noqa: PLR0915 custom_llm_provider = "ovhcloud" elif model.startswith("lemonade/"): custom_llm_provider = "lemonade" + elif model.startswith("clarifai/"): + custom_llm_provider = "clarifai" if not custom_llm_provider: if litellm.suppress_debug_info is False: print() # noqa @@ -794,6 +797,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.LemonadeChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "clarifai": + ( + api_base, + dynamic_api_key, + ) = litellm.ClarifaiConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) if api_base is not None and not isinstance(api_base, str): raise Exception("api base needs to be a string. api_base={}".format(api_base)) diff --git a/litellm/litellm_core_utils/get_provider_specific_headers.py b/litellm/litellm_core_utils/get_provider_specific_headers.py index cf9165cfda9..69a7ec72073 100644 --- a/litellm/litellm_core_utils/get_provider_specific_headers.py +++ b/litellm/litellm_core_utils/get_provider_specific_headers.py @@ -10,14 +10,20 @@ class ProviderSpecificHeaderUtils: custom_llm_provider: Optional[str], ) -> Dict: """ - Get the provider specific headers for the given custom llm provider + Get the provider specific headers for the given custom llm provider. + + Supports comma-separated provider lists for headers that work across multiple providers. Returns: - Optional[Dict]: The provider specific headers for the given custom llm provider + Dict: The provider specific headers for the given custom llm provider """ - if ( - provider_specific_header is not None - and provider_specific_header.get("custom_llm_provider") == custom_llm_provider - ): + if provider_specific_header is None or custom_llm_provider is None: + return {} + + stored_providers = provider_specific_header.get("custom_llm_provider", "") + provider_list = [p.strip() for p in stored_providers.split(",")] + + if custom_llm_provider in provider_list: return provider_specific_header.get("extra_headers", {}) - return {} \ No newline at end of file + + return {} diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 2f412479937..cc3916af069 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -2,11 +2,14 @@ Helper functions for health check calls. """ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Callable, Dict, Literal, Optional if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging +# Minimal PDF for health checks - base64 encoded 1-page PDF with just "test" +TEST_PDF_URL = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y=" + class HealthCheckHelpers: @@ -78,3 +81,113 @@ class HealthCheckHelpers: return { "tags": [LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME], } + + @staticmethod + def get_mode_handlers( + model: str, + custom_llm_provider: str, + model_params: dict, + prompt: Optional[str] = None, + input: Optional[list] = None, + ) -> Dict[ + Literal[ + "chat", + "completion", + "embedding", + "audio_speech", + "audio_transcription", + "image_generation", + "video_generation", + "rerank", + "realtime", + "batch", + "responses", + "ocr", + ], + Callable, + ]: + """ + Returns a dictionary of mode handlers for health check calls. + + Mode Handlers are Callables that need to be run for execution of the health check call. + + Args: + model: The model name + custom_llm_provider: The LLM provider + model_params: The model parameters + prompt: Optional prompt for health check + input: Optional input for health check + + Returns: + Dictionary mapping mode names to their handler functions + """ + import litellm + from litellm.litellm_core_utils.audio_utils.utils import ( + get_audio_file_for_health_check, + ) + from litellm.litellm_core_utils.health_check_utils import _filter_model_params + from litellm.realtime_api.main import _realtime_health_check + + return { + "chat": lambda: litellm.acompletion( + **model_params, + ), + "completion": lambda: litellm.atext_completion( + **_filter_model_params(model_params=model_params), + prompt=prompt or "test", + ), + "embedding": lambda: litellm.aembedding( + **_filter_model_params(model_params=model_params), + input=input or ["test"], + ), + "audio_speech": lambda: litellm.aspeech( + **{ + **_filter_model_params(model_params=model_params), + **( + {"voice": "alloy"} + if "voice" + not in _filter_model_params(model_params=model_params) + else {} + ), + }, + input=prompt or "test", + ), + "audio_transcription": lambda: litellm.atranscription( + **_filter_model_params(model_params=model_params), + file=get_audio_file_for_health_check(), + ), + "image_generation": lambda: litellm.aimage_generation( + **_filter_model_params(model_params=model_params), + prompt=prompt, + ), + "video_generation": lambda: litellm.avideo_generation( + **_filter_model_params(model_params=model_params), + prompt=prompt or "test video generation", + ), + "rerank": lambda: litellm.arerank( + **_filter_model_params(model_params=model_params), + query=prompt or "", + documents=["my sample text"], + ), + "realtime": lambda: _realtime_health_check( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=model_params.get("api_base", None), + api_key=model_params.get("api_key", None), + api_version=model_params.get("api_version", None), + ), + "batch": lambda: litellm.alist_batches( + **_filter_model_params(model_params=model_params), + ), + "responses": lambda: litellm.aresponses( + **_filter_model_params(model_params=model_params), + input=prompt or "test", + ), + "ocr": lambda: litellm.aocr( + **_filter_model_params(model_params=model_params), + document={ + "type": "document_url", + "document_url": TEST_PDF_URL, + }, + ), + } \ No newline at end of file diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index eafcab88557..41a5eed55d8 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -68,7 +68,9 @@ from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_custom_logger, redact_message_input_output_from_logging, ) +from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.responses.utils import ResponseAPILoggingUtils +from litellm.types.containers.main import ContainerObject from litellm.types.llms.openai import ( AllMessageValues, Batch, @@ -76,6 +78,7 @@ from litellm.types.llms.openai import ( HttpxBinaryResponseContent, OpenAIFileObject, OpenAIModerationResponse, + ResponseAPIUsage, ResponseCompletedEvent, ResponsesAPIResponse, ) @@ -115,6 +118,7 @@ from litellm.types.utils import ( TranscriptionResponse, Usage, ) +from litellm.types.videos.main import VideoObject from litellm.utils import _get_base_model_from_metadata, executor, print_verbose from ..integrations.argilla import ArgillaLogger @@ -699,6 +703,14 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["prompt_integration"] = ( vector_store_custom_logger.__class__.__name__ ) + # Add to global callbacks so post-call hooks are invoked + if ( + vector_store_custom_logger + and vector_store_custom_logger not in litellm.callbacks + ): + litellm.logging_callback_manager.add_litellm_callback( + vector_store_custom_logger + ) return vector_store_custom_logger return None @@ -1171,6 +1183,9 @@ class Logging(LiteLLMLoggingBaseClass): output_cost: float, total_cost: float, cost_for_built_in_tools_cost_usd_dollar: float, + original_cost: Optional[float] = None, + discount_percent: Optional[float] = None, + discount_amount: Optional[float] = None, ) -> None: """ Helper method to store cost breakdown in the logging object. @@ -1180,6 +1195,9 @@ class Logging(LiteLLMLoggingBaseClass): output_cost: Cost of output/completion tokens cost_for_built_in_tools_cost_usd_dollar: Cost of built-in tools total_cost: Total cost of request + original_cost: Cost before discount + discount_percent: Discount percentage (0.05 = 5%) + discount_amount: Discount amount in USD """ self.cost_breakdown = CostBreakdown( @@ -1188,9 +1206,14 @@ class Logging(LiteLLMLoggingBaseClass): total_cost=total_cost, tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar, ) - verbose_logger.debug( - f"Cost breakdown set - input: {input_cost}, output: {output_cost}, cost_for_built_in_tools_cost_usd_dollar: {cost_for_built_in_tools_cost_usd_dollar}, total: {total_cost}" - ) + + # Store discount information if provided + if original_cost is not None: + self.cost_breakdown["original_cost"] = original_cost + if discount_percent is not None: + self.cost_breakdown["discount_percent"] = discount_percent + if discount_amount is not None: + self.cost_breakdown["discount_amount"] = discount_amount def _response_cost_calculator( self, @@ -1220,6 +1243,7 @@ class Logging(LiteLLMLoggingBaseClass): used for consistent cost calculation across response headers + logging integrations. """ + if isinstance(result, BaseModel) and hasattr(result, "_hidden_params"): hidden_params = getattr(result, "_hidden_params", {}) if ( @@ -1288,6 +1312,7 @@ class Logging(LiteLLMLoggingBaseClass): response_cost = litellm.response_cost_calculator( **response_cost_calculator_kwargs ) + verbose_logger.debug(f"response_cost: {response_cost}") return response_cost except Exception as e: # error calculating cost @@ -1437,6 +1462,51 @@ class Logging(LiteLLMLoggingBaseClass): ) return logging_result + def _process_hidden_params_and_response_cost( + self, + logging_result, + start_time, + end_time, + ): + hidden_params = getattr(logging_result, "_hidden_params", {}) + if hidden_params: + if self.model_call_details.get("litellm_params") is not None: + self.model_call_details["litellm_params"].setdefault("metadata", {}) + if self.model_call_details["litellm_params"]["metadata"] is None: + self.model_call_details["litellm_params"]["metadata"] = {} + self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = getattr(logging_result, "_hidden_params", {}) # type: ignore + + if "response_cost" in hidden_params: + self.model_call_details["response_cost"] = hidden_params["response_cost"] + else: + self.model_call_details["response_cost"] = self._response_cost_calculator(result=logging_result) + + self.model_call_details["standard_logging_object"] = get_standard_logging_object_payload( + kwargs=self.model_call_details, + init_response_obj=logging_result, + start_time=start_time, + end_time=end_time, + logging_obj=self, + status="success", + standard_built_in_tools_params=self.standard_built_in_tools_params, + ) + + def _transform_usage_objects(self, result): + if isinstance(result, ResponsesAPIResponse): + result = result.model_copy() + transformed_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(result.usage) + setattr(result, "usage", transformed_usage.model_dump() if hasattr(transformed_usage, "model_dump") else dict(transformed_usage)) + if (standard_logging_payload := self.model_call_details.get("standard_logging_object")) is not None: + standard_logging_payload["response"] = result.model_dump() if hasattr(result, "model_dump") else dict(result) + elif isinstance(result, TranscriptionResponse): + from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( + TranscriptionUsageObjectTransformation, + ) + result = result.model_copy() + transformed_usage = TranscriptionUsageObjectTransformation.transform_transcription_usage_object(result.usage) # type: ignore + setattr(result, "usage", transformed_usage) + return result + def _success_handler_helper_fn( self, result=None, @@ -1452,116 +1522,40 @@ class Logging(LiteLLMLoggingBaseClass): end_time = datetime.datetime.now() if self.completion_start_time is None: self.completion_start_time = end_time - self.model_call_details["completion_start_time"] = ( - self.completion_start_time - ) + self.model_call_details["completion_start_time"] = self.completion_start_time + self.model_call_details["log_event_type"] = "successful_api_call" self.model_call_details["end_time"] = end_time self.model_call_details["cache_hit"] = cache_hit + if self.call_type == CallTypes.anthropic_messages.value: result = self._handle_anthropic_messages_response_logging(result=result) - elif ( - self.call_type == CallTypes.generate_content.value - or self.call_type == CallTypes.agenerate_content.value - ): - result = self._handle_non_streaming_google_genai_generate_content_response_logging( - result=result - ) - ## if model in model cost map - log the response cost - ## else set cost to None - + elif self.call_type == CallTypes.generate_content.value or self.call_type == CallTypes.agenerate_content.value: + result = self._handle_non_streaming_google_genai_generate_content_response_logging(result=result) + logging_result = self.normalize_logging_result(result=result) - if ( - standard_logging_object is None - and result is not None - and self.stream is not True - ): - if self._is_recognized_call_type_for_logging( - logging_result=logging_result - ): - ## HIDDEN PARAMS ## - hidden_params = getattr(logging_result, "_hidden_params", {}) - if hidden_params: - # add to metadata for logging - if self.model_call_details.get("litellm_params") is not None: - self.model_call_details["litellm_params"].setdefault( - "metadata", {} - ) - if ( - self.model_call_details["litellm_params"]["metadata"] - is None - ): - self.model_call_details["litellm_params"][ - "metadata" - ] = {} - - self.model_call_details["litellm_params"]["metadata"][ # type: ignore - "hidden_params" - ] = getattr( - logging_result, "_hidden_params", {} - ) - ## RESPONSE COST - Only calculate if not in hidden_params ## - if "response_cost" in hidden_params: - self.model_call_details["response_cost"] = hidden_params[ - "response_cost" - ] - else: - self.model_call_details["response_cost"] = ( - self._response_cost_calculator(result=logging_result) - ) - ## STANDARDIZED LOGGING PAYLOAD - - self.model_call_details["standard_logging_object"] = ( - get_standard_logging_object_payload( - kwargs=self.model_call_details, - init_response_obj=logging_result, - start_time=start_time, - end_time=end_time, - logging_obj=self, - status="success", - standard_built_in_tools_params=self.standard_built_in_tools_params, - ) - ) + if standard_logging_object is None and result is not None and self.stream is not True: + if self._is_recognized_call_type_for_logging(logging_result=logging_result): + self._process_hidden_params_and_response_cost(logging_result=logging_result, start_time=start_time, end_time=end_time) elif isinstance(result, dict) or isinstance(result, list): - ## STANDARDIZED LOGGING PAYLOAD - self.model_call_details["standard_logging_object"] = ( - get_standard_logging_object_payload( - kwargs=self.model_call_details, - init_response_obj=result, - start_time=start_time, - end_time=end_time, - logging_obj=self, - status="success", - standard_built_in_tools_params=self.standard_built_in_tools_params, - ) + self.model_call_details["standard_logging_object"] = get_standard_logging_object_payload( + kwargs=self.model_call_details, + init_response_obj=result, + start_time=start_time, + end_time=end_time, + logging_obj=self, + status="success", + standard_built_in_tools_params=self.standard_built_in_tools_params, ) elif standard_logging_object is not None: - self.model_call_details["standard_logging_object"] = ( - standard_logging_object - ) - else: # streaming chunks + image gen. + self.model_call_details["standard_logging_object"] = standard_logging_object + else: self.model_call_details["response_cost"] = None - ## RESPONSES API USAGE OBJECT TRANSFORMATION ## - # MAP RESPONSES API USAGE OBJECT TO LITELLM USAGE OBJECT - if isinstance(result, ResponsesAPIResponse): - result = result.model_copy() - setattr( - result, - "usage", - ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( - result.usage - ), - ) - - if ( - litellm.max_budget - and self.stream is False - and result is not None - and isinstance(result, dict) - and "content" in result - ): + result = self._transform_usage_objects(result=result) + + if litellm.max_budget and self.stream is False and result is not None and isinstance(result, dict) and "content" in result: time_diff = (end_time - start_time).total_seconds() float_diff = float(time_diff) litellm._current_cost += litellm.completion_cost( @@ -1598,6 +1592,11 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(logging_result, OpenAIFileObject) or isinstance(logging_result, LiteLLMRealtimeStreamLoggingObject) or isinstance(logging_result, OpenAIModerationResponse) + or isinstance(logging_result, OCRResponse) # OCR + or isinstance(logging_result, dict) + and logging_result.get("object") == "vector_store.search_results.page" + or isinstance(logging_result, VideoObject) + or isinstance(logging_result, ContainerObject) or (self.call_type == CallTypes.call_mcp_tool.value) ): return True @@ -2133,6 +2132,11 @@ class Logging(LiteLLMLoggingBaseClass): ) if capture_exception: # log this error to sentry for debugging capture_exception(e) + # Track callback logging failures in Prometheus + try: + self._handle_callback_failure(callback=callback) + except Exception: + pass except Exception as e: verbose_logger.exception( "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging {}".format( @@ -2438,8 +2442,31 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.error( f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while success logging {traceback.format_exc()}" ) + self._handle_callback_failure(callback=callback) pass + def _handle_callback_failure(self, callback: Any): + """ + Handle callback logging failures by incrementing Prometheus metrics. + + Works for both sync and async contexts since Prometheus counter increment is synchronous. + + Args: + callback: The callback that failed + """ + try: + callback_name = self._get_callback_name(callback) + + all_callbacks = litellm.logging_callback_manager._get_all_callbacks() + + for callback_obj in all_callbacks: + if hasattr(callback_obj, "increment_callback_logging_failure"): + callback_obj.increment_callback_logging_failure(callback_name=callback_name) # type: ignore + break # Only increment once + + except Exception as e: + verbose_logger.debug(f"Error in _handle_callback_failure: {str(e)}") + def _failure_handler_helper_fn( self, exception, traceback_exception, start_time=None, end_time=None ): @@ -2775,6 +2802,8 @@ class Logging(LiteLLMLoggingBaseClass): str(e), callback ) ) + # Track callback logging failures in Prometheus + self._handle_callback_failure(callback=callback) def _get_trace_id(self, service_name: Literal["langfuse"]) -> Optional[str]: """ @@ -2907,15 +2936,19 @@ class Logging(LiteLLMLoggingBaseClass): Helper to get the name of a callback function Args: - cb: The callback function/string to get the name of + cb: The callback object/function/string to get the name of Returns: The name of the callback """ + if isinstance(cb, str): + return cb if hasattr(cb, "__name__"): return cb.__name__ if hasattr(cb, "__func__"): return cb.__func__.__name__ + if hasattr(cb, "__class__"): + return cb.__class__.__name__ return str(cb) def _is_internal_litellm_proxy_callback(self, cb) -> bool: @@ -2969,6 +3002,23 @@ class Logging(LiteLLMLoggingBaseClass): elif isinstance(result, TextCompletionResponse): return result elif isinstance(result, ResponseCompletedEvent): + ## return unified Usage object + if isinstance(result.response.usage, ResponseAPIUsage): + transformed_usage = ( + ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( + result.response.usage + ) + ) + # Set as dict instead of Usage object so model_dump() serializes it correctly + setattr( + result.response, + "usage", + ( + transformed_usage.model_dump() + if hasattr(transformed_usage, "model_dump") + else dict(transformed_usage) + ), + ) return result.response else: return None @@ -3149,6 +3199,7 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915 event_scrubber=EventScrubber( denylist=SENTRY_DENYLIST, pii_denylist=SENTRY_PII_DENYLIST ), + environment=os.environ.get("SENTRY_ENVIRONMENT", "production"), ) capture_exception = sentry_sdk_instance.capture_exception add_breadcrumb = sentry_sdk_instance.add_breadcrumb @@ -4264,8 +4315,8 @@ class StandardLoggingPayloadSetup: from litellm.integrations.s3 import get_s3_object_key # Only generate object key if cold storage is configured - configured_cold_storage_logger = litellm.configured_cold_storage_logger - if configured_cold_storage_logger is None: + cold_storage_custom_logger = litellm.cold_storage_custom_logger + if cold_storage_custom_logger is None: return None try: @@ -4278,7 +4329,7 @@ class StandardLoggingPayloadSetup: # Try to get the actual logger instance from the logger name try: custom_logger = litellm.logging_callback_manager.get_active_custom_logger_for_callback_name( - configured_cold_storage_logger + cold_storage_custom_logger ) if ( custom_logger @@ -4292,7 +4343,7 @@ class StandardLoggingPayloadSetup: s3_object_key = get_s3_object_key( s3_path=s3_path, # Use actual s3_path from logger configuration - team_alias_prefix="", # Don't split by team alias for cold storage + prefix="", # Don't split by team alias for cold storage start_time=start_time, s3_file_name=s3_file_name, ) @@ -4428,12 +4479,18 @@ class StandardLoggingPayloadSetup: return header_tags if header_tags else None @staticmethod - def _get_request_tags(metadata: dict, proxy_server_request: dict) -> List[str]: - request_tags = ( - metadata.get("tags", []) - if isinstance(metadata.get("tags", []), list) - else [] - ) + def _get_request_tags( + litellm_params: dict, proxy_server_request: dict + ) -> List[str]: + # check for 'tags' in both 'metadata' and 'litellm_metadata' + metadata = litellm_params.get("metadata") or {} + litellm_metadata = litellm_params.get("litellm_metadata") or {} + if metadata.get("tags", []): + request_tags = metadata.get("tags", []) + elif litellm_metadata.get("tags", []): + request_tags = litellm_metadata.get("tags", []) + else: + request_tags = [] user_agent_tags = StandardLoggingPayloadSetup._get_user_agent_tags( proxy_server_request ) @@ -4447,20 +4504,19 @@ class StandardLoggingPayloadSetup: return request_tags - def _get_status_fields( status: StandardLoggingPayloadStatus, - guardrail_information: Optional[dict], - error_str: Optional[str] + guardrail_information: Optional[List[dict]], + error_str: Optional[str], ) -> "StandardLoggingPayloadStatusFields": """ Determine status fields based on request status and guardrail information. - + Args: status: Overall request status ("success" or "failure") guardrail_information: Guardrail information from metadata error_str: Error string if any - + Returns: StandardLoggingPayloadStatusFields with llm_api_status and guardrail_status """ @@ -4471,24 +4527,26 @@ def _get_status_fields( "guardrail_intervened": "guardrail_intervened", # direct "failure": "guardrail_failed_to_respond", # legacy "guardrail_failed_to_respond": "guardrail_failed_to_respond", # direct - "not_run": "not_run" + "not_run": "not_run", } - + # Set LLM API status llm_api_status: StandardLoggingPayloadStatus = status - ######################################################### # Map - guardrail_information.guardrail_status to guardrail_status ######################################################### guardrail_status: GuardrailStatus = "not_run" - if guardrail_information and isinstance(guardrail_information, dict): - raw_status = guardrail_information.get("guardrail_status", "not_run") - guardrail_status = GUARDRAIL_STATUS_MAP.get(raw_status, "not_run") + if guardrail_information and isinstance(guardrail_information, list): + for information in guardrail_information: + if isinstance(information, dict): + raw_status = information.get("guardrail_status", "not_run") + if raw_status != "not_run": + guardrail_status = GUARDRAIL_STATUS_MAP.get(raw_status, "not_run") + break return StandardLoggingPayloadStatusFields( - llm_api_status=llm_api_status, - guardrail_status=guardrail_status + llm_api_status=llm_api_status, guardrail_status=guardrail_status ) @@ -4537,7 +4595,7 @@ def get_standard_logging_object_payload( ) # standardize this function to be used across, s3, dynamoDB, langfuse logging - litellm_params = kwargs.get("litellm_params", {}) + litellm_params = kwargs.get("litellm_params", {}) or {} proxy_server_request = litellm_params.get("proxy_server_request") or {} metadata: dict = ( @@ -4562,7 +4620,7 @@ def get_standard_logging_object_payload( _model_group = metadata.get("model_group", "") request_tags = StandardLoggingPayloadSetup._get_request_tags( - metadata=metadata, proxy_server_request=proxy_server_request + litellm_params=litellm_params, proxy_server_request=proxy_server_request ) # cleanup timestamps @@ -4658,8 +4716,10 @@ def get_standard_logging_object_payload( status=status, status_fields=_get_status_fields( status=status, - guardrail_information=metadata.get("standard_logging_guardrail_information", None), - error_str=error_str + guardrail_information=metadata.get( + "standard_logging_guardrail_information", None + ), + error_str=error_str, ), custom_llm_provider=cast(Optional[str], kwargs.get("custom_llm_provider")), saved_cache_cost=saved_cache_cost, diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index b6113661777..4a4a2508d2e 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -314,9 +314,23 @@ class StandardBuiltInToolCostTracking: if isinstance(response_object, ModelResponse): # chat completions only include url_citation annotations when a web search call is made - return StandardBuiltInToolCostTracking.response_includes_annotation_type( + has_url_citations = StandardBuiltInToolCostTracking.response_includes_annotation_type( response_object=response_object, annotation_type="url_citation" ) + if has_url_citations: + return True + # Fallback: Check usage object for providers that use usage instead of annotations + # (e.g., Vertex AI Gemini uses usage.prompt_tokens_details.web_search_requests) + if usage is not None: + if ( + hasattr(usage, "prompt_tokens_details") + and usage.prompt_tokens_details is not None + and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper) + and hasattr(usage.prompt_tokens_details, "web_search_requests") + and usage.prompt_tokens_details.web_search_requests is not None + ): + return True + return False elif isinstance(response_object, ResponsesAPIResponse): # response api explicitly includes web_search_call in the output return StandardBuiltInToolCostTracking.response_includes_output_type( @@ -581,6 +595,31 @@ class StandardBuiltInToolCostTracking: # OpenAI doesn't charge separately for computer use yet return 0.0 + @staticmethod + def _get_code_interpreter_cost_from_model_map( + provider: str, + ) -> Optional[float]: + """ + Get code interpreter cost per session from model cost map. + """ + import litellm + + try: + container_model = f"{provider}/container" + model_info = litellm.get_model_info( + model=container_model, + custom_llm_provider=provider + ) + model_key = model_info.get("key") if isinstance(model_info, dict) else getattr(model_info, "key", None) + + if model_key and model_key in litellm.model_cost: + return litellm.model_cost[model_key].get("code_interpreter_cost_per_session") + + except Exception: + pass + + return None + @staticmethod def get_cost_for_code_interpreter( sessions: Optional[int] = None, @@ -590,7 +629,8 @@ class StandardBuiltInToolCostTracking: """ Calculate cost for code interpreter feature. - Azure: $0.03 USD per session + Azure: $0.03 USD per session (from model cost map) + OpenAI: $0.03 USD per session (from model cost map) """ if sessions is None or sessions == 0: return 0.0 @@ -599,13 +639,15 @@ class StandardBuiltInToolCostTracking: if model_info and "code_interpreter_cost_per_session" in model_info: return sessions * model_info["code_interpreter_cost_per_session"] - # Azure pricing for code interpreter - if provider == "azure": - from litellm.constants import AZURE_CODE_INTERPRETER_COST_PER_SESSION + # Try to get cost from model cost map for any provider + if provider: + cost_per_session = StandardBuiltInToolCostTracking._get_code_interpreter_cost_from_model_map( + provider=provider + ) + if cost_per_session is not None: + return sessions * cost_per_session + - return sessions * AZURE_CODE_INTERPRETER_COST_PER_SESSION - - # OpenAI doesn't charge separately for code interpreter yet return 0.0 @staticmethod diff --git a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py new file mode 100644 index 00000000000..1432e912fd8 --- /dev/null +++ b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py @@ -0,0 +1,38 @@ +from typing import Any, Optional, Union + +from litellm.types.utils import ( + PromptTokensDetailsWrapper, + TranscriptionUsageDurationObject, + TranscriptionUsageTokensObject, + Usage, +) + + +class TranscriptionUsageObjectTransformation: + @staticmethod + def is_transcription_usage_object( + usage_object: Any, + ) -> bool: + return isinstance(usage_object, TranscriptionUsageDurationObject) or isinstance( + usage_object, TranscriptionUsageTokensObject + ) + + @staticmethod + def transform_transcription_usage_object( + usage_object: Union[ + TranscriptionUsageDurationObject, TranscriptionUsageTokensObject + ], + ) -> Optional[Usage]: + if isinstance(usage_object, TranscriptionUsageDurationObject): + return None + elif isinstance(usage_object, TranscriptionUsageTokensObject): + return Usage( + prompt_tokens=usage_object.input_tokens, + completion_tokens=usage_object.output_tokens, + total_tokens=usage_object.total_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=usage_object.input_token_details.text_tokens, + audio_tokens=usage_object.input_token_details.audio_tokens, + ), + ) + return None diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 626a3f3625f..eff5376e49e 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1,7 +1,7 @@ # What is this? ## Helper utilities for cost_per_token() -from typing import Any, Literal, Optional, Tuple, TypedDict, cast +from typing import Literal, Optional, Tuple, TypedDict, cast import litellm from litellm._logging import verbose_logger @@ -11,8 +11,8 @@ from litellm.types.utils import ( ImageResponse, ModelInfo, PassthroughCallTypes, - Usage, ServiceTier, + Usage, ) from litellm.utils import get_model_info @@ -118,21 +118,21 @@ def _generic_cost_per_character( def _get_service_tier_cost_key(base_key: str, service_tier: Optional[str]) -> str: """ Get the appropriate cost key based on service tier. - + Args: base_key: The base cost key (e.g., "input_cost_per_token") service_tier: The service tier ("flex", "priority", or None for standard) - + Returns: str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token") """ if service_tier is None: return base_key - + # Only use service tier specific keys for "flex" and "priority" if service_tier.lower() in [ServiceTier.FLEX.value, ServiceTier.PRIORITY.value]: return f"{base_key}_{service_tier.lower()}" - + # For any other service tier, use standard pricing return base_key @@ -152,15 +152,15 @@ def _get_token_base_cost( # Get service tier aware cost keys input_cost_key = _get_service_tier_cost_key("input_cost_per_token", service_tier) output_cost_key = _get_service_tier_cost_key("output_cost_per_token", service_tier) - cache_creation_cost_key = _get_service_tier_cost_key("cache_creation_input_token_cost", service_tier) - cache_read_cost_key = _get_service_tier_cost_key("cache_read_input_token_cost", service_tier) - - prompt_base_cost = cast( - float, _get_cost_per_unit(model_info, input_cost_key) + cache_creation_cost_key = _get_service_tier_cost_key( + "cache_creation_input_token_cost", service_tier ) - completion_base_cost = cast( - float, _get_cost_per_unit(model_info, output_cost_key) + cache_read_cost_key = _get_service_tier_cost_key( + "cache_read_input_token_cost", service_tier ) + + prompt_base_cost = cast(float, _get_cost_per_unit(model_info, input_cost_key)) + completion_base_cost = cast(float, _get_cost_per_unit(model_info, output_cost_key)) cache_creation_cost = cast( float, _get_cost_per_unit(model_info, cache_creation_cost_key) ) @@ -168,9 +168,7 @@ def _get_token_base_cost( float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"), ) - cache_read_cost = cast( - float, _get_cost_per_unit(model_info, cache_read_cost_key) - ) + cache_read_cost = cast(float, _get_cost_per_unit(model_info, cache_read_cost_key)) ## CHECK IF ABOVE THRESHOLD threshold: Optional[float] = None @@ -278,7 +276,7 @@ def _get_cost_per_unit( verbose_logger.exception( f"litellm.litellm_core_utils.llm_cost_calc.utils.py::calculate_cost_per_component(): Exception occured - {cost_per_unit}\nDefaulting to 0.0" ) - + # If the service tier key doesn't exist or is None, try to fall back to the standard key if cost_per_unit is None: # Check if any service tier suffix exists in the cost key using ServiceTier enum @@ -286,7 +284,7 @@ def _get_cost_per_unit( suffix = f"_{service_tier.value}" if suffix in cost_key: # Extract the base key by removing the matched suffix - base_key = cost_key.replace(suffix, '') + base_key = cost_key.replace(suffix, "") fallback_cost = model_info.get(base_key) if isinstance(fallback_cost, float): return fallback_cost @@ -300,7 +298,7 @@ def _get_cost_per_unit( f"litellm.litellm_core_utils.llm_cost_calc.utils.py::_get_cost_per_unit(): Exception occured - {fallback_cost}\nDefaulting to 0.0" ) break # Only try the first matching suffix - + return default_value @@ -495,7 +493,10 @@ def _calculate_input_cost( def generic_cost_per_token( - model: str, usage: Usage, custom_llm_provider: str, service_tier: Optional[str] = None + model: str, + usage: Usage, + custom_llm_provider: str, + service_tier: Optional[str] = None, ) -> Tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -547,7 +548,9 @@ def generic_cost_per_token( cache_creation_cost, cache_creation_cost_above_1hr, cache_read_cost, - ) = _get_token_base_cost(model_info=model_info, usage=usage, service_tier=service_tier) + ) = _get_token_base_cost( + model_info=model_info, usage=usage, service_tier=service_tier + ) prompt_cost = _calculate_input_cost( prompt_tokens_details=prompt_tokens_details, @@ -631,12 +634,13 @@ class CostCalculatorUtils: @staticmethod def route_image_generation_cost_calculator( model: str, - completion_response: Any, + completion_response: ImageResponse, custom_llm_provider: Optional[str] = None, quality: Optional[str] = None, n: Optional[int] = None, size: Optional[str] = None, optional_params: Optional[dict] = None, + call_type: Optional[str] = None, ) -> float: """ Route the image generation cost calculator based on the custom_llm_provider @@ -658,6 +662,13 @@ class CostCalculatorUtils: cost_calculator as vertex_ai_image_cost_calculator, ) + if size is None: + size = completion_response.size or "1024-x-1024" + if quality is None: + quality = completion_response.quality or "standard" + if n is None: + n = len(completion_response.data) if completion_response.data else 0 + if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value: if isinstance(completion_response, ImageResponse): return vertex_ai_image_cost_calculator( @@ -693,7 +704,28 @@ class CostCalculatorUtils: model=model, image_response=completion_response, ) + elif custom_llm_provider == litellm.LlmProviders.COMETAPI.value: + from litellm.llms.cometapi.image_generation.cost_calculator import ( + cost_calculator as cometapi_image_cost_calculator, + ) + + return cometapi_image_cost_calculator( + model=model, + image_response=completion_response, + ) elif custom_llm_provider == litellm.LlmProviders.GEMINI.value: + if call_type in ( + CallTypes.image_edit.value, + CallTypes.aimage_edit.value, + ): + from litellm.llms.gemini.image_edit.cost_calculator import ( + cost_calculator as gemini_image_edit_cost_calculator, + ) + + return gemini_image_edit_cost_calculator( + model=model, + image_response=completion_response, + ) from litellm.llms.gemini.image_generation.cost_calculator import ( cost_calculator as gemini_image_cost_calculator, ) @@ -707,6 +739,24 @@ class CostCalculatorUtils: model=model, image_response=completion_response, ) + elif custom_llm_provider == litellm.LlmProviders.FAL_AI.value: + from litellm.llms.fal_ai.cost_calculator import ( + cost_calculator as fal_ai_image_cost_calculator, + ) + + return fal_ai_image_cost_calculator( + model=model, + image_response=completion_response, + ) + elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value: + from litellm.llms.runwayml.cost_calculator import ( + cost_calculator as runwayml_image_cost_calculator, + ) + + return runwayml_image_cost_calculator( + model=model, + image_response=completion_response, + ) else: return default_image_cost_calculator( model=model, diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 6ed9d5725e9..5a50806218f 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -37,6 +37,8 @@ from litellm.types.utils import ( TextChoices, TextCompletionResponse, TranscriptionResponse, + TranscriptionUsageDurationObject, + TranscriptionUsageTokensObject, Usage, ) @@ -684,6 +686,24 @@ def convert_to_model_response_object( # noqa: PLR0915 if key in response_object: setattr(model_response_object, key, response_object[key]) + if "usage" in response_object and response_object["usage"] is not None: + tr_usage_object: Optional[ + Union[ + TranscriptionUsageDurationObject, TranscriptionUsageTokensObject + ] + ] = None + + if response_object["usage"].get("type", None) == "duration": + tr_usage_object = TranscriptionUsageDurationObject( + **response_object["usage"] + ) + elif response_object["usage"].get("type", None) == "tokens": + tr_usage_object = TranscriptionUsageTokensObject( + **response_object["usage"] + ) + if tr_usage_object is not None: + setattr(model_response_object, "usage", tr_usage_object) + if hidden_params is not None: model_response_object._hidden_params = hidden_params diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index c5ef7237628..ccfdcfeb2ed 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -85,7 +85,7 @@ class ResponseMetadata: # Set total response time if supported if self.supports_response_time: self.result._response_ms = total_response_time_ms - + ######################################################### # 1. Add _response_ms total duration ######################################################### @@ -106,12 +106,21 @@ class ResponseMetadata: "litellm_overhead_time_ms": overhead_ms, } ) - + ######################################################### # 3. Add duration for reading from cache # In this case overhead from litellm is the difference between the cache read duration and the total response time ######################################################### - if logging_obj.caching_details is not None and logging_obj.caching_details.get("cache_hit") is True and (cache_duration_ms := logging_obj.caching_details.get("cache_duration_ms")) is not None: + if ( + logging_obj.caching_details is not None + and logging_obj.caching_details.get("cache_hit") is True + and ( + cache_duration_ms := logging_obj.caching_details.get( + "cache_duration_ms" + ) + ) + is not None + ): overhead_ms = total_response_time_ms - cache_duration_ms self._update_hidden_params( { diff --git a/litellm/litellm_core_utils/logging_worker.py b/litellm/litellm_core_utils/logging_worker.py index 3c475f133a8..20f0d70160a 100644 --- a/litellm/litellm_core_utils/logging_worker.py +++ b/litellm/litellm_core_utils/logging_worker.py @@ -1,4 +1,5 @@ import asyncio +import atexit import contextlib import contextvars from typing import Coroutine, Optional @@ -43,6 +44,9 @@ class LoggingWorker: self._queue: Optional[asyncio.Queue[LoggingTask]] = None self._worker_task: Optional[asyncio.Task] = None + # Register cleanup handler to flush remaining events on exit + atexit.register(self._flush_on_exit) + def _ensure_queue(self) -> None: """Initialize the queue if it doesn't exist.""" if self._queue is None: @@ -154,6 +158,61 @@ class LoggingWorker: except asyncio.QueueEmpty: break + def _flush_on_exit(self): + """ + Flush remaining events synchronously before process exit. + Called automatically via atexit handler. + + This ensures callbacks queued by async completions are processed + even when the script exits before the worker loop can handle them. + """ + if self._queue is None: + verbose_logger.debug("[LoggingWorker] atexit: No queue initialized") + return + + if self._queue.empty(): + verbose_logger.debug("[LoggingWorker] atexit: Queue is empty") + return + + queue_size = self._queue.qsize() + verbose_logger.info(f"[LoggingWorker] atexit: Flushing {queue_size} remaining events...") + + # Create a new event loop since the original is closed + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + try: + # Process remaining queue items with time limit + processed = 0 + start_time = loop.time() + + while not self._queue.empty() and processed < self.MAX_ITERATIONS_TO_CLEAR_QUEUE: + if loop.time() - start_time >= self.MAX_TIME_TO_CLEAR_QUEUE: + verbose_logger.warning( + f"[LoggingWorker] atexit: Reached time limit ({self.MAX_TIME_TO_CLEAR_QUEUE}s), stopping flush" + ) + break + + try: + task = self._queue.get_nowait() + except asyncio.QueueEmpty: + break + + # Run the coroutine synchronously in new loop + # Note: We run the coroutine directly, not via create_task, + # since we're in a new event loop context + try: + loop.run_until_complete(task["coroutine"]) + processed += 1 + except Exception as e: + # Silent failure to not break user's program + verbose_logger.debug(f"[LoggingWorker] atexit: Error flushing callback: {e}") + + verbose_logger.info(f"[LoggingWorker] atexit: Successfully flushed {processed} events!") + + finally: + loop.close() + # Global instance for backward compatibility GLOBAL_LOGGING_WORKER = LoggingWorker() diff --git a/litellm/litellm_core_utils/model_response_utils.py b/litellm/litellm_core_utils/model_response_utils.py index 974d12aef6f..00462221fe3 100644 --- a/litellm/litellm_core_utils/model_response_utils.py +++ b/litellm/litellm_core_utils/model_response_utils.py @@ -46,7 +46,8 @@ def is_model_response_stream_empty(model_response: ModelResponseStream) -> bool: return False # Check for any non-base fields that are set - for model_response_field in model_response.model_fields.keys(): + # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings + for model_response_field in type(model_response).model_fields.keys(): # Skip base fields that are always set if model_response_field in BASE_FIELDS: continue diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 19d5932ff28..69e3cc43322 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -94,6 +94,15 @@ def handle_messages_with_content_list_to_str_conversion( return messages +def strip_name_from_message(message: AllMessageValues, allowed_name_roles: List[str] = ["user"]) -> AllMessageValues: + """ + Removes 'name' from message + """ + msg_copy = message.copy() + if msg_copy.get("role") not in allowed_name_roles: + msg_copy.pop("name", None) # type: ignore + return msg_copy + def strip_name_from_messages( messages: List[AllMessageValues], allowed_name_roles: List[str] = ["user"] ) -> List[AllMessageValues]: @@ -654,6 +663,102 @@ def _get_image_mime_type_from_url(url: str) -> Optional[str]: return None +def infer_content_type_from_url_and_content( + url: str, + content: bytes, + current_content_type: Optional[str] = None, +) -> str: + """ + Infer content type from URL extension and binary content when content-type header is missing or generic. + + This helper implements a fallback strategy for determining MIME types when HTTP headers + are missing or provide generic values (like binary/octet-stream). It's commonly used + when processing images and documents from various sources (S3, URLs, etc.). + + Fallback Strategy: + 1. If current_content_type is valid (not None and not generic octet-stream), return it + 2. Try to infer from URL extension (handles query parameters) + 3. Try to detect from binary content signature (magic bytes) + 4. Raise ValueError if all methods fail + + Args: + url: The URL of the content (used to extract file extension) + content: The binary content (first ~100 bytes are sufficient for detection) + current_content_type: The current content-type from headers (may be None or generic) + + Returns: + str: The inferred MIME type (e.g., "image/png", "application/pdf") + + Raises: + ValueError: If content type cannot be determined by any method + + Example: + >>> content_type = infer_content_type_from_url_and_content( + ... url="https://s3.amazonaws.com/bucket/image.png?AWSAccessKeyId=123", + ... content=png_binary_data, + ... current_content_type="binary/octet-stream" + ... ) + >>> print(content_type) + "image/png" + """ + from litellm.litellm_core_utils.token_counter import get_image_type + + # If we have a valid content type that's not generic, use it + if current_content_type and current_content_type not in [ + "binary/octet-stream", + "application/octet-stream", + ]: + return current_content_type + + # Extension to MIME type mapping + # Supports images, documents, and other common file types + extension_to_mime = { + # Image formats + "jpg": "image/jpeg", + "jpeg": "image/jpeg", + "png": "image/png", + "gif": "image/gif", + "webp": "image/webp", + # Document formats + "pdf": "application/pdf", + "csv": "text/csv", + "doc": "application/msword", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "xls": "application/vnd.ms-excel", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "html": "text/html", + "txt": "text/plain", + "md": "text/markdown", + } + + # Try to infer from URL extension + if url: + extension = url.split(".")[-1].lower().split("?")[0] # Remove query params + inferred_type = extension_to_mime.get(extension) + if inferred_type: + return inferred_type + + # Try to detect from binary content signature (magic bytes) + if content: + detected_type = get_image_type(content[:100]) + if detected_type: + type_to_mime = { + "png": "image/png", + "jpeg": "image/jpeg", + "gif": "image/gif", + "webp": "image/webp", + "heic": "image/heic", + } + if detected_type in type_to_mime: + return type_to_mime[detected_type] + + # If all fallbacks failed, raise error + raise ValueError( + f"Unable to determine content type from URL: {url}. " + f"Response content-type: {current_content_type}" + ) + + def get_tool_call_names(tools: List[ChatCompletionToolParam]) -> List[str]: """ Get tool call names from tools diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index d2cad0abd93..717c2607657 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1,4 +1,5 @@ import copy +import hashlib import json import mimetypes import re @@ -38,7 +39,11 @@ from litellm.types.llms.vertex_ai import FunctionResponse as VertexFunctionRespo from litellm.types.llms.vertex_ai import PartType as VertexPartType from litellm.types.utils import GenericImageParsingChunk -from .common_utils import convert_content_list_to_str, is_non_content_values_set +from .common_utils import ( + convert_content_list_to_str, + infer_content_type_from_url_and_content, + is_non_content_values_set, +) from .image_handling import convert_url_to_base64 @@ -364,17 +369,19 @@ def phind_codellama_pt(messages): return prompt -def _render_chat_template(env, chat_template: str, bos_token: str, eos_token: str, messages: list) -> str: +def _render_chat_template( + env, chat_template: str, bos_token: str, eos_token: str, messages: list +) -> str: """ Shared template rendering logic for both sync and async hf_chat_template - + Args: env: Jinja2 environment chat_template: Chat template string bos_token: Beginning of sequence token eos_token: End of sequence token messages: Messages to render - + Returns: Rendered template string """ @@ -456,7 +463,7 @@ async def _afetch_and_extract_template( ) -> Tuple[str, str, str]: """ Async version: Fetch template and tokens from HuggingFace. - + Returns: (chat_template, bos_token, eos_token) """ from litellm.litellm_core_utils.prompt_templates.huggingface_template_handler import ( @@ -518,7 +525,7 @@ def _fetch_and_extract_template( ) -> Tuple[str, str, str]: """ Sync version: Fetch template and tokens from HuggingFace. - + Returns: (chat_template, bos_token, eos_token) """ from litellm.litellm_core_utils.prompt_templates.huggingface_template_handler import ( @@ -604,9 +611,7 @@ async def ahf_chat_template( ) -def hf_chat_template( - model: str, messages: list, chat_template: Optional[Any] = None -): +def hf_chat_template(model: str, messages: list, chat_template: Optional[Any] = None): """HuggingFace chat template (sync version)""" from litellm.litellm_core_utils.prompt_templates.huggingface_template_handler import ( _get_chat_template_file, @@ -1205,10 +1210,10 @@ def convert_to_gemini_tool_call_invoke( if tool_calls is not None: for tool in tool_calls: if "function" in tool: - gemini_function_call: Optional[ - VertexFunctionCall - ] = _gemini_tool_call_invoke_helper( - function_call_params=tool["function"] + gemini_function_call: Optional[VertexFunctionCall] = ( + _gemini_tool_call_invoke_helper( + function_call_params=tool["function"] + ) ) if gemini_function_call is not None: _parts_list.append( @@ -1486,7 +1491,7 @@ def convert_to_anthropic_tool_invoke( _content_element = add_cache_control_to_content( anthropic_content_element=_anthropic_tool_use_param, - orignal_content_element=dict(tool), + original_content_element=dict(tool), ) if "cache_control" in _content_element: @@ -1508,9 +1513,9 @@ def add_cache_control_to_content( AnthropicMessagesToolUseParam, ChatCompletionThinkingBlock, ], - orignal_content_element: Union[dict, AllMessageValues], + original_content_element: Union[dict, AllMessageValues], ): - cache_control_param = orignal_content_element.get("cache_control") + cache_control_param = original_content_element.get("cache_control") if cache_control_param is not None and isinstance(cache_control_param, dict): transformed_param = ChatCompletionCachedContent(**cache_control_param) # type: ignore @@ -1723,13 +1728,13 @@ def anthropic_messages_pt( # noqa: PLR0915 ) _content_element = add_cache_control_to_content( anthropic_content_element=_anthropic_content_element, - orignal_content_element=dict(m), + original_content_element=dict(m), ) if "cache_control" in _content_element: - _anthropic_content_element[ - "cache_control" - ] = _content_element["cache_control"] + _anthropic_content_element["cache_control"] = ( + _content_element["cache_control"] + ) user_content.append(_anthropic_content_element) elif m.get("type", "") == "text": m = cast(ChatCompletionTextObject, m) @@ -1741,7 +1746,7 @@ def anthropic_messages_pt( # noqa: PLR0915 ) _content_element = add_cache_control_to_content( anthropic_content_element=_anthropic_text_content_element, - orignal_content_element=dict(m), + original_content_element=dict(m), ) _content_element = cast( AnthropicMessagesTextParam, _content_element @@ -1763,13 +1768,13 @@ def anthropic_messages_pt( # noqa: PLR0915 } _content_element = add_cache_control_to_content( anthropic_content_element=_anthropic_content_text_element, - orignal_content_element=dict(user_message_types_block), + original_content_element=dict(user_message_types_block), ) if "cache_control" in _content_element: - _anthropic_content_text_element[ - "cache_control" - ] = _content_element["cache_control"] + _anthropic_content_text_element["cache_control"] = ( + _content_element["cache_control"] + ) user_content.append(_anthropic_content_text_element) @@ -1821,7 +1826,7 @@ def anthropic_messages_pt( # noqa: PLR0915 ) _cached_message = add_cache_control_to_content( anthropic_content_element=anthropic_message, - orignal_content_element=dict(m), + original_content_element=dict(m), ) assistant_content.append( @@ -1841,7 +1846,7 @@ def anthropic_messages_pt( # noqa: PLR0915 _content_element = add_cache_control_to_content( anthropic_content_element=_anthropic_text_content_element, - orignal_content_element=dict(assistant_content_block), + original_content_element=dict(assistant_content_block), ) if "cache_control" in _content_element: @@ -2536,13 +2541,17 @@ class BedrockImageProcessor: """Handles both sync and async image processing for Bedrock conversations.""" @staticmethod - def _post_call_image_processing(response: httpx.Response) -> Tuple[str, str]: + def _post_call_image_processing(response: httpx.Response, image_url: str = "") -> Tuple[str, str]: # Check the response's content type to ensure it is an image content_type = response.headers.get("content-type") - if not content_type: - raise ValueError( - f"URL does not contain content-type (content-type: {content_type})" - ) + + # Use helper function to infer content type with fallback logic + content_type = infer_content_type_from_url_and_content( + url=image_url, + content=response.content, + current_content_type=content_type, + ) + content_type = _parse_content_type(content_type) # Convert the image content to base64 bytes @@ -2561,7 +2570,7 @@ class BedrockImageProcessor: response = await client.get(image_url, follow_redirects=True) response.raise_for_status() # Raise an exception for HTTP errors - return BedrockImageProcessor._post_call_image_processing(response) + return BedrockImageProcessor._post_call_image_processing(response, image_url) except Exception as e: raise e @@ -2574,7 +2583,7 @@ class BedrockImageProcessor: response = client.get(image_url, follow_redirects=True) response.raise_for_status() # Raise an exception for HTTP errors - return BedrockImageProcessor._post_call_image_processing(response) + return BedrockImageProcessor._post_call_image_processing(response, image_url) except Exception as e: raise e @@ -2698,12 +2707,39 @@ class BedrockImageProcessor: for video_type in supported_video_formats ) + HASH_SAMPLE_BYTES = 64 * 1024 # hash up to 64 KB of data + if is_document: + # --- Prepare normalized bytes for hashing (without modifying original) --- + if isinstance(image_bytes, str): + # Remove whitespace/newlines so base64 variations hash identically + normalized = "".join(image_bytes.split()).encode("utf-8") + else: + normalized = image_bytes + + # --- Use only the first 64 KB for speed --- + if len(normalized) <= HASH_SAMPLE_BYTES: + sample = normalized + else: + sample = normalized[:HASH_SAMPLE_BYTES] + + # --- Compute deterministic hash (sample + total length) --- + hasher = hashlib.sha256() + hasher.update(sample) + hasher.update( + str(len(normalized)).encode("utf-8") + ) # include full length for uniqueness + full_hash = hasher.hexdigest() + content_hash = full_hash[:16] # short deterministic ID + + document_name = f"DocumentPDFmessages_{content_hash}_{image_format}" + + # --- Return content block --- return BedrockContentBlock( document=BedrockDocumentBlock( source=_blob, format=image_format, - name=f"DocumentPDFmessages_{str(uuid.uuid4())}", + name=document_name, ) ) elif is_video: @@ -3803,7 +3839,9 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 assistant_parts=assistants_parts, ) elif element["type"] == "text": - assistants_part = BedrockContentBlock(text=element["text"]) + # AWS Bedrock doesn't allow empty or whitespace-only text content, so use placeholder for empty strings + text_content = element["text"] if element["text"].strip() else "." + assistants_part = BedrockContentBlock(text=text_content) assistants_parts.append(assistants_part) elif element["type"] == "image_url": if isinstance(element["image_url"], dict): @@ -3827,7 +3865,9 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 assistants_parts.append(_cache_point_block) assistant_content.extend(assistants_parts) elif _assistant_content is not None and isinstance(_assistant_content, str): - assistant_content.append(BedrockContentBlock(text=_assistant_content)) + # AWS Bedrock doesn't allow empty or whitespace-only text content, so use placeholder for empty strings + text_content = _assistant_content if _assistant_content.strip() else "." + assistant_content.append(BedrockContentBlock(text=text_content)) # Add cache point block for assistant string content _cache_point_block = ( litellm.AmazonConverseConfig()._get_cache_point_block( @@ -3964,9 +4004,11 @@ def _bedrock_tools_pt(tools: List) -> List[BedrockToolBlock]: # related issue: https://github.com/BerriAI/litellm/issues/5007 # Bedrock tool names must satisfy regular expression pattern: [a-zA-Z][a-zA-Z0-9_]* ensure this is true name = make_valid_bedrock_tool_name(input_tool_name=name) - description = tool.get("function", {}).get( - "description", name - ) # converse api requires a description + _tool_description = tool.get("function", {}).get("description", None) + if _tool_description: # bedrock doesn't accept empty "" or None descriptions + description = _tool_description + else: + description = name defs = parameters.pop("$defs", {}) defs_copy = copy.deepcopy(defs) @@ -4171,8 +4213,11 @@ def prompt_factory( return azure_text_pt(messages=messages) elif custom_llm_provider == "watsonx": from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig - return IBMWatsonXChatConfig.apply_prompt_template(model=model, messages=messages) - + + return IBMWatsonXChatConfig.apply_prompt_template( + model=model, messages=messages + ) + try: if "meta-llama/llama-2" in model and "chat" in model: return llama_2_chat_pt(messages=messages) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 5ac38949e2b..0effed3db70 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -7,14 +7,17 @@ # # Thank you users! We ❤️ you! - Krrish & Ishaan +import asyncio import copy from typing import TYPE_CHECKING, Any, Optional import litellm from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import ( + get_metadata_variable_name_from_kwargs, +) from litellm.secret_managers.main import str_to_bool from litellm.types.utils import StandardCallbackDynamicParams -import asyncio if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( @@ -37,6 +40,38 @@ def redact_message_input_output_from_custom_logger( return result +def _redact_choice_content(choice): + """Helper to redact content in a choice (message or delta).""" + if isinstance(choice, litellm.Choices): + choice.message.content = "redacted-by-litellm" + if hasattr(choice.message, "reasoning_content"): + choice.message.reasoning_content = "redacted-by-litellm" + if hasattr(choice.message, "thinking_blocks"): + choice.message.thinking_blocks = None + elif isinstance(choice, litellm.utils.StreamingChoices): + choice.delta.content = "redacted-by-litellm" + if hasattr(choice.delta, "reasoning_content"): + choice.delta.reasoning_content = "redacted-by-litellm" + if hasattr(choice.delta, "thinking_blocks"): + choice.delta.thinking_blocks = None + + +def _redact_responses_api_output(output_items): + """Helper to redact ResponsesAPIResponse output items.""" + for output_item in output_items: + if hasattr(output_item, "content") and isinstance(output_item.content, list): + for content_part in output_item.content: + if hasattr(content_part, "text"): + content_part.text = "redacted-by-litellm" + + # Redact reasoning items in output array + if hasattr(output_item, "type") and output_item.type == "reasoning": + if hasattr(output_item, "summary") and isinstance(output_item.summary, list): + for summary_item in output_item.summary: + if hasattr(summary_item, "text"): + summary_item.text = "redacted-by-litellm" + + def perform_redaction(model_call_details: dict, result): """ Performs the actual redaction on the logging object and result. @@ -56,19 +91,12 @@ def perform_redaction(model_call_details: dict, result): _streaming_response = model_call_details["complete_streaming_response"] if hasattr(_streaming_response, "choices"): for choice in _streaming_response.choices: - if isinstance(choice, litellm.Choices): - choice.message.content = "redacted-by-litellm" - elif isinstance(choice, litellm.utils.StreamingChoices): - choice.delta.content = "redacted-by-litellm" + _redact_choice_content(choice) elif hasattr(_streaming_response, "output"): - # Handle ResponsesAPIResponse format - for output_item in _streaming_response.output: - if hasattr(output_item, "content") and isinstance( - output_item.content, list - ): - for content_part in output_item.content: - if hasattr(content_part, "text"): - content_part.text = "redacted-by-litellm" + _redact_responses_api_output(_streaming_response.output) + # Redact reasoning field in ResponsesAPIResponse + if hasattr(_streaming_response, "reasoning") and _streaming_response.reasoning is not None: + _streaming_response.reasoning = None # Redact result if result is not None: @@ -84,17 +112,13 @@ def perform_redaction(model_call_details: dict, result): if isinstance(_result, litellm.ModelResponse): if hasattr(_result, "choices") and _result.choices is not None: for choice in _result.choices: - if isinstance(choice, litellm.Choices): - choice.message.content = "redacted-by-litellm" - elif isinstance(choice, litellm.utils.StreamingChoices): - choice.delta.content = "redacted-by-litellm" + _redact_choice_content(choice) elif isinstance(_result, litellm.ResponsesAPIResponse): if hasattr(_result, "output"): - for output_item in _result.output: - if hasattr(output_item, "content") and isinstance(output_item.content, list): - for content_part in output_item.content: - if hasattr(content_part, "text"): - content_part.text = "redacted-by-litellm" + _redact_responses_api_output(_result.output) + # Redact reasoning field in ResponsesAPIResponse + if hasattr(_result, "reasoning") and _result.reasoning is not None: + _result.reasoning = None elif isinstance(_result, litellm.EmbeddingResponse): if hasattr(_result, "data") and _result.data is not None: _result.data = [] @@ -107,11 +131,13 @@ def should_redact_message_logging(model_call_details: dict) -> bool: """ Determine if message logging should be redacted. """ - _request_headers = ( - model_call_details.get("litellm_params", {}).get("metadata", {}) or {} - ) - - request_headers = _request_headers.get("headers", {}) + litellm_params = model_call_details.get("litellm_params", {}) + + metadata_field = get_metadata_variable_name_from_kwargs(litellm_params) + metadata = litellm_params.get(metadata_field, {}) + + # Get headers from the metadata + request_headers = metadata.get("headers", {}) if isinstance(metadata, dict) else {} possible_request_headers = [ "litellm-enable-message-redaction", # old header. maintain backwards compatibility diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index c714e36b5f9..8b50e41a795 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -49,4 +49,4 @@ def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str: return "Unserializable Object" safe_data = _serialize(data, set(), 0) - return json.dumps(safe_data, default=str) + return json.dumps(safe_data, default=str) \ No newline at end of file diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 1daf543cfcb..4d8e109d882 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -20,7 +20,9 @@ from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.types.llms.openai import ChatCompletionChunk from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import Delta +from litellm.types.utils import ( + Delta, +) from litellm.types.utils import GenericStreamingChunk as GChunk from litellm.types.utils import ( ModelResponse, @@ -732,6 +734,14 @@ class CustomStreamWrapper: "function_call" in completion_obj and completion_obj["function_call"] is not None ) + or ( + "tool_calls" in model_response.choices[0].delta + and model_response.choices[0].delta["tool_calls"] is not None + ) + or ( + "function_call" in model_response.choices[0].delta + and model_response.choices[0].delta["function_call"] is not None + ) or ( "reasoning_content" in model_response.choices[0].delta and model_response.choices[0].delta.reasoning_content is not None @@ -1295,7 +1305,7 @@ class CustomStreamWrapper: else: # openai / azure chat model if self.custom_llm_provider == "azure": if isinstance(chunk, BaseModel) and hasattr(chunk, "model"): - # for azure, we need to pass the model from the orignal chunk + # for azure, we need to pass the model from the original chunk self.model = getattr(chunk, "model", self.model) response_obj = self.handle_openai_chat_completion_chunk(chunk) if response_obj is None: @@ -1520,6 +1530,43 @@ class CustomStreamWrapper: """ self.logging_loop = loop + async def _call_post_streaming_deployment_hook(self, chunk): + """ + Call the post-call streaming deployment hook for callbacks. + + This allows callbacks to modify streaming chunks before they're returned. + """ + try: + import litellm + from litellm.integrations.custom_logger import CustomLogger + from litellm.types.utils import CallTypes + + # Get request kwargs from logging object + request_data = self.logging_obj.model_call_details + call_type_str = self.logging_obj.call_type + + try: + typed_call_type = CallTypes(call_type_str) + except ValueError: + typed_call_type = None + + # Call hooks for all callbacks + for callback in litellm.callbacks: + if isinstance(callback, CustomLogger) and hasattr(callback, "async_post_call_streaming_deployment_hook"): + result = await callback.async_post_call_streaming_deployment_hook( + request_data=request_data, + response_chunk=chunk, + call_type=typed_call_type, + ) + if result is not None: + chunk = result + + return chunk + except Exception as e: + from litellm._logging import verbose_logger + verbose_logger.exception(f"Error in post-call streaming deployment hook: {str(e)}") + return chunk + def cache_streaming_response(self, processed_chunk, cache_hit: bool): """ Caches the streaming response @@ -1825,6 +1872,11 @@ class CustomStreamWrapper: if self.sent_last_chunk is True and self.stream_options is None: usage = calculate_total_usage(chunks=self.chunks) processed_chunk._hidden_params["usage"] = usage + + # Call post-call streaming deployment hook for final chunk + if self.sent_last_chunk is True: + processed_chunk = await self._call_post_streaming_deployment_hook(processed_chunk) + return processed_chunk raise StopAsyncIteration else: # temporary patch for non-aiohttp async calls diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index fab2c1e76ee..a21ebd56f60 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -3,7 +3,17 @@ import base64 import io import struct -from typing import Callable, List, Literal, Optional, Tuple, Union, cast +from typing import ( + Any, + Callable, + List, + Literal, + Mapping, + Optional, + Tuple, + Union, + cast, +) import tiktoken @@ -20,6 +30,10 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.default_encoding import encoding as default_encoding from litellm.llms.custom_httpx.http_handler import _get_httpx_client +from litellm.types.llms.anthropic import ( + AnthropicMessagesToolResultParam, + AnthropicMessagesToolUseParam, +) from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionNamedToolChoiceParam, @@ -552,6 +566,131 @@ def _fix_model_name(model: str) -> str: return "gpt-3.5-turbo" +def _count_image_tokens( + image_url: Any, + use_default_image_token_count: bool, +) -> int: + """ + Count tokens for an image_url content block. + + Args: + image_url: The image URL data - can be a string URL or dict with 'url' and 'detail' + use_default_image_token_count: Whether to use default image token counts + + Returns: + int: Number of tokens for the image + + Raises: + ValueError: If image_url is invalid type or detail value is invalid + """ + if isinstance(image_url, dict): + detail = image_url.get("detail", "auto") + if detail not in ["low", "high", "auto"]: + raise ValueError( + f"Invalid detail value: {detail}. Expected 'low', 'high', or 'auto'." + ) + url = image_url.get("url") + if not url: + raise ValueError("Missing required key 'url' in image_url dict.") + return calculate_img_tokens( + data=url, + mode=detail, # type: ignore + use_default_image_token_count=use_default_image_token_count, + ) + elif isinstance(image_url, str): + if not image_url.strip(): + raise ValueError("Empty image_url string is not valid.") + return calculate_img_tokens( + data=image_url, + mode="auto", + use_default_image_token_count=use_default_image_token_count, + ) + else: + raise ValueError( + f"Invalid image_url type: {type(image_url).__name__}. " + "Expected str or dict with 'url' field." + ) + + +def _validate_anthropic_content(content: Mapping[str, Any]) -> type: + """ + Validate and determine which Anthropic TypedDict applies. + + Returns the corresponding TypedDict class if recognized, otherwise raises. + """ + content_type = content.get("type") + if not content_type: + raise ValueError("Anthropic content missing required field: 'type'") + + mapping = { + "tool_use": AnthropicMessagesToolUseParam, + "tool_result": AnthropicMessagesToolResultParam, + } + + expected_cls = mapping.get(content_type) + if expected_cls is None: + raise ValueError(f"Unknown Anthropic content type: '{content_type}'") + + missing = [ + k for k in getattr(expected_cls, "__required_keys__", set()) if k not in content + ] + if missing: + raise ValueError( + f"Missing required fields in {content_type} block: {', '.join(missing)}" + ) + + return expected_cls + + +def _count_anthropic_content( + content: Mapping[str, Any], + count_function: TokenCounterFunction, + use_default_image_token_count: bool, + default_token_count: Optional[int], +) -> int: + """ + Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.). + + Uses TypedDict definitions from litellm.types.llms.anthropic to determine + what fields to count and how to handle nested structures. + + Dynamically infers which fields to count based on the TypedDict definition, + avoiding hardcoded field names. + """ + typeddict_cls = _validate_anthropic_content(content) + type_hints = getattr(typeddict_cls, "__annotations__", {}) + tokens = 0 + + # Fields to skip (metadata/identifiers that don't contribute to prompt tokens) + skip_fields = {"type", "id", "tool_use_id", "cache_control", "is_error"} + + # Iterate over all fields defined in the TypedDict + for field_name, field_type in type_hints.items(): + if field_name in skip_fields: + continue + + field_value = content.get(field_name) + if field_value is None: + continue + try: + if isinstance(field_value, str): + tokens += count_function(field_value) + elif isinstance(field_value, list): + tokens += _count_content_list( + count_function, + field_value, # type: ignore + use_default_image_token_count, + default_token_count, + ) + elif isinstance(field_value, dict): + tokens += count_function(str(field_value)) + except Exception as e: + if default_token_count is not None: + return default_token_count + raise ValueError(f"Error counting field '{field_name}': {e}") + return tokens + + def _count_content_list( count_function: TokenCounterFunction, content_list: OpenAIMessageContent, @@ -559,7 +698,7 @@ def _count_content_list( default_token_count: Optional[int], ) -> int: """ - Get the number of tokens from a list of content. + Recursively count tokens from a list of content blocks. """ try: num_tokens = 0 @@ -567,42 +706,32 @@ def _count_content_list( if isinstance(c, str): num_tokens += count_function(c) elif c["type"] == "text": - num_tokens += count_function(c["text"]) + num_tokens += count_function(c.get("text", "")) elif c["type"] == "image_url": - if isinstance(c["image_url"], dict): - image_url_dict = c["image_url"] - detail = image_url_dict.get("detail", "auto") - if detail not in ["low", "high", "auto"]: - raise ValueError( - f"Invalid detail value: {detail}. Expected 'low', 'high', or 'auto'." - ) - url = image_url_dict.get("url") - num_tokens += calculate_img_tokens( - data=url, - mode=detail, # type: ignore - use_default_image_token_count=use_default_image_token_count, - ) - elif isinstance(c["image_url"], str): - image_url_str = c["image_url"] - num_tokens += calculate_img_tokens( - data=image_url_str, - mode="auto", - use_default_image_token_count=use_default_image_token_count, - ) - else: - raise ValueError( - f"Invalid image_url type: {type(c['image_url'])}. Expected str or dict." - ) + image_url = c.get("image_url") + num_tokens += _count_image_tokens( + image_url, use_default_image_token_count + ) + elif c["type"] in ("tool_use", "tool_result"): + num_tokens += _count_anthropic_content( + c, + count_function, + use_default_image_token_count, + default_token_count, + ) else: raise ValueError( - f"Invalid content type: {type(c)}. Expected str or dict." + f"Invalid content item type: {type(c).__name__}. " + f"Expected str or dict with 'type' field. " + f"Value: {c!r}" ) return num_tokens except Exception as e: if default_token_count is not None: return default_token_count raise ValueError( - f"Error getting number of tokens from content list: {e}, default_token_count={default_token_count}" + f"Error getting number of tokens from content list: {e}, " + f"default_token_count={default_token_count}" ) diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index 18973add86d..15c035ceec8 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -1,8 +1,16 @@ -from typing import TYPE_CHECKING, Optional +import importlib +import os +from typing import TYPE_CHECKING, Dict, Optional, Type + +from litellm._logging import verbose_logger +from litellm.types.utils import CallTypes from . import * if TYPE_CHECKING: + from litellm.llms.base_llm.guardrail_translation.base_translation import ( + BaseTranslation, + ) from litellm.types.utils import ModelInfo, Usage @@ -31,5 +39,129 @@ def get_cost_for_web_search_request( ) return cost_per_web_search_request_vertex_ai(usage=usage, model_info=model_info) + elif custom_llm_provider == "perplexity": + # Perplexity handles search costs internally in its own cost calculator + # Return 0.0 to indicate costs are already accounted for + return 0.0 + elif custom_llm_provider == "xai": + from .xai.cost_calculator import cost_per_web_search_request + return cost_per_web_search_request(usage=usage, model_info=model_info) else: return None + + +def discover_guardrail_translation_mappings() -> ( + Dict[CallTypes, Type["BaseTranslation"]] +): + """ + Discover guardrail translation mappings by scanning the llms directory structure. + + Scans for modules with guardrail_translation_mappings dictionaries and aggregates them. + + Returns: + Dict[CallTypes, Type[BaseTranslation]]: A dictionary mapping call types to their translation handler classes + """ + discovered_mappings: Dict[CallTypes, Type["BaseTranslation"]] = {} + + try: + # Get the path to the llms directory + current_dir = os.path.dirname(__file__) + llms_dir = current_dir + + if not os.path.exists(llms_dir): + verbose_logger.debug("llms directory not found") + return discovered_mappings + + # Recursively scan for guardrail_translation directories + for root, dirs, files in os.walk(llms_dir): + # Skip __pycache__ and base_llm directories + dirs[:] = [d for d in dirs if not d.startswith("__") and d != "base_llm"] + + # Check if this is a guardrail_translation directory with __init__.py + if ( + os.path.basename(root) == "guardrail_translation" + and "__init__.py" in files + ): + # Build the module path relative to litellm + rel_path = os.path.relpath(root, os.path.dirname(llms_dir)) + module_path = "litellm." + rel_path.replace(os.sep, ".") + + try: + # Import the module + verbose_logger.debug( + f"Discovering guardrail translations in: {module_path}" + ) + + module = importlib.import_module(module_path) + + # Check for guardrail_translation_mappings dictionary + if hasattr(module, "guardrail_translation_mappings"): + mappings = getattr(module, "guardrail_translation_mappings") + if isinstance(mappings, dict): + discovered_mappings.update(mappings) + verbose_logger.debug( + f"Found guardrail_translation_mappings in {module_path}: {list(mappings.keys())}" + ) + + except ImportError as e: + verbose_logger.error(f"Could not import {module_path}: {e}") + continue + except Exception as e: + verbose_logger.error(f"Error processing {module_path}: {e}") + continue + + verbose_logger.debug( + f"Discovered {len(discovered_mappings)} guardrail translation mappings: {list(discovered_mappings.keys())}" + ) + + except Exception as e: + verbose_logger.error(f"Error discovering guardrail translation mappings: {e}") + + return discovered_mappings + + +# Cache the discovered mappings +endpoint_guardrail_translation_mappings: Optional[ + Dict[CallTypes, Type["BaseTranslation"]] +] = None + + +def load_guardrail_translation_mappings(): + global endpoint_guardrail_translation_mappings + if endpoint_guardrail_translation_mappings is None: + endpoint_guardrail_translation_mappings = ( + discover_guardrail_translation_mappings() + ) + return endpoint_guardrail_translation_mappings + + +def get_guardrail_translation_mapping(call_type: CallTypes) -> Type["BaseTranslation"]: + """ + Get the guardrail translation handler for a given call type. + + Args: + call_type: The type of call (e.g., completion, acompletion, anthropic_messages) + + Returns: + The translation handler class for the given call type + + Raises: + ValueError: If no translation mapping exists for the given call type + """ + global endpoint_guardrail_translation_mappings + + # Lazy load the mappings on first access + if endpoint_guardrail_translation_mappings is None: + endpoint_guardrail_translation_mappings = ( + discover_guardrail_translation_mappings() + ) + + # Get the translation handler class for the call type + if call_type not in endpoint_guardrail_translation_mappings: + raise ValueError( + f"No guardrail translation mapping found for call_type: {call_type}. " + f"Available mappings: {list(endpoint_guardrail_translation_mappings.keys())}" + ) + + # Return the handler class directly + return endpoint_guardrail_translation_mappings[call_type] diff --git a/litellm/llms/aiml/image_generation/transformation.py b/litellm/llms/aiml/image_generation/transformation.py index 3b586689ea7..006a2c16d7e 100644 --- a/litellm/llms/aiml/image_generation/transformation.py +++ b/litellm/llms/aiml/image_generation/transformation.py @@ -172,16 +172,32 @@ class AimlImageGenerationConfig(BaseImageGenerationConfig): if not model_response.data: model_response.data = [] - # AI/ML API can return images in two different formats: - # 1. output.choices array with image_base64 - # 2. images array with url (and optional width, height, content_type) + # AI/ML API can return images in multiple formats: + # 1. Top-level data array with url (OpenAI-like format) + # 2. output.choices array with image_base64 + # 3. images array with url (and optional width, height, content_type) - if "output" in response_data and "choices" in response_data["output"]: + if "data" in response_data and isinstance(response_data["data"], list): + # Handle OpenAI-like format: {"data": [{"url": "...", "width": 1024, "height": 768, "content_type": "image/jpeg"}]} + for image in response_data["data"]: + if "url" in image: + model_response.data.append(ImageObject( + b64_json=None, + url=image["url"], + revised_prompt=image.get("revised_prompt"), + )) + elif "b64_json" in image or "image_base64" in image: + model_response.data.append(ImageObject( + b64_json=image.get("b64_json") or image.get("image_base64"), + url=None, + revised_prompt=image.get("revised_prompt"), + )) + elif "output" in response_data and "choices" in response_data["output"]: for choice in response_data["output"]["choices"]: if "image_base64" in choice: model_response.data.append(ImageObject( b64_json=choice["image_base64"], - url=None, # AI/ML API returns base64, not URLs + url=None, )) elif "url" in choice: model_response.data.append(ImageObject( diff --git a/litellm/llms/anthropic/chat/guardrail_translation/__init__.py b/litellm/llms/anthropic/chat/guardrail_translation/__init__.py new file mode 100644 index 00000000000..ab327ee9f2c --- /dev/null +++ b/litellm/llms/anthropic/chat/guardrail_translation/__init__.py @@ -0,0 +1,10 @@ +from litellm.llms.anthropic.chat.guardrail_translation.handler import ( + AnthropicMessagesHandler, +) +from litellm.types.utils import CallTypes + +guardrail_translation_mappings = { + CallTypes.anthropic_messages: AnthropicMessagesHandler, +} + +__all__ = ["guardrail_translation_mappings"] diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py new file mode 100644 index 00000000000..06a1b92e1b0 --- /dev/null +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -0,0 +1,270 @@ +""" +Anthropic Message Handler for Unified Guardrails + +This module provides a class-based handler for Anthropic-format messages. +The class methods can be overridden for custom behavior. + +Pattern Overview: +----------------- +1. Extract text content from messages/responses (both string and list formats) +2. Create async tasks to apply guardrails to each text segment +3. Track mappings to know where each response belongs +4. Apply guardrail responses back to the original structure +""" + +import asyncio +from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Optional, Tuple, cast + +from litellm._logging import verbose_proxy_logger +from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation + +if TYPE_CHECKING: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, + AnthropicResponseTextBlock, + ) + + +class AnthropicMessagesHandler(BaseTranslation): + """ + Handler for processing Anthropic messages with guardrails. + + This class provides methods to: + 1. Process input messages (pre-call hook) + 2. Process output responses (post-call hook) + + Methods can be overridden to customize behavior for different message formats. + """ + + async def process_input_messages( + self, + data: dict, + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process input messages by applying guardrails to text content. + """ + messages = data.get("messages") + if messages is None: + return data + + tasks: List[Coroutine[Any, Any, str]] = [] + task_mappings: List[Tuple[int, Optional[int]]] = [] + # Track (message_index, content_index) for each task + # content_index is None for string content, int for list content + + # Step 1: Extract all text content and create guardrail tasks + for msg_idx, message in enumerate(messages): + await self._extract_input_text_and_create_tasks( + message=message, + msg_idx=msg_idx, + tasks=tasks, + task_mappings=task_mappings, + guardrail_to_apply=guardrail_to_apply, + ) + + # Step 2: Run all guardrail tasks in parallel + responses = await asyncio.gather(*tasks) + + # Step 3: Map guardrail responses back to original message structure + await self._apply_guardrail_responses_to_input( + messages=messages, + responses=responses, + task_mappings=task_mappings, + ) + + verbose_proxy_logger.debug( + "Anthropic Messages: Processed input messages: %s", messages + ) + + return data + + async def _extract_input_text_and_create_tasks( + self, + message: Dict[str, Any], + msg_idx: int, + tasks: List, + task_mappings: List[Tuple[int, Optional[int]]], + guardrail_to_apply: "CustomGuardrail", + ) -> None: + """ + Extract text content from a message and create guardrail tasks. + + Override this method to customize text extraction logic. + """ + content = message.get("content", None) + if content is None: + return + + if isinstance(content, str): + # Simple string content + tasks.append(guardrail_to_apply.apply_guardrail(text=content)) + task_mappings.append((msg_idx, None)) + + elif isinstance(content, list): + # List content (e.g., multimodal with text and images) + for content_idx, content_item in enumerate(content): + text_str = content_item.get("text", None) + if text_str is None: + continue + tasks.append(guardrail_to_apply.apply_guardrail(text=text_str)) + task_mappings.append((msg_idx, int(content_idx))) + + async def _apply_guardrail_responses_to_input( + self, + messages: List[Dict[str, Any]], + responses: List[str], + task_mappings: List[Tuple[int, Optional[int]]], + ) -> None: + """ + Apply guardrail responses back to input messages. + + Override this method to customize how responses are applied. + """ + for task_idx, guardrail_response in enumerate(responses): + mapping = task_mappings[task_idx] + msg_idx = cast(int, mapping[0]) + content_idx_optional = cast(Optional[int], mapping[1]) + + content = messages[msg_idx].get("content", None) + if content is None: + continue + + if isinstance(content, str) and content_idx_optional is None: + # Replace string content with guardrail response + messages[msg_idx]["content"] = guardrail_response + + elif isinstance(content, list) and content_idx_optional is not None: + # Replace specific text item in list content + messages[msg_idx]["content"][content_idx_optional][ + "text" + ] = guardrail_response + + async def process_output_response( + self, + response: "AnthropicMessagesResponse", + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process output response by applying guardrails to text content. + + Args: + response: Anthropic MessagesResponse object + guardrail_to_apply: The guardrail instance to apply + + Returns: + Modified response with guardrail applied to content + + Response Format Support: + - List content: response.content = [{"type": "text", "text": "text here"}, ...] + """ + # Step 0: Check if response has any text content to process + if not self._has_text_content(response): + verbose_proxy_logger.warning( + "Anthropic Messages: No text content in response, skipping guardrail" + ) + return response + + tasks: List[Coroutine[Any, Any, str]] = [] + task_mappings: List[Tuple[int, Optional[int]]] = [] + # Track (choice_index, content_index) for each task + + response_content = response.get("content", []) + if not response_content: + return response + # Step 1: Extract all text content from response choices + for content_idx, content_block in enumerate(response_content): + # Check if this is a text block by checking the 'type' field + if isinstance(content_block, dict) and content_block.get("type") == "text": + # Cast to dict to handle the union type properly + await self._extract_output_text_and_create_tasks( + content_block=cast(Dict[str, Any], content_block), + content_idx=content_idx, + tasks=tasks, + task_mappings=task_mappings, + guardrail_to_apply=guardrail_to_apply, + ) + + # Step 2: Run all guardrail tasks in parallel + responses = await asyncio.gather(*tasks) + + # Step 3: Map guardrail responses back to original response structure + await self._apply_guardrail_responses_to_output( + response=response, + responses=responses, + task_mappings=task_mappings, + ) + + verbose_proxy_logger.debug( + "Anthropic Messages: Processed output response: %s", response + ) + + return response + + def _has_text_content(self, response: "AnthropicMessagesResponse") -> bool: + """ + Check if response has any text content to process. + + Override this method to customize text content detection. + """ + response_content = response.get("content", []) + if not response_content: + return False + for content_block in response_content: + # Check if this is a text block by checking the 'type' field + if isinstance(content_block, dict) and content_block.get("type") == "text": + content_text = content_block.get("text") + if content_text and isinstance(content_text, str): + return True + return False + + async def _extract_output_text_and_create_tasks( + self, + content_block: Dict[str, Any], + content_idx: int, + tasks: List, + task_mappings: List[Tuple[int, Optional[int]]], + guardrail_to_apply: "CustomGuardrail", + ) -> None: + """ + Extract text content from a response choice and create guardrail tasks. + + Override this method to customize text extraction logic. + """ + content_text = content_block.get("text") + if content_text and isinstance(content_text, str): + # Simple string content + tasks.append(guardrail_to_apply.apply_guardrail(text=content_text)) + task_mappings.append((content_idx, None)) + + async def _apply_guardrail_responses_to_output( + self, + response: "AnthropicMessagesResponse", + responses: List[str], + task_mappings: List[Tuple[int, Optional[int]]], + ) -> None: + """ + Apply guardrail responses back to output response. + + Override this method to customize how responses are applied. + """ + for task_idx, guardrail_response in enumerate(responses): + mapping = task_mappings[task_idx] + content_idx = cast(int, mapping[0]) + + response_content = response.get("content", []) + if not response_content: + continue + + # Get the content block at the index + if content_idx >= len(response_content): + continue + + content_block = response_content[content_idx] + + # Verify it's a text block and update the text field + if isinstance(content_block, dict) and content_block.get("type") == "text": + # Cast to dict to handle the union type properly for assignment + content_block = cast("AnthropicResponseTextBlock", content_block) + content_block["text"] = guardrail_response diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 691b46af8da..6aeb4f5bb9a 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -12,6 +12,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, + DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, RESPONSE_FORMAT_TOOL_NAME, ) from litellm.litellm_core_utils.core_helpers import map_finish_reason @@ -52,10 +53,7 @@ from litellm.types.utils import ( CompletionTokensDetailsWrapper, ) from litellm.types.utils import Message as LitellmMessage -from litellm.types.utils import ( - PromptTokensDetailsWrapper, - ServerToolUse, -) +from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse from litellm.utils import ( ModelResponse, Usage, @@ -118,7 +116,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return super().get_config() def get_supported_openai_params(self, model: str): - params = [ "stream", "stop", @@ -379,6 +376,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): type="enabled", budget_tokens=DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, ) + elif reasoning_effort == "minimal": + return AnthropicThinkingParam( + type="enabled", + budget_tokens=DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, + ) else: raise ValueError(f"Unmapped reasoning effort: {reasoning_effort}") @@ -517,6 +519,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self._add_tools_to_optional_params( optional_params=optional_params, tools=[hosted_web_search_tool] ) + elif param == "extra_headers": + optional_params["extra_headers"] = value ## handle thinking tokens self.update_optional_params_with_thinking_tokens( @@ -641,13 +645,25 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) ) return tools - - def update_headers_with_optional_anthropic_beta(self, headers: dict, optional_params: dict) -> dict: + + def update_headers_with_optional_anthropic_beta( + self, headers: dict, optional_params: dict + ) -> dict: """Update headers with optional anthropic beta.""" _tools = optional_params.get("tools", []) for tool in _tools: - if tool.get("type", None) and tool.get("type").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_FETCH.value): - headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value + if tool.get("type", None) and tool.get("type").startswith( + ANTHROPIC_HOSTED_TOOLS.WEB_FETCH.value + ): + headers["anthropic-beta"] = ( + ANTHROPIC_BETA_HEADER_VALUES.WEB_FETCH_2025_09_10.value + ) + elif tool.get("type", None) and tool.get("type").startswith( + ANTHROPIC_HOSTED_TOOLS.MEMORY.value + ): + headers["anthropic-beta"] = ( + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value + ) return headers def transform_request( @@ -685,7 +701,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): llm_provider="anthropic", ) - headers = self.update_headers_with_optional_anthropic_beta(headers=headers, optional_params=optional_params) + headers = self.update_headers_with_optional_anthropic_beta( + headers=headers, optional_params=optional_params + ) # Separate system prompt from rest of message anthropic_system_message_list = self.translate_system_message(messages=messages) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 68b5341e954..0d00a3b4632 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -63,13 +63,14 @@ class AnthropicModelInfo(BaseLLMModelInfo): def is_computer_tool_used( self, tools: Optional[List[AllAnthropicToolsValues]] - ) -> bool: + ) -> Optional[str]: + """Returns the computer tool version if used, e.g. 'computer_20250124' or None""" if tools is None: - return False + return None for tool in tools: if "type" in tool and tool["type"].startswith("computer_"): - return True - return False + return tool["type"] + return None def is_pdf_used(self, messages: List[AllMessageValues]) -> bool: """ @@ -94,11 +95,29 @@ class AnthropicModelInfo(BaseLLMModelInfo): return None return anthropic_beta_header.split(",") + def get_computer_tool_beta_header(self, computer_tool_version: str) -> str: + """ + Get the appropriate beta header for a given computer tool version. + + Args: + computer_tool_version: The computer tool version (e.g., 'computer_20250124', 'computer_20241022') + + Returns: + The corresponding beta header string + """ + computer_tool_beta_mapping = { + "computer_20250124": "computer-use-2025-01-24", + "computer_20241022": "computer-use-2024-10-22", + } + return computer_tool_beta_mapping.get( + computer_tool_version, "computer-use-2024-10-22" # Default fallback + ) + def get_anthropic_headers( self, api_key: str, anthropic_version: Optional[str] = None, - computer_tool_used: bool = False, + computer_tool_used: Optional[str] = None, prompt_caching_set: bool = False, pdf_used: bool = False, file_id_used: bool = False, @@ -110,7 +129,8 @@ class AnthropicModelInfo(BaseLLMModelInfo): if prompt_caching_set: betas.add("prompt-caching-2024-07-31") if computer_tool_used: - betas.add("computer-use-2024-10-22") + beta_header = self.get_computer_tool_beta_header(computer_tool_used) + betas.add(beta_header) # if pdf_used: # betas.add("pdfs-2024-09-25") if file_id_used: diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 88a63fc6f5d..795f9a4cd09 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -152,32 +152,27 @@ class LiteLLMMessagesToCompletionTransformationHandler: ) ) - try: - completion_response = await litellm.acompletion(**completion_kwargs) + completion_response = await litellm.acompletion(**completion_kwargs) - if stream: - transformed_stream = ( - ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( - completion_response, - model=model, - ) + if stream: + transformed_stream = ( + ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( + completion_response, + model=model, ) - if transformed_stream is not None: - return transformed_stream - raise ValueError("Failed to transform streaming response") - else: - anthropic_response = ( - ANTHROPIC_ADAPTER.translate_completion_output_params( - cast(ModelResponse, completion_response) - ) - ) - if anthropic_response is not None: - return anthropic_response - raise ValueError("Failed to transform response to Anthropic format") - except Exception as e: # noqa: BLE001 - raise ValueError( - f"Error calling litellm.acompletion for non-Anthropic model: {str(e)}" ) + if transformed_stream is not None: + return transformed_stream + raise ValueError("Failed to transform streaming response") + else: + anthropic_response = ( + ANTHROPIC_ADAPTER.translate_completion_output_params( + cast(ModelResponse, completion_response) + ) + ) + if anthropic_response is not None: + return anthropic_response + raise ValueError("Failed to transform response to Anthropic format") @staticmethod def anthropic_messages_handler( @@ -239,29 +234,24 @@ class LiteLLMMessagesToCompletionTransformationHandler: ) ) - try: - completion_response = litellm.completion(**completion_kwargs) + completion_response = litellm.completion(**completion_kwargs) - if stream: - transformed_stream = ( - ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( - completion_response, - model=model, - ) + if stream: + transformed_stream = ( + ANTHROPIC_ADAPTER.translate_completion_output_params_streaming( + completion_response, + model=model, ) - if transformed_stream is not None: - return transformed_stream - raise ValueError("Failed to transform streaming response") - else: - anthropic_response = ( - ANTHROPIC_ADAPTER.translate_completion_output_params( - cast(ModelResponse, completion_response) - ) - ) - if anthropic_response is not None: - return anthropic_response - raise ValueError("Failed to transform response to Anthropic format") - except Exception as e: # noqa: BLE001 - raise ValueError( - f"Error calling litellm.completion for non-Anthropic model: {str(e)}" ) + if transformed_stream is not None: + return transformed_stream + raise ValueError("Failed to transform streaming response") + else: + anthropic_response = ( + ANTHROPIC_ADAPTER.translate_completion_output_params( + cast(ModelResponse, completion_response) + ) + ) + if anthropic_response is not None: + return anthropic_response + raise ValueError("Failed to transform response to Anthropic format") diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py index 47263dc1748..ecad7a50011 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py @@ -31,7 +31,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): sent_first_chunk: bool = False sent_content_block_start: bool = False sent_content_block_finish: bool = False - current_content_block_type: Literal["text", "tool_use"] = "text" + current_content_block_type: Literal["text", "tool_use", "thinking"] = "text" sent_last_message: bool = False holding_chunk: Optional[Any] = None holding_stop_reason_chunk: Optional[Any] = None diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 7de2a1e1c66..a786f06921f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -20,11 +20,15 @@ from litellm.types.llms.anthropic import ( AnthropicMessagesRequest, AnthropicMessagesToolChoice, AnthropicMessagesUserMessageParam, + AnthropicResponseContentBlockRedactedThinking, AnthropicResponseContentBlockText, + AnthropicResponseContentBlockThinking, AnthropicResponseContentBlockToolUse, ContentBlockDelta, ContentJsonBlockDelta, ContentTextBlockDelta, + ContentThinkingBlockDelta, + ContentThinkingSignatureBlockDelta, MessageBlockDelta, MessageDelta, UsageDelta, @@ -39,9 +43,11 @@ from litellm.types.llms.openai import ( ChatCompletionAssistantToolCall, ChatCompletionImageObject, ChatCompletionImageUrlObject, + ChatCompletionRedactedThinkingBlock, ChatCompletionRequest, ChatCompletionSystemMessage, ChatCompletionTextObject, + ChatCompletionThinkingBlock, ChatCompletionToolCallFunctionChunk, ChatCompletionToolChoiceFunctionParam, ChatCompletionToolChoiceObjectParam, @@ -51,7 +57,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolParamFunctionChunk, ChatCompletionUserMessage, ) -from litellm.types.utils import Choices, ModelResponse, Usage +from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage from .streaming_iterator import AnthropicStreamWrapper @@ -103,7 +109,6 @@ class AnthropicAdapter: def translate_completion_output_params( self, response: ModelResponse ) -> Optional[AnthropicMessagesResponse]: - return LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic( response=response ) @@ -162,14 +167,20 @@ class LiteLLMAnthropicMessagesAdapter: ) new_user_content_list.append(text_obj) elif content.get("type") == "image": - image_url = ChatCompletionImageUrlObject( - url=f"data:{content.get('type', '')};base64,{content.get('source', '')}" - ) - image_obj = ChatCompletionImageObject( - type="image_url", image_url=image_url + # Convert Anthropic image format to OpenAI format + source = content.get("source", {}) + openai_image_url = ( + self._translate_anthropic_image_to_openai(source) ) - new_user_content_list.append(image_obj) + if openai_image_url: + image_url_obj = ChatCompletionImageUrlObject( + url=openai_image_url + ) + image_obj = ChatCompletionImageObject( + type="image_url", image_url=image_url_obj + ) + new_user_content_list.append(image_obj) elif content.get("type") == "tool_result": if "content" not in content: tool_result = ChatCompletionToolMessage( @@ -205,13 +216,21 @@ class LiteLLMAnthropicMessagesAdapter: ) tool_message_list.append(tool_result) elif c.get("type") == "image": - image_str = f"data:{c.get('type', '')};base64,{c.get('source', '')}" + # Convert Anthropic image format to OpenAI format for tool results + source = c.get("source", {}) + openai_image_url = ( + self._translate_anthropic_image_to_openai( + source + ) + or "" + ) + tool_result = ChatCompletionToolMessage( role="tool", tool_call_id=content.get( "tool_use_id", "" ), - content=image_str, + content=openai_image_url, ) tool_message_list.append(tool_result) @@ -227,6 +246,9 @@ class LiteLLMAnthropicMessagesAdapter: ## ASSISTANT MESSAGE ## assistant_message_str: Optional[str] = None tool_calls: List[ChatCompletionAssistantToolCall] = [] + thinking_blocks: List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] = [] if m["role"] == "assistant": if isinstance(m.get("content"), str): assistant_message_str = str(m.get("content", "")) @@ -253,14 +275,40 @@ class LiteLLMAnthropicMessagesAdapter: function=function_chunk, ) ) + elif content.get("type") == "thinking": + thinking_block = ChatCompletionThinkingBlock( + type="thinking", + thinking=content.get("thinking") or "", + signature=content.get("signature") or "", + cache_control=content.get("cache_control", {}), + ) + thinking_blocks.append(thinking_block) + elif content.get("type") == "redacted_thinking": + redacted_thinking_block = ( + ChatCompletionRedactedThinkingBlock( + type="redacted_thinking", + data=content.get("data") or "", + cache_control=content.get("cache_control", {}), + ) + ) + thinking_blocks.append(redacted_thinking_block) - if assistant_message_str is not None or len(tool_calls) > 0: + if ( + assistant_message_str is not None + or len(tool_calls) > 0 + or len(thinking_blocks) > 0 + ): assistant_message = ChatCompletionAssistantMessage( role="assistant", content=assistant_message_str, + thinking_blocks=( + thinking_blocks if len(thinking_blocks) > 0 else None + ), ) if len(tool_calls) > 0: assistant_message["tool_calls"] = tool_calls + if len(thinking_blocks) > 0: + assistant_message["thinking_blocks"] = thinking_blocks # type: ignore new_messages.append(assistant_message) return new_messages @@ -313,6 +361,7 @@ class LiteLLMAnthropicMessagesAdapter: """ This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format. """ + # Debug: Processing Anthropic message request new_messages: List[AllMessageValues] = [] ## CONVERT ANTHROPIC MESSAGES TO OPENAI @@ -380,17 +429,84 @@ class LiteLLMAnthropicMessagesAdapter: return new_kwargs - def _translate_openai_content_to_anthropic( - self, choices: List[Choices] - ) -> List[ - Union[AnthropicResponseContentBlockText, AnthropicResponseContentBlockToolUse] + def _translate_anthropic_image_to_openai(self, image_source: dict) -> Optional[str]: + """ + Translate Anthropic image source format to OpenAI-compatible image URL. + + Anthropic supports two image source formats: + 1. Base64: {"type": "base64", "media_type": "image/jpeg", "data": "..."} + 2. URL: {"type": "url", "url": "https://..."} + + Returns the properly formatted image URL string, or None if invalid format. + """ + if not isinstance(image_source, dict): + return None + + source_type = image_source.get("type") + + if source_type == "base64": + # Base64 image format + media_type = image_source.get("media_type", "image/jpeg") + image_data = image_source.get("data", "") + if image_data: + return f"data:{media_type};base64,{image_data}" + elif source_type == "url": + # URL-referenced image format + return image_source.get("url", "") + + return None + + def _translate_openai_content_to_anthropic(self, choices: List[Choices]) -> List[ + Union[ + AnthropicResponseContentBlockText, + AnthropicResponseContentBlockToolUse, + AnthropicResponseContentBlockThinking, + AnthropicResponseContentBlockRedactedThinking, + ] ]: new_content: List[ Union[ - AnthropicResponseContentBlockText, AnthropicResponseContentBlockToolUse + AnthropicResponseContentBlockText, + AnthropicResponseContentBlockToolUse, + AnthropicResponseContentBlockThinking, + AnthropicResponseContentBlockRedactedThinking, ] ] = [] for choice in choices: + # Handle thinking blocks first + if ( + hasattr(choice.message, "thinking_blocks") + and choice.message.thinking_blocks + ): + for thinking_block in choice.message.thinking_blocks: + if thinking_block.get("type") == "thinking": + thinking_value = thinking_block.get("thinking", "") + signature_value = thinking_block.get("signature", "") + new_content.append( + AnthropicResponseContentBlockThinking( + type="thinking", + thinking=( + str(thinking_value) + if thinking_value is not None + else "" + ), + signature=( + str(signature_value) + if signature_value is not None + else None + ), + ) + ) + elif thinking_block.get("type") == "redacted_thinking": + data_value = thinking_block.get("data", "") + new_content.append( + AnthropicResponseContentBlockRedactedThinking( + type="redacted_thinking", + data=str(data_value) if data_value is not None else "", + ) + ) + + # Handle tool calls if ( choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0 @@ -401,9 +517,14 @@ class LiteLLMAnthropicMessagesAdapter: type="tool_use", id=tool_call.id, name=tool_call.function.name or "", - input=json.loads(tool_call.function.arguments) if tool_call.function.arguments else {}, + input=( + json.loads(tool_call.function.arguments) + if tool_call.function.arguments + else {} + ), ) ) + # Handle text content elif choice.message.content is not None: new_content.append( AnthropicResponseContentBlockText( @@ -453,13 +574,12 @@ class LiteLLMAnthropicMessagesAdapter: return translated_obj def _translate_streaming_openai_chunk_to_anthropic_content_block( - self, choices: List[OpenAIStreamingChoice] + self, choices: List[Union[OpenAIStreamingChoice, StreamingChoices]] ) -> Tuple[ - Literal["text", "tool_use"], + Literal["text", "tool_use", "thinking"], "ContentBlockContentBlockDict", ]: from litellm._uuid import uuid - from litellm.types.llms.anthropic import TextBlock, ToolUseBlock for choice in choices: @@ -476,17 +596,45 @@ class LiteLLMAnthropicMessagesAdapter: name=choice.delta.tool_calls[0].function.name or "", input={}, ) + elif isinstance(choice, StreamingChoices) and hasattr( + choice.delta, "thinking_blocks" + ): + thinking_blocks = choice.delta.thinking_blocks or [] + if len(thinking_blocks) > 0: + thinking_block = thinking_blocks[0] + if thinking_block["type"] == "thinking": + thinking = thinking_block.get("thinking") or "" + signature = thinking_block.get("signature") or "" + + assert isinstance(thinking, str) + assert isinstance(signature, str) + + if thinking and signature: + raise ValueError( + "Both `thinking` and `signature` in a single streaming chunk isn't supported." + ) + + return "thinking", ChatCompletionThinkingBlock( + type="thinking", thinking=thinking, signature=signature + ) return "text", TextBlock(type="text", text="") def _translate_streaming_openai_chunk_to_anthropic( - self, choices: List[OpenAIStreamingChoice] + self, choices: List[Union[OpenAIStreamingChoice, StreamingChoices]] ) -> Tuple[ - Literal["text_delta", "input_json_delta"], - Union[ContentTextBlockDelta, ContentJsonBlockDelta], + Literal["text_delta", "input_json_delta", "thinking_delta", "signature_delta"], + Union[ + ContentTextBlockDelta, + ContentJsonBlockDelta, + ContentThinkingBlockDelta, + ContentThinkingSignatureBlockDelta, + ], ]: text: str = "" + reasoning_content: str = "" + reasoning_signature: str = "" partial_json: Optional[str] = None for choice in choices: if choice.delta.content is not None and len(choice.delta.content) > 0: @@ -498,11 +646,40 @@ class LiteLLMAnthropicMessagesAdapter: tool.function is not None and tool.function.arguments is not None ): - partial_json += tool.function.arguments + partial_json = (partial_json or "") + tool.function.arguments + elif isinstance(choice, StreamingChoices) and hasattr( + choice.delta, "thinking_blocks" + ): + thinking_blocks = choice.delta.thinking_blocks or [] + if len(thinking_blocks) > 0: + for thinking_block in thinking_blocks: + if thinking_block["type"] == "thinking": + thinking = thinking_block.get("thinking") or "" + signature = thinking_block.get("signature") or "" + + assert isinstance(thinking, str) + assert isinstance(signature, str) + + reasoning_content += thinking + reasoning_signature += signature + + if reasoning_content and reasoning_signature: + raise ValueError( + "Both `reasoning` and `signature` in a single streaming chunk isn't supported." + ) + if partial_json is not None: return "input_json_delta", ContentJsonBlockDelta( type="input_json_delta", partial_json=partial_json ) + elif reasoning_content: + return "thinking_delta", ContentThinkingBlockDelta( + type="thinking_delta", thinking=reasoning_content + ) + elif reasoning_signature: + return "signature_delta", ContentThinkingSignatureBlockDelta( + type="signature_delta", signature=reasoning_signature + ) else: return "text_delta", ContentTextBlockDelta(type="text_delta", text=text) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 46ba96f2605..85b9ae1f034 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -2,7 +2,7 @@ from typing import Any, AsyncIterator, Dict, List, Optional, Tuple import httpx -from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj, verbose_logger from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -94,6 +94,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): status_code=400, ) ####### get required params for all anthropic messages requests ###### + verbose_logger.debug(f"🔍 TRANSFORMATION DEBUG - Messages: {messages}") anthropic_messages_request: AnthropicMessagesRequest = AnthropicMessagesRequest( messages=messages, max_tokens=max_tokens, diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index 7c5b693b453..e7aa93ac882 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -36,6 +36,7 @@ from .common_utils import ( process_azure_headers, select_azure_base_url_or_endpoint, ) +from .image_generation import get_azure_image_generation_config class AzureOpenAIAssistantsAPIConfig: @@ -1011,7 +1012,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): async def aimage_generation( self, data: dict, - model_response: ModelResponse, + model_response: Optional[ImageResponse], azure_client_params: dict, api_key: str, input: list, @@ -1020,6 +1021,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): client=None, timeout=None, ) -> litellm.ImageResponse: + response: Optional[dict] = None try: # response = await azure_client.images.generate(**data, timeout=timeout) @@ -1052,21 +1054,38 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): data=data, headers=headers, ) - response = httpx_response.json() - stringified_response = response - ## LOGGING - logging_obj.post_call( - input=input, - api_key=api_key, - additional_args={"complete_input_dict": data}, - original_response=stringified_response, - ) - return convert_to_model_response_object( # type: ignore - response_object=stringified_response, - model_response_object=model_response, - response_type="image_generation", + provider_config = get_azure_image_generation_config( + data.get("model", "dall-e-2") ) + if provider_config is not None: + return provider_config.transform_image_generation_response( + model=data.get("model", "dall-e-2"), + raw_response=httpx_response, + model_response=model_response or ImageResponse(), + logging_obj=logging_obj, + request_data=data, + optional_params=data, + litellm_params=data, + encoding=litellm.encoding, + ) + + else: + response = httpx_response.json() + + stringified_response = response + ## LOGGING + logging_obj.post_call( + input=input, + api_key=api_key, + additional_args={"complete_input_dict": data}, + original_response=stringified_response, + ) + return convert_to_model_response_object( # type: ignore + response_object=stringified_response, + model_response_object=model_response, + response_type="image_generation", + ) except Exception as e: ## LOGGING logging_obj.post_call( @@ -1110,7 +1129,11 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): "base_model" ) - data = {"model": model, "prompt": prompt, **optional_params} + # Azure image generation API doesn't support extra_body parameter + extra_body = optional_params.pop("extra_body", {}) + flattened_params = {**optional_params, **extra_body} + + data = {"model": model, "prompt": prompt, **flattened_params} max_retries = data.pop("max_retries", 2) if not isinstance(max_retries, int): raise AzureOpenAIError( @@ -1120,9 +1143,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): if api_key is None and azure_ad_token_provider is not None: azure_ad_token = azure_ad_token_provider() if azure_ad_token: - headers.pop( - "api-key", None - ) + headers.pop("api-key", None) headers["Authorization"] = f"Bearer {azure_ad_token}" # init AzureOpenAI Client diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index dfe662cc165..74520942619 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -500,23 +500,18 @@ class BaseAzureLLM(BaseOpenAILLM): azure_ad_token_provider = litellm_params.get("azure_ad_token_provider") # If we have api_key, then we have higher priority azure_ad_token = litellm_params.get("azure_ad_token") - tenant_id = litellm_params.get("tenant_id", os.getenv("AZURE_TENANT_ID")) - client_id = litellm_params.get("client_id", os.getenv("AZURE_CLIENT_ID")) - client_secret = litellm_params.get( - "client_secret", os.getenv("AZURE_CLIENT_SECRET") - ) - azure_username = litellm_params.get( - "azure_username", os.getenv("AZURE_USERNAME") - ) - azure_password = litellm_params.get( - "azure_password", os.getenv("AZURE_PASSWORD") - ) - scope = litellm_params.get( - "azure_scope", - os.getenv("AZURE_SCOPE", "https://cognitiveservices.azure.com/.default"), - ) + + # litellm_params sometimes contains the key, but the value is None + # We should respect environment variables in this case + tenant_id = self._resolve_env_var(litellm_params, "tenant_id", "AZURE_TENANT_ID") + client_id = self._resolve_env_var(litellm_params, "client_id", "AZURE_CLIENT_ID") + client_secret = self._resolve_env_var(litellm_params, "client_secret", "AZURE_CLIENT_SECRET") + azure_username = self._resolve_env_var(litellm_params, "azure_username", "AZURE_USERNAME") + azure_password = self._resolve_env_var(litellm_params, "azure_password", "AZURE_PASSWORD") + scope = self._resolve_env_var(litellm_params, "azure_scope", "AZURE_SCOPE") if scope is None: scope = "https://cognitiveservices.azure.com/.default" + max_retries = litellm_params.get("max_retries") timeout = litellm_params.get("timeout") if ( @@ -759,4 +754,17 @@ class BaseAzureLLM(BaseOpenAILLM): def _is_azure_v1_api_version(api_version: Optional[str]) -> bool: if api_version is None: return False - return api_version == "preview" or api_version == "latest" + return api_version in {"preview", "latest", "v1"} + + def _resolve_env_var(self, litellm_params: Dict[str, Any], param_key: str, env_var_key: str) -> Optional[str]: + """Resolve the environment variable for a given parameter key. + + The logic here is different from `params.get(key, os.getenv(env_var))` because + litellm_params may contain the key with a None value, in which case we want + to fallback to the environment variable. + """ + param_value = litellm_params.get(param_key) + if param_value is not None: + return param_value + return os.getenv(env_var_key) + diff --git a/litellm/llms/azure/exception_mapping.py b/litellm/llms/azure/exception_mapping.py new file mode 100644 index 00000000000..70c2609c6b4 --- /dev/null +++ b/litellm/llms/azure/exception_mapping.py @@ -0,0 +1,42 @@ +from typing import Optional + +from litellm.exceptions import ContentPolicyViolationError + + +class AzureOpenAIExceptionMapping: + """ + Class for creating Azure OpenAI specific exceptions + """ + @staticmethod + def create_content_policy_violation_error( + message: str, + model: str, + extra_information: str, + original_exception: Exception, + ) -> ContentPolicyViolationError: + """ + Create a content policy violation error + """ + raise ContentPolicyViolationError( + message=f"litellm.ContentPolicyViolationError: AzureException - {message}", + llm_provider="azure", + model=model, + litellm_debug_info=extra_information, + response=getattr(original_exception, "response", None), + provider_specific_fields={ + "innererror": AzureOpenAIExceptionMapping._get_innererror_from_exception(original_exception) + }, + ) + + @staticmethod + def _get_innererror_from_exception(original_exception: Exception) -> Optional[dict]: + """ + Azure OpenAI returns the innererror in the body of the exception + This method extracts the innererror from the exception + """ + innererror = None + body_dict = getattr(original_exception, "body", None) or {} + if isinstance(body_dict, dict): + innererror = body_dict.get("innererror") + return innererror + \ No newline at end of file diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index c5447b4ccd9..23c04e640c4 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -6,6 +6,8 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. from typing import Any, Optional, cast +from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES + from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from ....litellm_core_utils.realtime_streaming import RealTimeStreaming from ..azure import AzureChatCompletion @@ -64,6 +66,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): extra_headers={ "api-key": api_key, # type: ignore }, + max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ) as backend_ws: realtime_streaming = RealTimeStreaming( websocket, cast(ClientConnection, backend_ws), logging_obj diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 1516ed089ee..d621cb209d7 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -50,8 +50,6 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): try: # Ensure required fields are present for ResponseReasoningItem item_data = dict(item) - if "id" not in item_data: - item_data["id"] = f"rs_{hash(str(item_data))}" if "summary" not in item_data: item_data["summary"] = ( item_data.get("reasoning_content", "")[:100] + "..." diff --git a/litellm/llms/azure/text_to_speech/__init__.py b/litellm/llms/azure/text_to_speech/__init__.py new file mode 100644 index 00000000000..ee923f122bd --- /dev/null +++ b/litellm/llms/azure/text_to_speech/__init__.py @@ -0,0 +1,8 @@ +"""Azure Text-to-Speech module""" + +from .transformation import AzureAVATextToSpeechConfig + +__all__ = [ + "AzureAVATextToSpeechConfig", +] + diff --git a/litellm/llms/azure/text_to_speech/transformation.py b/litellm/llms/azure/text_to_speech/transformation.py new file mode 100644 index 00000000000..0f8911ac2b8 --- /dev/null +++ b/litellm/llms/azure/text_to_speech/transformation.py @@ -0,0 +1,487 @@ +""" +Azure AVA (Cognitive Services) Text-to-Speech transformation + +Maps OpenAI TTS spec to Azure Cognitive Services TTS API +""" + +from typing import TYPE_CHECKING, Any, Coroutine, Dict, Optional, Tuple, Union +from urllib.parse import urlparse + +import httpx + +import litellm +from litellm.llms.base_llm.text_to_speech.transformation import ( + BaseTextToSpeechConfig, + TextToSpeechRequestData, +) +from litellm.secret_managers.main import get_secret_str + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.llms.openai import HttpxBinaryResponseContent +else: + LiteLLMLoggingObj = Any + HttpxBinaryResponseContent = Any + + +class AzureAVATextToSpeechConfig(BaseTextToSpeechConfig): + """ + Configuration for Azure AVA (Cognitive Services) Text-to-Speech + + Reference: https://learn.microsoft.com/en-us/azure/ai-services/speech-service/rest-text-to-speech + """ + + # Azure endpoint domains + DEFAULT_VOICE = "en-US-AriaNeural" + COGNITIVE_SERVICES_DOMAIN = "api.cognitive.microsoft.com" + TTS_SPEECH_DOMAIN = "tts.speech.microsoft.com" + TTS_ENDPOINT_PATH = "/cognitiveservices/v1" + + # Voice name mappings from OpenAI voices to Azure voices + VOICE_MAPPINGS = { + "alloy": "en-US-JennyNeural", + "echo": "en-US-GuyNeural", + "fable": "en-GB-RyanNeural", + "onyx": "en-US-DavisNeural", + "nova": "en-US-AmberNeural", + "shimmer": "en-US-AriaNeural", + } + + # Response format mappings from OpenAI to Azure + FORMAT_MAPPINGS = { + "mp3": "audio-24khz-48kbitrate-mono-mp3", + "opus": "ogg-48khz-16bit-mono-opus", + "aac": "audio-24khz-48kbitrate-mono-mp3", # Azure doesn't have AAC, use MP3 + "flac": "audio-24khz-48kbitrate-mono-mp3", # Azure doesn't have FLAC, use MP3 + "wav": "riff-24khz-16bit-mono-pcm", + "pcm": "raw-24khz-16bit-mono-pcm", + } + + def dispatch_text_to_speech( + self, + model: str, + input: str, + voice: Optional[Union[str, Dict]], + optional_params: Dict, + litellm_params_dict: Dict, + logging_obj: "LiteLLMLoggingObj", + timeout: Union[float, httpx.Timeout], + extra_headers: Optional[Dict[str, Any]], + base_llm_http_handler: Any, + aspeech: bool, + api_base: Optional[str], + api_key: Optional[str], + **kwargs: Any, + ) -> Union[ + "HttpxBinaryResponseContent", + Coroutine[Any, Any, "HttpxBinaryResponseContent"], + ]: + """ + Dispatch method to handle Azure AVA TTS requests + + This method encapsulates Azure-specific credential resolution and parameter handling + + Args: + base_llm_http_handler: The BaseLLMHTTPHandler instance from main.py + """ + # Resolve api_base from multiple sources + api_base = ( + api_base + or litellm_params_dict.get("api_base") + or litellm.api_base + or get_secret_str("AZURE_API_BASE") + ) + + # Resolve api_key from multiple sources (Azure-specific) + api_key = ( + api_key + or litellm_params_dict.get("api_key") + or litellm.api_key + or litellm.azure_key + or get_secret_str("AZURE_OPENAI_API_KEY") + or get_secret_str("AZURE_API_KEY") + ) + + # Convert voice to string if it's a dict (for Azure AVA, voice must be a string) + voice_str: Optional[str] = None + if isinstance(voice, str): + voice_str = voice + elif isinstance(voice, dict): + # Extract voice name from dict if needed + voice_str = voice.get("name") if voice else None + + litellm_params_dict.update({ + "api_key": api_key, + "api_base": api_base, + }) + # Call the text_to_speech_handler + response = base_llm_http_handler.text_to_speech_handler( + model=model, + input=input, + voice=voice_str, + text_to_speech_provider_config=self, + text_to_speech_optional_params=optional_params, + custom_llm_provider="azure", + litellm_params=litellm_params_dict, + logging_obj=logging_obj, + timeout=timeout, + extra_headers=extra_headers, + client=None, + _is_async=aspeech, + ) + + return response + + def get_supported_openai_params(self, model: str) -> list: + """ + Azure AVA TTS supports these OpenAI parameters + + Note: Azure also supports additional SSML-specific parameters (style, styledegree, role) + which can be passed but are not part of the OpenAI spec + """ + return ["voice", "response_format", "speed"] + + def _convert_speed_to_azure_rate(self, speed: float) -> str: + """ + Convert OpenAI speed value to Azure SSML prosody rate percentage + + Args: + speed: OpenAI speed value (0.25-4.0, default 1.0) + + Returns: + Azure rate string with percentage (e.g., "+50%", "-50%", "+0%") + + Examples: + speed=1.0 -> "+0%" (default) + speed=2.0 -> "+100%" + speed=0.5 -> "-50%" + """ + rate_percentage = int((speed - 1.0) * 100) + return f"{rate_percentage:+d}%" + + def _build_express_as_element( + self, + content: str, + style: Optional[str] = None, + styledegree: Optional[str] = None, + role: Optional[str] = None, + ) -> str: + """ + Build mstts:express-as element with optional style, styledegree, and role attributes + + Args: + content: The inner content to wrap + style: Speaking style (e.g., "cheerful", "sad", "angry") + styledegree: Style intensity (0.01 to 2) + role: Voice role (e.g., "Girl", "Boy", "SeniorFemale", "SeniorMale") + + Returns: + Content wrapped in mstts:express-as if any attributes provided, otherwise raw content + """ + if not (style or styledegree or role): + return content + + express_as_attrs = [] + if style: + express_as_attrs.append(f"style='{style}'") + if styledegree: + express_as_attrs.append(f"styledegree='{styledegree}'") + if role: + express_as_attrs.append(f"role='{role}'") + + express_as_attrs_str = " ".join(express_as_attrs) + return f"{content}" + + def _get_voice_language( + self, + voice_name: Optional[str], + explicit_lang: Optional[str] = None, + ) -> Optional[str]: + """ + Get the language for the voice element's xml:lang attribute + + Args: + voice_name: The Azure voice name (e.g., "en-US-AriaNeural") + explicit_lang: Explicitly provided language code (takes precedence) + + Returns: + Language code if available (e.g., "es-ES"), or None + + Examples: + - explicit_lang="es-ES" → "es-ES" (explicit takes precedence) + - voice_name="en-US-AriaNeural", explicit_lang=None → None (use default from voice) + - voice_name="en-US-AvaMultilingualNeural", explicit_lang="fr-FR" → "fr-FR" + """ + # If explicit language is provided, use it (for multilingual voices) + if explicit_lang: + return explicit_lang + + # For non-multilingual voices, we don't need to set xml:lang on the voice element + # The voice name already encodes the language (e.g., en-US-AriaNeural) + # Only return a language if explicitly set + return None + + def map_openai_params( + self, + model: str, + optional_params: Dict, + voice: Optional[Union[str, Dict]] = None, + drop_params: bool = False, + kwargs: Dict = {}, + ) -> Tuple[Optional[str], Dict]: + """ + Map OpenAI parameters to Azure AVA TTS parameters + """ + mapped_params = {} + ########################################################## + # Map voice + # OpenAI uses voice as a required param, hence not in optional_params + ########################################################## + # If it's already an Azure voice, use it directly + mapped_voice: Optional[str] = None + if isinstance(voice, str): + if voice in self.VOICE_MAPPINGS: + mapped_voice = self.VOICE_MAPPINGS[voice] + else: + # Assume it's already an Azure voice name + mapped_voice = voice + + # Map response format + if "response_format" in optional_params: + format_name = optional_params["response_format"] + if format_name in self.FORMAT_MAPPINGS: + mapped_params["output_format"] = self.FORMAT_MAPPINGS[format_name] + else: + # Try to use it directly as Azure format + mapped_params["output_format"] = format_name + else: + # Default to MP3 + mapped_params["output_format"] = "audio-24khz-48kbitrate-mono-mp3" + + # Map speed (OpenAI: 0.25-4.0, Azure: prosody rate) + if "speed" in optional_params: + speed = optional_params["speed"] + if speed is not None: + mapped_params["rate"] = self._convert_speed_to_azure_rate(speed=speed) + + # Pass through Azure-specific SSML parameters + if "style" in kwargs: + mapped_params["style"] = kwargs["style"] + + if "styledegree" in kwargs: + mapped_params["styledegree"] = kwargs["styledegree"] + + if "role" in kwargs: + mapped_params["role"] = kwargs["role"] + + if "lang" in kwargs: + mapped_params["lang"] = kwargs["lang"] + return mapped_voice, mapped_params + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate Azure environment and set up authentication headers + """ + validated_headers = headers.copy() + + # Azure AVA TTS requires either: + # 1. Ocp-Apim-Subscription-Key header, or + # 2. Authorization: Bearer header + + # We'll use the token-based auth via our token handler + # The token will be added later in the handler + + if api_key: + # If subscription key is provided, use it directly + validated_headers["Ocp-Apim-Subscription-Key"] = api_key + + # Content-Type for SSML + validated_headers["Content-Type"] = "application/ssml+xml" + + # User-Agent + validated_headers["User-Agent"] = "litellm" + + return validated_headers + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the complete URL for Azure AVA TTS request + + Azure TTS endpoint format: + https://{region}.tts.speech.microsoft.com/cognitiveservices/v1 + """ + if api_base is None: + raise ValueError( + f"api_base is required for Azure AVA TTS. " + f"Format: https://{{region}}.{self.COGNITIVE_SERVICES_DOMAIN} or " + f"https://{{region}}.{self.TTS_SPEECH_DOMAIN}" + ) + + # Remove trailing slash and parse URL + api_base = api_base.rstrip("/") + parsed_url = urlparse(api_base) + hostname = parsed_url.hostname or "" + + # Check if it's a Cognitive Services endpoint (convert to TTS endpoint) + if self._is_cognitive_services_endpoint(hostname=hostname): + region = self._extract_region_from_hostname( + hostname=hostname, + domain=self.COGNITIVE_SERVICES_DOMAIN + ) + return self._build_tts_url(region=region) + + # Check if it's already a TTS endpoint + if self._is_tts_endpoint(hostname=hostname): + if not api_base.endswith(self.TTS_ENDPOINT_PATH): + return f"{api_base}{self.TTS_ENDPOINT_PATH}" + return api_base + + # Assume it's a custom endpoint, append the path + return f"{api_base}{self.TTS_ENDPOINT_PATH}" + + def _is_cognitive_services_endpoint(self, hostname: str) -> bool: + """Check if hostname is a Cognitive Services endpoint""" + return ( + hostname == self.COGNITIVE_SERVICES_DOMAIN + or hostname.endswith(f".{self.COGNITIVE_SERVICES_DOMAIN}") + ) + + def _is_tts_endpoint(self, hostname: str) -> bool: + """Check if hostname is a TTS endpoint""" + return ( + hostname == self.TTS_SPEECH_DOMAIN + or hostname.endswith(f".{self.TTS_SPEECH_DOMAIN}") + ) + + def _extract_region_from_hostname(self, hostname: str, domain: str) -> str: + """ + Extract region from hostname + + Examples: + eastus.api.cognitive.microsoft.com -> eastus + api.cognitive.microsoft.com -> "" + """ + if hostname.endswith(f".{domain}"): + return hostname[:-len(f".{domain}")] + return "" + + def _build_tts_url(self, region: str) -> str: + """Build the complete TTS URL with region""" + if region: + return f"https://{region}.{self.TTS_SPEECH_DOMAIN}{self.TTS_ENDPOINT_PATH}" + return f"https://{self.TTS_SPEECH_DOMAIN}{self.TTS_ENDPOINT_PATH}" + + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: Optional[str], + optional_params: Dict, + litellm_params: Dict, + headers: dict, + ) -> TextToSpeechRequestData: + """ + Transform OpenAI TTS request to Azure AVA TTS SSML format + + Note: optional_params should already be mapped via map_openai_params in main.py + + Supports Azure-specific SSML features: + - style: Speaking style (e.g., "cheerful", "sad", "angry") + - styledegree: Style intensity (0.01 to 2) + - role: Voice role (e.g., "Girl", "Boy", "SeniorFemale", "SeniorMale") + - lang: Language code for multilingual voices (e.g., "es-ES", "fr-FR") + + Returns: + TextToSpeechRequestData: Contains SSML body and Azure-specific headers + """ + # Get voice (already mapped in main.py, or use default) + azure_voice = voice or self.DEFAULT_VOICE + + # Get output format (already mapped in main.py) + output_format = optional_params.get( + "output_format", "audio-24khz-48kbitrate-mono-mp3" + ) + headers["X-Microsoft-OutputFormat"] = output_format + + # Build SSML + rate = optional_params.get("rate", "0%") + style = optional_params.get("style") + styledegree = optional_params.get("styledegree") + role = optional_params.get("role") + lang = optional_params.get("lang") + + # Escape XML special characters in input text + escaped_input = ( + input.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) + + # Determine if we need mstts namespace (for express-as element) + use_mstts = style or role or styledegree + + # Build the xmlns attributes + if use_mstts: + xmlns = "xmlns='http://www.w3.org/2001/10/synthesis' xmlns:mstts='https://www.w3.org/2001/mstts'" + else: + xmlns = "xmlns='http://www.w3.org/2001/10/synthesis'" + + # Build the inner content with prosody + prosody_content = f"{escaped_input}" + + # Wrap in mstts:express-as if style or role is specified + voice_content = self._build_express_as_element( + content=prosody_content, + style=style, + styledegree=styledegree, + role=role, + ) + + # Build voice element with optional xml:lang attribute + voice_lang = self._get_voice_language( + voice_name=azure_voice, + explicit_lang=lang, + ) + voice_lang_attr = f" xml:lang='{voice_lang}'" if voice_lang else "" + + ssml_body = f""" + + {voice_content} + +""" + + return { + "ssml_body": ssml_body, + "headers": headers, + } + + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> "HttpxBinaryResponseContent": + """ + Transform Azure AVA TTS response to standard format + + Azure returns the audio data directly in the response body + """ + from litellm.types.llms.openai import HttpxBinaryResponseContent + + # Azure returns audio data directly in the response body + # Wrap it in HttpxBinaryResponseContent for consistent return type + return HttpxBinaryResponseContent(raw_response) + diff --git a/litellm/llms/azure/videos/transformation.py b/litellm/llms/azure/videos/transformation.py new file mode 100644 index 00000000000..3af9e0778bc --- /dev/null +++ b/litellm/llms/azure/videos/transformation.py @@ -0,0 +1,89 @@ +from typing import TYPE_CHECKING, Any, Dict, Optional + +from litellm.types.videos.main import VideoCreateOptionalRequestParams +from litellm.secret_managers.main import get_secret_str +from litellm.llms.azure.common_utils import BaseAzureLLM +import litellm +from litellm.llms.openai.videos.transformation import OpenAIVideoConfig +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + from ...base_llm.videos.transformation import BaseVideoConfig as _BaseVideoConfig + from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException + + LiteLLMLoggingObj = _LiteLLMLoggingObj + BaseVideoConfig = _BaseVideoConfig + BaseLLMException = _BaseLLMException +else: + LiteLLMLoggingObj = Any + BaseVideoConfig = Any + BaseLLMException = Any + + +class AzureVideoConfig(OpenAIVideoConfig): + """ + Configuration class for OpenAI video generation. + """ + + def __init__(self): + super().__init__() + + def get_supported_openai_params(self, model: str) -> list: + """ + Get the list of supported OpenAI parameters for video generation. + """ + return [ + "model", + "prompt", + "input_reference", + "seconds", + "size", + "user", + "extra_headers", + ] + + def map_openai_params( + self, + video_create_optional_params: VideoCreateOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict: + """No mapping applied since inputs are in OpenAI spec already""" + return dict(video_create_optional_params) + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret_str("AZURE_OPENAI_API_KEY") + or get_secret_str("AZURE_API_KEY") + ) + + headers.update( + { + "Authorization": f"Bearer {api_key}", + } + ) + return headers + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Constructs a complete URL for the API request. + """ + return BaseAzureLLM._get_base_azure_url( + api_base=api_base, + litellm_params=litellm_params, + route="/openai/v1/videos", + default_api_version="", + ) \ No newline at end of file diff --git a/litellm/llms/azure_ai/ocr/__init__.py b/litellm/llms/azure_ai/ocr/__init__.py new file mode 100644 index 00000000000..7182a750b45 --- /dev/null +++ b/litellm/llms/azure_ai/ocr/__init__.py @@ -0,0 +1,13 @@ +"""Azure AI OCR module.""" +from .common_utils import get_azure_ai_ocr_config +from .document_intelligence.transformation import ( + AzureDocumentIntelligenceOCRConfig, +) +from .transformation import AzureAIOCRConfig + +__all__ = [ + "AzureAIOCRConfig", + "AzureDocumentIntelligenceOCRConfig", + "get_azure_ai_ocr_config", +] + diff --git a/litellm/llms/azure_ai/ocr/common_utils.py b/litellm/llms/azure_ai/ocr/common_utils.py new file mode 100644 index 00000000000..ef470c74923 --- /dev/null +++ b/litellm/llms/azure_ai/ocr/common_utils.py @@ -0,0 +1,53 @@ +""" +Common utilities for Azure AI OCR providers. + +This module provides routing logic to determine which OCR configuration to use +based on the model name. +""" + +from typing import TYPE_CHECKING, Optional + +from litellm._logging import verbose_logger + +if TYPE_CHECKING: + from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig + + +def get_azure_ai_ocr_config(model: str) -> Optional["BaseOCRConfig"]: + """ + Determine which Azure AI OCR configuration to use based on the model name. + + Azure AI supports multiple OCR services: + - Azure Document Intelligence: azure_ai/doc-intelligence/ + - Mistral OCR (via Azure AI): azure_ai/ + + Args: + model: The model name (e.g., "azure_ai/doc-intelligence/prebuilt-read", + "azure_ai/pixtral-12b-2409") + + Returns: + OCR configuration instance for the specified model + + Examples: + >>> get_azure_ai_ocr_config("azure_ai/doc-intelligence/prebuilt-read") + + + >>> get_azure_ai_ocr_config("azure_ai/pixtral-12b-2409") + + """ + from litellm.llms.azure_ai.ocr.document_intelligence.transformation import ( + AzureDocumentIntelligenceOCRConfig, + ) + from litellm.llms.azure_ai.ocr.transformation import AzureAIOCRConfig + + # Check for Azure Document Intelligence models + if "doc-intelligence" in model or "documentintelligence" in model: + verbose_logger.debug( + f"Routing {model} to Azure Document Intelligence OCR config" + ) + return AzureDocumentIntelligenceOCRConfig() + + # Default to Mistral-based OCR for other azure_ai models + verbose_logger.debug(f"Routing {model} to Azure AI (Mistral) OCR config") + return AzureAIOCRConfig() + diff --git a/litellm/llms/azure_ai/ocr/document_intelligence/__init__.py b/litellm/llms/azure_ai/ocr/document_intelligence/__init__.py new file mode 100644 index 00000000000..372a6a8d761 --- /dev/null +++ b/litellm/llms/azure_ai/ocr/document_intelligence/__init__.py @@ -0,0 +1,5 @@ +"""Azure Document Intelligence OCR module.""" +from .transformation import AzureDocumentIntelligenceOCRConfig + +__all__ = ["AzureDocumentIntelligenceOCRConfig"] + diff --git a/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py b/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py new file mode 100644 index 00000000000..b1ccfc36d0d --- /dev/null +++ b/litellm/llms/azure_ai/ocr/document_intelligence/transformation.py @@ -0,0 +1,696 @@ +""" +Azure Document Intelligence OCR transformation implementation. + +Azure Document Intelligence (formerly Form Recognizer) provides advanced document analysis capabilities. +This implementation transforms between Mistral OCR format and Azure Document Intelligence API v4.0. + +Note: Azure Document Intelligence API is async - POST returns 202 Accepted with Operation-Location header. +The operation location must be polled until the analysis completes. +""" +import asyncio +import re +import time +from typing import Any, Dict, Optional + +import httpx + +from litellm._logging import verbose_logger +from litellm.constants import ( + AZURE_DOCUMENT_INTELLIGENCE_API_VERSION, + AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI, + AZURE_OPERATION_POLLING_TIMEOUT, +) +from litellm.llms.base_llm.ocr.transformation import ( + BaseOCRConfig, + DocumentType, + OCRPage, + OCRPageDimensions, + OCRRequestData, + OCRResponse, + OCRUsageInfo, +) +from litellm.secret_managers.main import get_secret_str + + +class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig): + """ + Azure Document Intelligence OCR transformation configuration. + + Supports Azure Document Intelligence v4.0 (2024-11-30) API. + Model route: azure_ai/doc-intelligence/ + + Supported models: + - prebuilt-layout: Extracts text with markdown, tables, and structure (closest to Mistral OCR) + - prebuilt-read: Basic text extraction optimized for reading + - prebuilt-document: General document analysis + + Reference: https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/ + """ + + def __init__(self) -> None: + super().__init__() + + def get_supported_ocr_params(self, model: str) -> list: + """ + Get supported OCR parameters for Azure Document Intelligence. + + Azure DI has minimal optional parameters compared to Mistral OCR. + Most Mistral-specific params are ignored during transformation. + """ + return [] + + def validate_environment( + self, + headers: Dict, + model: str, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + litellm_params: Optional[dict] = None, + **kwargs, + ) -> Dict: + """ + Validate environment and return headers for Azure Document Intelligence. + + Authentication uses Ocp-Apim-Subscription-Key header. + """ + # Get API key from environment if not provided + if api_key is None: + api_key = get_secret_str("AZURE_DOCUMENT_INTELLIGENCE_API_KEY") + + if api_key is None: + raise ValueError( + "Missing Azure Document Intelligence API Key - Set AZURE_DOCUMENT_INTELLIGENCE_API_KEY environment variable or pass api_key parameter" + ) + + # Validate API base/endpoint is provided + if api_base is None: + api_base = get_secret_str("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") + + if api_base is None: + raise ValueError( + "Missing Azure Document Intelligence Endpoint - Set AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT environment variable or pass api_base parameter" + ) + + headers = { + "Ocp-Apim-Subscription-Key": api_key, + "Content-Type": "application/json", + **headers, + } + + return headers + + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: dict, + litellm_params: Optional[dict] = None, + **kwargs, + ) -> str: + """ + Get complete URL for Azure Document Intelligence endpoint. + + Format: {endpoint}/documentintelligence/documentModels/{modelId}:analyze?api-version=2024-11-30 + + Note: API version 2024-11-30 uses /documentintelligence/ path (not /formrecognizer/) + + Args: + api_base: Azure Document Intelligence endpoint (e.g., https://your-resource.cognitiveservices.azure.com) + model: Model ID (e.g., "prebuilt-layout", "prebuilt-read") + optional_params: Optional parameters + + Returns: Complete URL for Azure DI analyze endpoint + """ + if api_base is None: + raise ValueError( + "Missing Azure Document Intelligence Endpoint - Set AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT environment variable or pass api_base parameter" + ) + + # Ensure no trailing slash + api_base = api_base.rstrip("/") + + # Extract model ID from full model path if needed + # Model can be "prebuilt-layout" or "azure_ai/doc-intelligence/prebuilt-layout" + model_id = model + if "/" in model: + # Extract the last part after the last slash + model_id = model.split("/")[-1] + + # Azure Document Intelligence analyze endpoint + # Note: API version 2024-11-30+ uses /documentintelligence/ (not /formrecognizer/) + return f"{api_base}/documentintelligence/documentModels/{model_id}:analyze?api-version={AZURE_DOCUMENT_INTELLIGENCE_API_VERSION}" + + def _extract_base64_from_data_uri(self, data_uri: str) -> str: + """ + Extract base64 content from a data URI. + + Args: + data_uri: Data URI like "data:application/pdf;base64,..." + + Returns: + Base64 string without the data URI prefix + """ + # Match pattern: data:[][;base64], + match = re.match(r"data:([^;]+)(?:;base64)?,(.+)", data_uri) + if match: + return match.group(2) + return data_uri + + def transform_ocr_request( + self, + model: str, + document: DocumentType, + optional_params: dict, + headers: dict, + **kwargs, + ) -> OCRRequestData: + """ + Transform OCR request to Azure Document Intelligence format. + + Mistral OCR format: + { + "document": { + "type": "document_url", + "document_url": "https://example.com/doc.pdf" + } + } + + Azure DI format: + { + "urlSource": "https://example.com/doc.pdf" + } + OR + { + "base64Source": "base64_encoded_content" + } + + Args: + model: Model name + document: Document dict from user (Mistral format) + optional_params: Already mapped optional parameters + headers: Request headers + + Returns: + OCRRequestData with JSON data + """ + verbose_logger.debug( + f"Azure Document Intelligence transform_ocr_request - model: {model}" + ) + + if not isinstance(document, dict): + raise ValueError(f"Expected document dict, got {type(document)}") + + # Extract document URL from Mistral format + doc_type = document.get("type") + document_url = None + + if doc_type == "document_url": + document_url = document.get("document_url", "") + elif doc_type == "image_url": + document_url = document.get("image_url", "") + else: + raise ValueError( + f"Invalid document type: {doc_type}. Must be 'document_url' or 'image_url'" + ) + + if not document_url: + raise ValueError("Document URL is required") + + # Build Azure DI request + data: Dict[str, Any] = {} + + # Check if it's a data URI (base64) + if document_url.startswith("data:"): + # Extract base64 content + base64_content = self._extract_base64_from_data_uri(document_url) + data["base64Source"] = base64_content + verbose_logger.debug("Using base64Source for Azure Document Intelligence") + else: + # Regular URL + data["urlSource"] = document_url + verbose_logger.debug("Using urlSource for Azure Document Intelligence") + + # Azure DI doesn't support most Mistral-specific params + # Ignore pages, include_image_base64, etc. + + return OCRRequestData(data=data, files=None) + + def _extract_page_markdown(self, page_data: Dict[str, Any]) -> str: + """ + Extract text from Azure DI page and format as markdown. + + Azure DI provides text in 'lines' array. We concatenate them with newlines. + + Args: + page_data: Azure DI page object + + Returns: + Markdown-formatted text + """ + lines = page_data.get("lines", []) + if not lines: + return "" + + # Extract text content from each line + text_lines = [line.get("content", "") for line in lines] + + # Join with newlines to preserve structure + return "\n".join(text_lines) + + def _convert_dimensions( + self, width: float, height: float, unit: str + ) -> OCRPageDimensions: + """ + Convert Azure DI dimensions to pixels. + + Azure DI provides dimensions in inches. We convert to pixels using configured DPI. + + Args: + width: Width in specified unit + height: Height in specified unit + unit: Unit of measurement (e.g., "inch") + + Returns: + OCRPageDimensions with pixel values + """ + # Convert to pixels using configured DPI + dpi = AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI + if unit == "inch": + width_px = int(width * dpi) + height_px = int(height * dpi) + else: + # If unit is not inches, assume it's already in pixels + width_px = int(width) + height_px = int(height) + + return OCRPageDimensions(width=width_px, height=height_px, dpi=dpi) + + @staticmethod + def _check_timeout(start_time: float, timeout_secs: int) -> None: + """ + Check if operation has timed out. + + Args: + start_time: Start time of the operation + timeout_secs: Timeout duration in seconds + + Raises: + TimeoutError: If operation has exceeded timeout + """ + if time.time() - start_time > timeout_secs: + raise TimeoutError( + f"Azure Document Intelligence operation polling timed out after {timeout_secs} seconds" + ) + + @staticmethod + def _get_retry_after(response: httpx.Response) -> int: + """ + Get retry-after duration from response headers. + + Args: + response: HTTP response + + Returns: + Retry-after duration in seconds (default: 2) + """ + retry_after = int(response.headers.get("retry-after", "2")) + verbose_logger.debug(f"Retry polling after: {retry_after} seconds") + return retry_after + + @staticmethod + def _check_operation_status(response: httpx.Response) -> str: + """ + Check Azure DI operation status from response. + + Args: + response: HTTP response from operation endpoint + + Returns: + Operation status string + + Raises: + ValueError: If operation failed or status is unknown + """ + try: + result = response.json() + status = result.get("status") + + verbose_logger.debug(f"Azure DI operation status: {status}") + + if status == "succeeded": + return "succeeded" + elif status == "failed": + error_msg = result.get("error", {}).get("message", "Unknown error") + raise ValueError( + f"Azure Document Intelligence analysis failed: {error_msg}" + ) + elif status in ["running", "notStarted"]: + return "running" + else: + raise ValueError(f"Unknown operation status: {status}") + + except Exception as e: + if "succeeded" in str(e) or "failed" in str(e): + raise + # If we can't parse JSON, something went wrong + raise ValueError(f"Failed to parse Azure DI operation response: {e}") + + def _poll_operation_sync( + self, + operation_url: str, + headers: Dict[str, str], + timeout_secs: int, + ) -> httpx.Response: + """ + Poll Azure Document Intelligence operation until completion (sync). + + Azure DI POST returns 202 with Operation-Location header. + We need to poll that URL until status is "succeeded" or "failed". + + Args: + operation_url: The Operation-Location URL to poll + headers: Request headers (including auth) + timeout_secs: Total timeout in seconds + + Returns: + Final response with completed analysis + """ + from litellm.llms.custom_httpx.http_handler import _get_httpx_client + + client = _get_httpx_client() + start_time = time.time() + + verbose_logger.debug(f"Polling Azure DI operation: {operation_url}") + + while True: + self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) + + # Poll the operation status + response = client.get(url=operation_url, headers=headers) + + # Check operation status + status = self._check_operation_status(response=response) + + if status == "succeeded": + return response + elif status == "running": + # Wait before polling again + retry_after = self._get_retry_after(response=response) + time.sleep(retry_after) + + async def _poll_operation_async( + self, + operation_url: str, + headers: Dict[str, str], + timeout_secs: int, + ) -> httpx.Response: + """ + Poll Azure Document Intelligence operation until completion (async). + + Args: + operation_url: The Operation-Location URL to poll + headers: Request headers (including auth) + timeout_secs: Total timeout in seconds + + Returns: + Final response with completed analysis + """ + import litellm + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + + client = get_async_httpx_client(llm_provider=litellm.LlmProviders.AZURE_AI) + start_time = time.time() + + verbose_logger.debug(f"Polling Azure DI operation (async): {operation_url}") + + while True: + self._check_timeout(start_time=start_time, timeout_secs=timeout_secs) + + # Poll the operation status + response = await client.get(url=operation_url, headers=headers) + + # Check operation status + status = self._check_operation_status(response=response) + + if status == "succeeded": + return response + elif status == "running": + # Wait before polling again + retry_after = self._get_retry_after(response=response) + await asyncio.sleep(retry_after) + + def transform_ocr_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: Any, + **kwargs, + ) -> OCRResponse: + """ + Transform Azure Document Intelligence response to Mistral OCR format. + + Handles async operation polling: If response is 202 Accepted, polls Operation-Location + until analysis completes. + + Azure DI response (after polling): + { + "status": "succeeded", + "analyzeResult": { + "content": "Full document text...", + "pages": [ + { + "pageNumber": 1, + "width": 8.5, + "height": 11, + "unit": "inch", + "lines": [{"content": "text", "boundingBox": [...]}] + } + ] + } + } + + Mistral OCR format: + { + "pages": [ + { + "index": 0, + "markdown": "extracted text", + "dimensions": {"width": 816, "height": 1056, "dpi": 96} + } + ], + "model": "azure_ai/doc-intelligence/prebuilt-layout", + "usage_info": {"pages_processed": 1}, + "object": "ocr" + } + + Args: + model: Model name + raw_response: Raw HTTP response from Azure DI (may be 202 Accepted) + logging_obj: Logging object + + Returns: + OCRResponse in Mistral format + """ + try: + # Check if we got 202 Accepted (async operation started) + if raw_response.status_code == 202: + verbose_logger.debug( + "Azure DI returned 202 Accepted, polling operation..." + ) + + # Get Operation-Location header + operation_url = raw_response.headers.get("Operation-Location") + if not operation_url: + raise ValueError( + "Azure Document Intelligence returned 202 but no Operation-Location header found" + ) + + # Get headers for polling (need auth) + poll_headers = { + "Ocp-Apim-Subscription-Key": raw_response.request.headers.get( + "Ocp-Apim-Subscription-Key", "" + ) + } + + # Get timeout from kwargs or use default + timeout_secs = AZURE_OPERATION_POLLING_TIMEOUT + + # Poll until operation completes + raw_response = self._poll_operation_sync( + operation_url=operation_url, + headers=poll_headers, + timeout_secs=timeout_secs, + ) + + # Now parse the completed response + response_json = raw_response.json() + + verbose_logger.debug( + f"Azure Document Intelligence response status: {response_json.get('status')}" + ) + + # Check if request succeeded + status = response_json.get("status") + if status != "succeeded": + raise ValueError( + f"Azure Document Intelligence analysis failed with status: {status}" + ) + + # Extract analyze result + analyze_result = response_json.get("analyzeResult", {}) + azure_pages = analyze_result.get("pages", []) + + # Transform pages to Mistral format + mistral_pages = [] + for azure_page in azure_pages: + page_number = azure_page.get("pageNumber", 1) + index = page_number - 1 # Convert to 0-based index + + # Extract markdown text + markdown = self._extract_page_markdown(azure_page) + + # Convert dimensions + width = azure_page.get("width", 8.5) + height = azure_page.get("height", 11) + unit = azure_page.get("unit", "inch") + dimensions = self._convert_dimensions( + width=width, height=height, unit=unit + ) + + # Build OCR page + ocr_page = OCRPage( + index=index, markdown=markdown, dimensions=dimensions + ) + mistral_pages.append(ocr_page) + + # Build usage info + usage_info = OCRUsageInfo( + pages_processed=len(mistral_pages), doc_size_bytes=None + ) + + # Return Mistral OCR response + return OCRResponse( + pages=mistral_pages, + model=model, + usage_info=usage_info, + object="ocr", + ) + + except Exception as e: + verbose_logger.error( + f"Error parsing Azure Document Intelligence response: {e}" + ) + raise e + + async def async_transform_ocr_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: Any, + **kwargs, + ) -> OCRResponse: + """ + Async transform Azure Document Intelligence response to Mistral OCR format. + + Handles async operation polling: If response is 202 Accepted, polls Operation-Location + until analysis completes using async polling. + + Args: + model: Model name + raw_response: Raw HTTP response from Azure DI (may be 202 Accepted) + logging_obj: Logging object + + Returns: + OCRResponse in Mistral format + """ + try: + # Check if we got 202 Accepted (async operation started) + if raw_response.status_code == 202: + verbose_logger.debug( + "Azure DI returned 202 Accepted, polling operation (async)..." + ) + + # Get Operation-Location header + operation_url = raw_response.headers.get("Operation-Location") + if not operation_url: + raise ValueError( + "Azure Document Intelligence returned 202 but no Operation-Location header found" + ) + + # Get headers for polling (need auth) + poll_headers = { + "Ocp-Apim-Subscription-Key": raw_response.request.headers.get( + "Ocp-Apim-Subscription-Key", "" + ) + } + + # Get timeout from kwargs or use default + timeout_secs = AZURE_OPERATION_POLLING_TIMEOUT + + # Poll until operation completes (async) + raw_response = await self._poll_operation_async( + operation_url=operation_url, + headers=poll_headers, + timeout_secs=timeout_secs, + ) + + # Now parse the completed response + response_json = raw_response.json() + + verbose_logger.debug( + f"Azure Document Intelligence response status: {response_json.get('status')}" + ) + + # Check if request succeeded + status = response_json.get("status") + if status != "succeeded": + raise ValueError( + f"Azure Document Intelligence analysis failed with status: {status}" + ) + + # Extract analyze result + analyze_result = response_json.get("analyzeResult", {}) + azure_pages = analyze_result.get("pages", []) + + # Transform pages to Mistral format + mistral_pages = [] + for azure_page in azure_pages: + page_number = azure_page.get("pageNumber", 1) + index = page_number - 1 # Convert to 0-based index + + # Extract markdown text + markdown = self._extract_page_markdown(azure_page) + + # Convert dimensions + width = azure_page.get("width", 8.5) + height = azure_page.get("height", 11) + unit = azure_page.get("unit", "inch") + dimensions = self._convert_dimensions( + width=width, height=height, unit=unit + ) + + # Build OCR page + ocr_page = OCRPage( + index=index, markdown=markdown, dimensions=dimensions + ) + mistral_pages.append(ocr_page) + + # Build usage info + usage_info = OCRUsageInfo( + pages_processed=len(mistral_pages), doc_size_bytes=None + ) + + # Return Mistral OCR response + return OCRResponse( + pages=mistral_pages, + model=model, + usage_info=usage_info, + object="ocr", + ) + + except Exception as e: + verbose_logger.error( + f"Error parsing Azure Document Intelligence response (async): {e}" + ) + raise e + diff --git a/litellm/llms/azure_ai/ocr/transformation.py b/litellm/llms/azure_ai/ocr/transformation.py new file mode 100644 index 00000000000..24fc9e86134 --- /dev/null +++ b/litellm/llms/azure_ai/ocr/transformation.py @@ -0,0 +1,270 @@ +""" +Azure AI OCR transformation implementation. +""" +from typing import Dict, Optional + +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.prompt_templates.image_handling import ( + async_convert_url_to_base64, + convert_url_to_base64, +) +from litellm.llms.base_llm.ocr.transformation import DocumentType, OCRRequestData +from litellm.llms.mistral.ocr.transformation import MistralOCRConfig +from litellm.secret_managers.main import get_secret_str + + +class AzureAIOCRConfig(MistralOCRConfig): + """ + Azure AI OCR transformation configuration. + + Azure AI uses Mistral's OCR API but with a different endpoint format. + Inherits transformation logic from MistralOCRConfig since they use the same format. + + Reference: Azure AI Foundry OCR documentation + + Important: Azure AI only supports base64 data URIs (data:image/..., data:application/pdf;base64,...). + Regular URLs are not supported. + """ + + def __init__(self) -> None: + super().__init__() + + def validate_environment( + self, + headers: Dict, + model: str, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + litellm_params: Optional[dict] = None, + **kwargs, + ) -> Dict: + """ + Validate environment and return headers for Azure AI OCR. + + Azure AI uses Bearer token authentication with AZURE_AI_API_KEY. + """ + # Get API key from environment if not provided + if api_key is None: + api_key = get_secret_str("AZURE_AI_API_KEY") + + if api_key is None: + raise ValueError( + "Missing Azure AI API Key - A call is being made to Azure AI but no key is set either in the environment variables or via params" + ) + + # Validate API base is provided + if api_base is None: + api_base = get_secret_str("AZURE_AI_API_BASE") + + if api_base is None: + raise ValueError( + "Missing Azure AI API Base - Set AZURE_AI_API_BASE environment variable or pass api_base parameter" + ) + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + **headers, + } + + return headers + + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: dict, + litellm_params: Optional[dict] = None, + **kwargs, + ) -> str: + """ + Get complete URL for Azure AI OCR endpoint. + + Azure AI endpoint format: https:///providers/mistral/azure/ocr + + Args: + api_base: Azure AI API base URL + model: Model name (not used in URL construction) + optional_params: Optional parameters + + Returns: Complete URL for Azure AI OCR endpoint + """ + if api_base is None: + raise ValueError( + "Missing Azure AI API Base - Set AZURE_AI_API_BASE environment variable or pass api_base parameter" + ) + + # Ensure no trailing slash + api_base = api_base.rstrip("/") + + # Azure AI OCR endpoint format + return f"{api_base}/providers/mistral/azure/ocr" + + def _convert_url_to_data_uri_sync(self, url: str) -> str: + """ + Synchronously convert a URL to a base64 data URI. + + Azure AI OCR doesn't have internet access, so we need to fetch URLs + and convert them to base64 data URIs. + + Args: + url: The URL to convert + + Returns: + Base64 data URI string + """ + verbose_logger.debug(f"Azure AI OCR: Converting URL to base64 data URI (sync): {url}") + + # Fetch and convert to base64 data URI + # convert_url_to_base64 already returns a full data URI like "data:image/jpeg;base64,..." + data_uri = convert_url_to_base64(url=url) + + verbose_logger.debug(f"Azure AI OCR: Converted URL to data URI (length: {len(data_uri)})") + + return data_uri + + async def _convert_url_to_data_uri_async(self, url: str) -> str: + """ + Asynchronously convert a URL to a base64 data URI. + + Azure AI OCR doesn't have internet access, so we need to fetch URLs + and convert them to base64 data URIs. + + Args: + url: The URL to convert + + Returns: + Base64 data URI string + """ + verbose_logger.debug(f"Azure AI OCR: Converting URL to base64 data URI (async): {url}") + + # Fetch and convert to base64 data URI asynchronously + # async_convert_url_to_base64 already returns a full data URI like "data:image/jpeg;base64,..." + data_uri = await async_convert_url_to_base64(url=url) + + verbose_logger.debug(f"Azure AI OCR: Converted URL to data URI (length: {len(data_uri)})") + + return data_uri + + def transform_ocr_request( + self, + model: str, + document: DocumentType, + optional_params: dict, + headers: dict, + **kwargs, + ) -> OCRRequestData: + """ + Transform OCR request for Azure AI, converting URLs to base64 data URIs (sync). + + Azure AI OCR doesn't have internet access, so we automatically fetch + any URLs and convert them to base64 data URIs synchronously. + + Args: + model: Model name + document: Document dict from user + optional_params: Already mapped optional parameters + headers: Request headers + **kwargs: Additional arguments + + Returns: + OCRRequestData with JSON data + """ + verbose_logger.debug(f"Azure AI OCR transform_ocr_request (sync) - model: {model}") + + if not isinstance(document, dict): + raise ValueError(f"Expected document dict, got {type(document)}") + + # Check if we need to convert URL to base64 + doc_type = document.get("type") + transformed_document = document.copy() + + if doc_type == "document_url": + document_url = document.get("document_url", "") + # If it's not already a data URI, convert it + if document_url and not document_url.startswith("data:"): + verbose_logger.debug( + "Azure AI OCR: Converting document URL to base64 data URI (sync)" + ) + data_uri = self._convert_url_to_data_uri_sync(url=document_url) + transformed_document["document_url"] = data_uri + elif doc_type == "image_url": + image_url = document.get("image_url", "") + # If it's not already a data URI, convert it + if image_url and not image_url.startswith("data:"): + verbose_logger.debug( + "Azure AI OCR: Converting image URL to base64 data URI (sync)" + ) + data_uri = self._convert_url_to_data_uri_sync(url=image_url) + transformed_document["image_url"] = data_uri + + # Call parent's transform to build the request + return super().transform_ocr_request( + model=model, + document=transformed_document, + optional_params=optional_params, + headers=headers, + **kwargs, + ) + + async def async_transform_ocr_request( + self, + model: str, + document: DocumentType, + optional_params: dict, + headers: dict, + **kwargs, + ) -> OCRRequestData: + """ + Transform OCR request for Azure AI, converting URLs to base64 data URIs (async). + + Azure AI OCR doesn't have internet access, so we automatically fetch + any URLs and convert them to base64 data URIs asynchronously. + + Args: + model: Model name + document: Document dict from user + optional_params: Already mapped optional parameters + headers: Request headers + **kwargs: Additional arguments + + Returns: + OCRRequestData with JSON data + """ + verbose_logger.debug(f"Azure AI OCR async_transform_ocr_request - model: {model}") + + if not isinstance(document, dict): + raise ValueError(f"Expected document dict, got {type(document)}") + + # Check if we need to convert URL to base64 + doc_type = document.get("type") + transformed_document = document.copy() + + if doc_type == "document_url": + document_url = document.get("document_url", "") + # If it's not already a data URI, convert it + if document_url and not document_url.startswith("data:"): + verbose_logger.debug( + "Azure AI OCR: Converting document URL to base64 data URI (async)" + ) + data_uri = await self._convert_url_to_data_uri_async(url=document_url) + transformed_document["document_url"] = data_uri + elif doc_type == "image_url": + image_url = document.get("image_url", "") + # If it's not already a data URI, convert it + if image_url and not image_url.startswith("data:"): + verbose_logger.debug( + "Azure AI OCR: Converting image URL to base64 data URI (async)" + ) + data_uri = await self._convert_url_to_data_uri_async(url=image_url) + transformed_document["image_url"] = data_uri + + # Call parent's transform to build the request + return super().transform_ocr_request( + model=model, + document=transformed_document, + optional_params=optional_params, + headers=headers, + **kwargs, + ) + diff --git a/litellm/llms/azure_ai/rerank/transformation.py b/litellm/llms/azure_ai/rerank/transformation.py index 4465e0d70a2..a47b6082c37 100644 --- a/litellm/llms/azure_ai/rerank/transformation.py +++ b/litellm/llms/azure_ai/rerank/transformation.py @@ -18,7 +18,12 @@ class AzureAIRerankConfig(CohereRerankConfig): Azure AI Rerank - Follows the same Spec as Cohere Rerank """ - def get_complete_url(self, api_base: Optional[str], model: str) -> str: + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: Optional[dict] = None, + ) -> str: if api_base is None: raise ValueError( "Azure AI API Base is required. api_base=None. Set in call or via `AZURE_AI_API_BASE` env var." @@ -32,6 +37,7 @@ class AzureAIRerankConfig(CohereRerankConfig): headers: dict, model: str, api_key: Optional[str] = None, + optional_params: Optional[dict] = None, ) -> dict: if api_key is None: api_key = get_secret_str("AZURE_AI_API_KEY") or litellm.azure_key diff --git a/litellm/llms/azure_ai/vector_stores/__init__.py b/litellm/llms/azure_ai/vector_stores/__init__.py new file mode 100644 index 00000000000..74ffe1afb17 --- /dev/null +++ b/litellm/llms/azure_ai/vector_stores/__init__.py @@ -0,0 +1,4 @@ +from litellm.llms.azure_ai.vector_stores.transformation import AzureAIVectorStoreConfig + +__all__ = ["AzureAIVectorStoreConfig"] + diff --git a/litellm/llms/azure_ai/vector_stores/transformation.py b/litellm/llms/azure_ai/vector_stores/transformation.py new file mode 100644 index 00000000000..96cea064ce1 --- /dev/null +++ b/litellm/llms/azure_ai/vector_stores/transformation.py @@ -0,0 +1,258 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +import httpx + +import litellm +from litellm.llms.azure.common_utils import BaseAzureLLM +from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig +from litellm.types.router import GenericLiteLLMParams +from litellm.types.vector_stores import ( + BaseVectorStoreAuthCredentials, + VectorStoreCreateOptionalRequestParams, + VectorStoreCreateResponse, + VectorStoreIndexEndpoints, + VectorStoreResultContent, + VectorStoreSearchOptionalRequestParams, + VectorStoreSearchResponse, + VectorStoreSearchResult, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class AzureAIVectorStoreConfig(BaseVectorStoreConfig, BaseAzureLLM): + """ + Configuration for Azure AI Search Vector Store + + This implementation uses the Azure AI Search API for vector store operations. + Supports vector search with embeddings generated via litellm.embeddings. + """ + + def __init__(self): + super().__init__() + + def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints: + return { + "read": [("GET", "/docs/search"), ("POST", "/docs/search")], + "write": [("PUT", "/docs")], + } + + def get_auth_credentials( + self, litellm_params: dict + ) -> BaseVectorStoreAuthCredentials: + api_key = litellm_params.get("api_key") + if api_key is None: + raise ValueError("api_key is required") + + return { + "headers": { + "api-key": api_key, + } + } + + def validate_environment( + self, headers: dict, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + + basic_headers = self._base_validate_azure_environment(headers, litellm_params) + basic_headers.update({"Content-Type": "application/json"}) + return basic_headers + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the base endpoint for Azure AI Search API + + Expected format: https://{search_service_name}.search.windows.net + """ + if api_base: + return api_base.rstrip("/") + + # Get search service name from litellm_params + search_service_name = litellm_params.get("azure_search_service_name") + + if not search_service_name: + raise ValueError( + "Azure AI Search service name is required. " + "Provide it via litellm_params['azure_search_service_name'] or api_base parameter" + ) + + # Azure AI Search endpoint + return f"https://{search_service_name}.search.windows.net" + + def transform_search_vector_store_request( + self, + vector_store_id: str, + query: Union[str, List[str]], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: LiteLLMLoggingObj, + litellm_params: dict, + ) -> Tuple[str, Dict[str, Any]]: + """ + Transform search request for Azure AI Search API + + Generates embeddings using litellm.embeddings and constructs Azure AI Search request + """ + # Convert query to string if it's a list + if isinstance(query, list): + query = " ".join(query) + + # Get embedding model from litellm_params (required) + embedding_model = litellm_params.get("litellm_embedding_model") + if not embedding_model: + raise ValueError( + "embedding_model is required in litellm_params for Azure AI Search. " + "Example: litellm_params['embedding_model'] = 'azure/text-embedding-3-large'" + ) + + embedding_config = litellm_params.get("litellm_embedding_config", {}) + if not embedding_config: + raise ValueError( + "embedding_config is required in litellm_params for Azure AI Search. " + "Example: litellm_params['embedding_config'] = {'api_base': 'https://krris-mh44uf7y-eastus2.cognitiveservices.azure.com/', 'api_key': 'os.environ/AZURE_API_KEY', 'api_version': '2025-09-01'}" + ) + + # Get vector field name (defaults to contentVector) + vector_field = litellm_params.get("azure_search_vector_field", "contentVector") + + # Get top_k (number of results to return) + top_k = vector_store_search_optional_params.get("top_k", 10) + + # Generate embedding for the query using litellm.embeddings + try: + embedding_response = litellm.embedding( + model=embedding_model, + input=[query], + **embedding_config, + ) + query_vector = embedding_response.data[0]["embedding"] + except Exception as e: + raise Exception(f"Failed to generate embedding for query: {str(e)}") + + # Azure AI Search endpoint for search + index_name = vector_store_id # vector_store_id is the index name + url = f"{api_base}/indexes/{index_name}/docs/search?api-version=2024-07-01" + + # Build the request body for Azure AI Search with vector search + request_body = { + "search": "*", # Get all documents (filtered by vector similarity) + "vectorQueries": [ + { + "vector": query_vector, + "fields": vector_field, + "kind": "vector", + "k": top_k, # Number of nearest neighbors to return + } + ], + "select": "id,content", # Fields to return (customize based on schema) + "top": top_k, + } + + ######################################################### + # Update logging object with details of the request + ######################################################### + litellm_logging_obj.model_call_details["input"] = query + litellm_logging_obj.model_call_details["embedding_model"] = embedding_model + litellm_logging_obj.model_call_details["top_k"] = top_k + + return url, request_body + + def transform_search_vector_store_response( + self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj + ) -> VectorStoreSearchResponse: + """ + Transform Azure AI Search API response to standard vector store search response + + Handles the format from Azure AI Search which returns: + { + "value": [ + { + "id": "...", + "content": "...", + "@search.score": 0.95, + ... (other fields) + } + ] + } + """ + try: + response_json = response.json() + + # Extract results from Azure AI Search API response + results = response_json.get("value", []) + + # Transform results to standard format + search_results: List[VectorStoreSearchResult] = [] + for result in results: + # Extract document ID + document_id = result.get("id", "") + + # Extract text content + text_content = result.get("content", "") + + content = [ + VectorStoreResultContent( + text=text_content, + type="text", + ) + ] + + # Get the search score (relevance score from Azure AI Search) + score = result.get("@search.score", 0.0) + + # Use document ID as both file_id and filename + file_id = document_id + filename = f"Document {document_id}" + + # Build attributes with all available metadata + # Exclude system fields and already-processed fields + attributes = {} + for key, value in result.items(): + if key not in ["id", "content", "contentVector", "@search.score"]: + attributes[key] = value + + # Always include document_id in attributes + attributes["document_id"] = document_id + + result_obj = VectorStoreSearchResult( + score=score, + content=content, + file_id=file_id, + filename=filename, + attributes=attributes, + ) + search_results.append(result_obj) + + return VectorStoreSearchResponse( + object="vector_store.search_results.page", + search_query=litellm_logging_obj.model_call_details.get("input", ""), + data=search_results, + ) + + except Exception as e: + raise self.get_error_class( + error_message=str(e), + status_code=response.status_code, + headers=response.headers, + ) + + def transform_create_vector_store_request( + self, + vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams, + api_base: str, + ) -> Tuple[str, Dict]: + raise NotImplementedError + + def transform_create_vector_store_response( + self, response: httpx.Response + ) -> VectorStoreCreateResponse: + raise NotImplementedError diff --git a/litellm/llms/base_llm/base_model_iterator.py b/litellm/llms/base_llm/base_model_iterator.py index 347301e7b37..6953b1c5878 100644 --- a/litellm/llms/base_llm/base_model_iterator.py +++ b/litellm/llms/base_llm/base_model_iterator.py @@ -13,6 +13,50 @@ from litellm.types.utils import ( ) +def convert_model_response_to_streaming( + model_response: ModelResponse, +) -> ModelResponseStream: + """ + Convert a ModelResponse to ModelResponseStream. + + This function transforms a standard completion response into a streaming chunk format + by converting 'message' fields to 'delta' fields. + + Args: + model_response: The ModelResponse to convert + + Returns: + ModelResponseStream: A streaming chunk version of the response + + Raises: + ValueError: If the conversion fails + """ + try: + streaming_choices: List[StreamingChoices] = [] + for choice in model_response.choices: + streaming_choices.append( + StreamingChoices( + index=choice.index, + delta=Delta( + **cast(Choices, choice).message.model_dump(), + ), + finish_reason=choice.finish_reason, + ) + ) + processed_chunk = ModelResponseStream( + id=model_response.id, + object="chat.completion.chunk", + created=model_response.created, + model=model_response.model, + choices=streaming_choices, + ) + return processed_chunk + except Exception as e: + raise ValueError( + f"Failed to convert ModelResponse to ModelResponseStream: {model_response}. Error: {e}" + ) + + class BaseModelResponseIterator: def __init__( self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False @@ -147,28 +191,7 @@ class MockResponseIterator: # for returning ai21 streaming responses return self def _chunk_parser(self, chunk_data: ModelResponse) -> ModelResponseStream: - try: - streaming_choices: List[StreamingChoices] = [] - for choice in chunk_data.choices: - streaming_choices.append( - StreamingChoices( - index=choice.index, - delta=Delta( - **cast(Choices, choice).message.model_dump(), - ), - finish_reason=choice.finish_reason, - ) - ) - processed_chunk = ModelResponseStream( - id=chunk_data.id, - object="chat.completion", - created=chunk_data.created, - model=chunk_data.model, - choices=streaming_choices, - ) - return processed_chunk - except Exception as e: - raise ValueError(f"Failed to decode chunk: {chunk_data}. Error: {e}") + return convert_model_response_to_streaming(chunk_data) def __next__(self): if self.is_done: diff --git a/litellm/llms/base_llm/containers/transformation.py b/litellm/llms/base_llm/containers/transformation.py new file mode 100644 index 00000000000..429f5a76e2e --- /dev/null +++ b/litellm/llms/base_llm/containers/transformation.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import types +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + +import httpx + +from litellm.types.containers.main import ContainerCreateOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.types.containers.main import ( + ContainerListResponse as _ContainerListResponse, + ) + from litellm.types.containers.main import ( + ContainerObject as _ContainerObject, + ) + from litellm.types.containers.main import ( + DeleteContainerResult as _DeleteContainerResult, + ) + + from ..chat.transformation import BaseLLMException as _BaseLLMException + + LiteLLMLoggingObj = _LiteLLMLoggingObj + BaseLLMException = _BaseLLMException + ContainerObject = _ContainerObject + DeleteContainerResult = _DeleteContainerResult + ContainerListResponse = _ContainerListResponse +else: + LiteLLMLoggingObj = Any + BaseLLMException = Any + ContainerObject = Any + DeleteContainerResult = Any + ContainerListResponse = Any + + +class BaseContainerConfig(ABC): + def __init__(self): + pass + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not k.startswith("_abc") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + @abstractmethod + def get_supported_openai_params(self) -> list: + pass + + @abstractmethod + def map_openai_params( + self, + container_create_optional_params: ContainerCreateOptionalRequestParams, + drop_params: bool, + ) -> dict: + pass + + @abstractmethod + def validate_environment( + self, + headers: dict, + api_key: str | None = None, + ) -> dict: + return {} + + @abstractmethod + def get_complete_url( + self, + api_base: str | None, + litellm_params: dict, + ) -> str: + """Get the complete url for the request. + + OPTIONAL - Some providers need `model` in `api_base`. + """ + if api_base is None: + msg = "api_base is required" + raise ValueError(msg) + return api_base + + @abstractmethod + def transform_container_create_request( + self, + name: str, + container_create_optional_request_params: dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> dict: + """Transform the container creation request. + + Returns: + dict: Request data for container creation. + """ + ... + + @abstractmethod + def transform_container_create_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ContainerObject: + """Transform the container creation response.""" + ... + + @abstractmethod + def transform_container_list_request( + self, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + after: str | None = None, + limit: int | None = None, + order: str | None = None, + extra_query: dict[str, Any] | None = None, + ) -> tuple[str, dict]: + """Transform the container list request into a URL and params. + + Returns: + tuple[str, dict]: (url, params) for the container list request. + """ + ... + + @abstractmethod + def transform_container_list_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ContainerListResponse: + """Transform the container list response.""" + ... + + @abstractmethod + def transform_container_retrieve_request( + self, + container_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> tuple[str, dict]: + """Transform the container retrieve request into a URL and data/params. + + Returns: + tuple[str, dict]: (url, params) for the container retrieve request. + """ + ... + + @abstractmethod + def transform_container_retrieve_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ContainerObject: + """Transform the container retrieve response.""" + ... + + @abstractmethod + def transform_container_delete_request( + self, + container_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> tuple[str, dict]: + """Transform the container delete request into a URL and data. + + Returns: + tuple[str, dict]: (url, data) for the container delete request. + """ + ... + + @abstractmethod + def transform_container_delete_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> DeleteContainerResult: + """Transform the container delete response.""" + ... + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict | httpx.Headers, + ) -> BaseLLMException: + from ..chat.transformation import BaseLLMException + + raise BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + ) + diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py new file mode 100644 index 00000000000..4599af1b745 --- /dev/null +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -0,0 +1,23 @@ +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from litellm.integrations.custom_guardrail import CustomGuardrail + + +class BaseTranslation(ABC): + @abstractmethod + async def process_input_messages( + self, + data: dict, + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + pass + + @abstractmethod + async def process_output_response( + self, + response: Any, + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + pass diff --git a/litellm/llms/base_llm/ocr/__init__.py b/litellm/llms/base_llm/ocr/__init__.py new file mode 100644 index 00000000000..5965af5f2b7 --- /dev/null +++ b/litellm/llms/base_llm/ocr/__init__.py @@ -0,0 +1,22 @@ +"""Base OCR transformation module.""" +from .transformation import ( + BaseOCRConfig, + DocumentType, + OCRPage, + OCRPageDimensions, + OCRPageImage, + OCRRequestData, + OCRResponse, + OCRUsageInfo, +) + +__all__ = [ + "BaseOCRConfig", + "DocumentType", + "OCRResponse", + "OCRPage", + "OCRPageDimensions", + "OCRPageImage", + "OCRUsageInfo", + "OCRRequestData", +] diff --git a/litellm/llms/base_llm/ocr/transformation.py b/litellm/llms/base_llm/ocr/transformation.py new file mode 100644 index 00000000000..fb13332c464 --- /dev/null +++ b/litellm/llms/base_llm/ocr/transformation.py @@ -0,0 +1,243 @@ +""" +Base OCR transformation configuration. +""" +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +import httpx +from pydantic import PrivateAttr + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.llms.base import LiteLLMPydanticObjectBase + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +# DocumentType for OCR - Mistral format document dict +DocumentType = Dict[str, str] + + +class OCRPageDimensions(LiteLLMPydanticObjectBase): + """Page dimensions from OCR response.""" + dpi: Optional[int] = None + height: Optional[int] = None + width: Optional[int] = None + + +class OCRPageImage(LiteLLMPydanticObjectBase): + """Image extracted from OCR page.""" + image_base64: Optional[str] = None + bbox: Optional[Dict[str, Any]] = None + + model_config = {"extra": "allow"} + + +class OCRPage(LiteLLMPydanticObjectBase): + """Single page from OCR response.""" + index: int + markdown: str + images: Optional[List[OCRPageImage]] = None + dimensions: Optional[OCRPageDimensions] = None + + model_config = {"extra": "allow"} + + +class OCRUsageInfo(LiteLLMPydanticObjectBase): + """Usage information from OCR response.""" + pages_processed: Optional[int] = None + doc_size_bytes: Optional[int] = None + + model_config = {"extra": "allow"} + + +class OCRResponse(LiteLLMPydanticObjectBase): + """ + Standard OCR response format. + Standardized to Mistral OCR format - other providers should transform to this format. + """ + pages: List[OCRPage] + model: str + document_annotation: Optional[Any] = None + usage_info: Optional[OCRUsageInfo] = None + object: str = "ocr" + + model_config = {"extra": "allow"} + + # Define private attributes using PrivateAttr + _hidden_params: dict = PrivateAttr(default_factory=dict) + + +class OCRRequestData(LiteLLMPydanticObjectBase): + """OCR request data structure.""" + data: Optional[Union[Dict, bytes]] = None + files: Optional[Dict[str, Any]] = None + + +class BaseOCRConfig: + """ + Base configuration for OCR transformations. + Handles provider-agnostic OCR operations. + """ + + def __init__(self) -> None: + pass + + def get_supported_ocr_params(self, model: str) -> list: + """ + Get supported OCR parameters for this provider. + Override this method in provider-specific implementations. + """ + return [] + + def map_ocr_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + ) -> dict: + """Map OCR parameters to provider-specific parameters.""" + return optional_params + + def validate_environment( + self, + headers: Dict, + model: str, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + litellm_params: Optional[dict] = None, + **kwargs, + ) -> Dict: + """ + Validate environment and return headers. + Override in provider-specific implementations. + """ + return headers + + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: dict, + litellm_params: Optional[dict] = None, + **kwargs, + ) -> str: + """ + Get complete URL for OCR endpoint. + Override in provider-specific implementations. + """ + raise NotImplementedError("get_complete_url must be implemented by provider") + + def transform_ocr_request( + self, + model: str, + document: DocumentType, + optional_params: dict, + headers: dict, + **kwargs, + ) -> OCRRequestData: + """ + Transform OCR request to provider-specific format. + Override in provider-specific implementations. + + Args: + model: Model name + document: Document to process (Mistral format dict, or file path, bytes, etc.) + optional_params: Optional parameters for the request + headers: Request headers + + Returns: + OCRRequestData with data and files fields + """ + raise NotImplementedError("transform_ocr_request must be implemented by provider") + + async def async_transform_ocr_request( + self, + model: str, + document: DocumentType, + optional_params: dict, + headers: dict, + **kwargs, + ) -> OCRRequestData: + """ + Async transform OCR request to provider-specific format. + Optional method - providers can override if they need async transformations + (e.g., Azure AI for URL-to-base64 conversion). + + Default implementation falls back to sync transform_ocr_request. + + Args: + model: Model name + document: Document to process (Mistral format dict, or file path, bytes, etc.) + optional_params: Optional parameters for the request + headers: Request headers + + Returns: + OCRRequestData with data and files fields + """ + # Default implementation: call sync version + return self.transform_ocr_request( + model=model, + document=document, + optional_params=optional_params, + headers=headers, + **kwargs, + ) + + def transform_ocr_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs, + ) -> OCRResponse: + """ + Transform provider-specific OCR response to standard format. + Override in provider-specific implementations. + """ + raise NotImplementedError("transform_ocr_response must be implemented by provider") + + async def async_transform_ocr_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs, + ) -> OCRResponse: + """ + Async transform provider-specific OCR response to standard format. + Optional method - providers can override if they need async transformations + (e.g., Azure Document Intelligence for async operation polling). + + Default implementation falls back to sync transform_ocr_response. + + Args: + model: Model name + raw_response: Raw HTTP response + logging_obj: Logging object + + Returns: + OCRResponse in standard format + """ + # Default implementation: call sync version + return self.transform_ocr_response( + model=model, + raw_response=raw_response, + logging_obj=logging_obj, + **kwargs, + ) + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict, + ) -> Exception: + """Get appropriate error class for the provider.""" + return BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + ) + diff --git a/litellm/llms/base_llm/rerank/transformation.py b/litellm/llms/base_llm/rerank/transformation.py index 6e9c03dee89..b22d85e82be 100644 --- a/litellm/llms/base_llm/rerank/transformation.py +++ b/litellm/llms/base_llm/rerank/transformation.py @@ -23,6 +23,7 @@ class BaseRerankConfig(ABC): headers: dict, model: str, api_key: Optional[str] = None, + optional_params: Optional[dict] = None, ) -> dict: pass @@ -50,7 +51,12 @@ class BaseRerankConfig(ABC): return model_response @abstractmethod - def get_complete_url(self, api_base: Optional[str], model: str) -> str: + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: Optional[dict] = None, + ) -> str: """ OPTIONAL diff --git a/litellm/llms/base_llm/search/__init__.py b/litellm/llms/base_llm/search/__init__.py new file mode 100644 index 00000000000..5a46482ed43 --- /dev/null +++ b/litellm/llms/base_llm/search/__init__.py @@ -0,0 +1,15 @@ +""" +Base Search API module. +""" +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) + +__all__ = [ + "BaseSearchConfig", + "SearchResponse", + "SearchResult", +] + diff --git a/litellm/llms/base_llm/search/transformation.py b/litellm/llms/base_llm/search/transformation.py new file mode 100644 index 00000000000..14941911f17 --- /dev/null +++ b/litellm/llms/base_llm/search/transformation.py @@ -0,0 +1,169 @@ +""" +Base Search transformation configuration. +""" +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union + +import httpx +from pydantic import PrivateAttr + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.llms.base import LiteLLMPydanticObjectBase + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class SearchResult(LiteLLMPydanticObjectBase): + """Single search result.""" + title: str + url: str + snippet: str + date: Optional[str] = None + last_updated: Optional[str] = None + + model_config = {"extra": "allow"} + + +class SearchResponse(LiteLLMPydanticObjectBase): + """ + Standard Search response format. + Standardized to Perplexity Search format - other providers should transform to this format. + """ + results: List[SearchResult] + object: str = "search" + + model_config = {"extra": "allow"} + + # Define private attributes using PrivateAttr + _hidden_params: dict = PrivateAttr(default_factory=dict) + + +class BaseSearchConfig: + """ + Base configuration for Search transformations. + Handles provider-agnostic Search operations. + """ + + def __init__(self) -> None: + pass + + @staticmethod + def ui_friendly_name() -> str: + """ + UI-friendly name for the search provider. + Override in provider-specific implementations. + """ + return "Unknown Search Provider" + + def get_http_method(self) -> Literal["GET", "POST"]: + """ + Get HTTP method for search requests. + Override in provider-specific implementations if needed. + + Returns: + HTTP method ('GET' or 'POST'). Default is 'POST'. + """ + return "POST" + + @staticmethod + def get_supported_perplexity_optional_params() -> set: + """ + Get the set of Perplexity unified search parameters. + These are the standard parameters that providers should transform from. + + Returns: + Set of parameter names that are part of the unified spec + """ + return { + "max_results", + "search_domain_filter", + "country", + "max_tokens_per_page", + } + + def validate_environment( + self, + headers: Dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + **kwargs, + ) -> Dict: + """ + Validate environment and return headers. + Override in provider-specific implementations. + """ + return headers + + def get_complete_url( + self, + api_base: Optional[str], + optional_params: dict, + data: Optional[Union[Dict, List[Dict]]] = None, + **kwargs, + ) -> str: + """ + Get complete URL for Search endpoint. + + Args: + api_base: Base URL for the API + optional_params: Optional parameters for the request + data: Transformed request body from transform_search_request(). + Some providers (e.g., Google PSE) use GET requests and need + the request body to construct query parameters in the URL. + Can be a dict or list of dicts depending on provider. + **kwargs: Additional keyword arguments + + Returns: + Complete URL for the search endpoint + + Note: + Override in provider-specific implementations. + """ + raise NotImplementedError("get_complete_url must be implemented by provider") + + def transform_search_request( + self, + query: Union[str, List[str]], + optional_params: dict, + **kwargs, + ) -> Union[Dict, List[Dict]]: + """ + Transform Search request to provider-specific format. + Override in provider-specific implementations. + + Args: + query: Search query (string or list of strings) + optional_params: Optional parameters for the request + + Returns: + Dict with request data + """ + raise NotImplementedError("transform_search_request must be implemented by provider") + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs, + ) -> SearchResponse: + """ + Transform provider-specific Search response to standard format. + Override in provider-specific implementations. + """ + raise NotImplementedError("transform_search_response must be implemented by provider") + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict, + ) -> Exception: + """Get appropriate error class for the provider.""" + return BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + ) + diff --git a/litellm/llms/base_llm/text_to_speech/transformation.py b/litellm/llms/base_llm/text_to_speech/transformation.py new file mode 100644 index 00000000000..31f581cec0f --- /dev/null +++ b/litellm/llms/base_llm/text_to_speech/transformation.py @@ -0,0 +1,149 @@ +import types +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union + +import httpx + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.types.llms.openai import ( + HttpxBinaryResponseContent as _HttpxBinaryResponseContent, + ) + + from ..chat.transformation import BaseLLMException as _BaseLLMException + + LiteLLMLoggingObj = _LiteLLMLoggingObj + BaseLLMException = _BaseLLMException + HttpxBinaryResponseContent = _HttpxBinaryResponseContent +else: + LiteLLMLoggingObj = Any + BaseLLMException = Any + HttpxBinaryResponseContent = Any + + +class TextToSpeechRequestData(TypedDict, total=False): + """ + Structured return type for text-to-speech transformations. + + This ensures a consistent interface across all TTS providers. + Providers should set ONE of: dict_body, ssml_body, or text_body. + """ + dict_body: Dict[str, Any] # JSON request body (e.g., OpenAI TTS) + ssml_body: str # SSML/XML string body (e.g., Azure AVA TTS) + headers: Dict[str, str] # Provider-specific headers to merge with base headers + + +class BaseTextToSpeechConfig(ABC): + def __init__(self): + pass + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not k.startswith("_abc") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + @abstractmethod + def get_supported_openai_params(self, model: str) -> list: + """ + Get list of OpenAI TTS parameters supported by this provider + """ + pass + + @abstractmethod + def map_openai_params( + self, + model: str, + optional_params: Dict, + voice: Optional[Union[str, Dict]] = None, + drop_params: bool = False, + kwargs: Dict = {}, + ) -> Tuple[Optional[str], Dict]: + """ + Map OpenAI TTS parameters to provider-specific parameters + """ + pass + + @abstractmethod + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate environment and return headers + """ + return {} + + @abstractmethod + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the complete url for the request + """ + if api_base is None: + raise ValueError("api_base is required") + return api_base + + @abstractmethod + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: Optional[str], + optional_params: Dict, + litellm_params: Dict, + headers: dict, + ) -> TextToSpeechRequestData: + """ + Transform request to provider-specific format. + + Returns: + TextToSpeechRequestData: A structured dict containing: + - body: The request body (JSON dict, XML string, or binary data) + - headers: Provider-specific headers to merge with base headers + """ + pass + + @abstractmethod + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> "HttpxBinaryResponseContent": + """ + Transform provider response to standard format + """ + pass + + def get_error_class( + self, error_message: str, status_code: int, headers: Dict + ) -> BaseLLMException: + from ..chat.transformation import BaseLLMException + + raise BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + ) + diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index b50fd957587..89f2094d5df 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -5,8 +5,11 @@ import httpx from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_stores import ( + BaseVectorStoreAuthCredentials, + VECTOR_STORE_OPENAI_PARAMS, VectorStoreCreateOptionalRequestParams, VectorStoreCreateResponse, + VectorStoreIndexEndpoints, VectorStoreSearchOptionalRequestParams, VectorStoreSearchResponse, ) @@ -22,7 +25,32 @@ else: LiteLLMLoggingObj = Any BaseLLMException = Any + class BaseVectorStoreConfig: + + def get_supported_openai_params( + self, model: str + ) -> List[VECTOR_STORE_OPENAI_PARAMS]: + return [] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + drop_params: bool, + ) -> dict: + return optional_params + + @abstractmethod + def get_auth_credentials( + self, litellm_params: dict + ) -> BaseVectorStoreAuthCredentials: + pass + + @abstractmethod + def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints: + pass + @abstractmethod def transform_search_vector_store_request( self, @@ -33,10 +61,13 @@ class BaseVectorStoreConfig: litellm_logging_obj: LiteLLMLoggingObj, litellm_params: dict, ) -> Tuple[str, Dict]: + pass @abstractmethod - def transform_search_vector_store_response(self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj) -> VectorStoreSearchResponse: + def transform_search_vector_store_response( + self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj + ) -> VectorStoreSearchResponse: pass @abstractmethod @@ -48,7 +79,9 @@ class BaseVectorStoreConfig: pass @abstractmethod - def transform_create_vector_store_response(self, response: httpx.Response) -> VectorStoreCreateResponse: + def transform_create_vector_store_response( + self, response: httpx.Response + ) -> VectorStoreCreateResponse: pass @abstractmethod @@ -73,7 +106,6 @@ class BaseVectorStoreConfig: if api_base is None: raise ValueError("api_base is required") return api_base - def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] @@ -102,3 +134,8 @@ class BaseVectorStoreConfig: """ return headers, None + def calculate_vector_store_cost( + self, + response: VectorStoreSearchResponse, + ) -> Tuple[float, float]: + return 0.0, 0.0 diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py new file mode 100644 index 00000000000..7e990b42650 --- /dev/null +++ b/litellm/llms/base_llm/videos/transformation.py @@ -0,0 +1,275 @@ +import types +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union + +import httpx +from httpx._types import RequestFiles + +from litellm.types.responses.main import * +from litellm.types.router import GenericLiteLLMParams +from litellm.types.videos.main import VideoCreateOptionalRequestParams + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.types.videos.main import VideoObject as _VideoObject + + from ..chat.transformation import BaseLLMException as _BaseLLMException + + LiteLLMLoggingObj = _LiteLLMLoggingObj + BaseLLMException = _BaseLLMException + VideoObject = _VideoObject +else: + LiteLLMLoggingObj = Any + BaseLLMException = Any + VideoObject = Any + + +class BaseVideoConfig(ABC): + def __init__(self): + pass + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not k.startswith("_abc") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + @abstractmethod + def get_supported_openai_params(self, model: str) -> list: + pass + + @abstractmethod + def map_openai_params( + self, + video_create_optional_params: VideoCreateOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict: + pass + + @abstractmethod + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + return {} + + @abstractmethod + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + OPTIONAL + + Get the complete url for the request + + Some providers need `model` in `api_base` + """ + if api_base is None: + raise ValueError("api_base is required") + return api_base + + @abstractmethod + def transform_video_create_request( + self, + model: str, + prompt: str, + api_base: str, + video_create_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[Dict, RequestFiles, str]: + pass + + @abstractmethod + def transform_video_create_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict] = None, + ) -> VideoObject: + pass + + @abstractmethod + def transform_video_content_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform the video content request into a URL and data/params + + Returns: + Tuple[str, Dict]: (url, params) for the video content request + """ + pass + + @abstractmethod + def transform_video_content_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> bytes: + pass + + async def async_transform_video_content_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> bytes: + """ + Async transform video content download response to bytes. + Optional method - providers can override if they need async transformations + (e.g., RunwayML for downloading video from CloudFront URL). + + Default implementation falls back to sync transform_video_content_response. + + Args: + raw_response: Raw HTTP response + logging_obj: Logging object + + Returns: + Video content as bytes + """ + # Default implementation: call sync version + return self.transform_video_content_response( + raw_response=raw_response, + logging_obj=logging_obj, + ) + + @abstractmethod + def transform_video_remix_request( + self, + video_id: str, + prompt: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + extra_body: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + """ + Transform the video remix request into a URL and data + + Returns: + Tuple[str, Dict]: (url, data) for the video remix request + """ + pass + + @abstractmethod + def transform_video_remix_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + ) -> VideoObject: + pass + + @abstractmethod + def transform_video_list_request( + self, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + after: Optional[str] = None, + limit: Optional[int] = None, + order: Optional[str] = None, + extra_query: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + """ + Transform the video list request into a URL and params + + Returns: + Tuple[str, Dict]: (url, params) for the video list request + """ + pass + + @abstractmethod + def transform_video_list_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + ) -> Dict[str,str]: + pass + + @abstractmethod + def transform_video_delete_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform the video delete request into a URL and data + + Returns: + Tuple[str, Dict]: (url, data) for the video delete request + """ + pass + + @abstractmethod + def transform_video_delete_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> VideoObject: + pass + + @abstractmethod + def transform_video_status_retrieve_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform the video retrieve request into a URL and data/params + + Returns: + Tuple[str, Dict]: (url, params) for the video retrieve request + """ + pass + + @abstractmethod + def transform_video_status_retrieve_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + ) -> VideoObject: + pass + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + from ..chat.transformation import BaseLLMException + + raise BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + ) diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index 8211addaf95..72e270428ac 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -1,6 +1,7 @@ import hashlib import json import os +import urllib.parse from datetime import datetime from typing import ( TYPE_CHECKING, @@ -331,16 +332,61 @@ class BaseAWSLLM: return provider return None + @staticmethod + def get_bedrock_model_id( + optional_params: dict, + provider: Optional[BEDROCK_INVOKE_PROVIDERS_LITERAL], + model: str, + ) -> str: + model_id = optional_params.pop("model_id", None) + if model_id is not None: + model_id = BaseAWSLLM.encode_model_id(model_id=model_id) + else: + model_id = model + + model_id = model_id.replace("invoke/", "", 1) + if provider == "llama" and "llama/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="llama" + ) + elif provider == "deepseek_r1" and "deepseek_r1/" in model_id: + model_id = BaseAWSLLM._get_model_id_from_model_with_spec( + model_id, spec="deepseek_r1" + ) + return model_id + + @staticmethod + def _get_model_id_from_model_with_spec( + model: str, + spec: str, + ) -> str: + """ + Remove `llama` from modelID since `llama` is simply a spec to follow for custom bedrock models + """ + model_id = model.replace(spec + "/", "") + return BaseAWSLLM.encode_model_id(model_id=model_id) + + @staticmethod + def encode_model_id(model_id: str) -> str: + """ + Double encode the model ID to ensure it matches the expected double-encoded format. + Args: + model_id (str): The model ID to encode. + Returns: + str: The double-encoded model ID. + """ + return urllib.parse.quote(model_id, safe="") + @staticmethod def get_bedrock_embedding_provider( model: str, ) -> Optional[BEDROCK_EMBEDDING_PROVIDERS_LITERAL]: """ Helper function to get the bedrock embedding provider from the model - + Handles scenarios like: 1. model=cohere.embed-english-v3:0 -> Returns `cohere` - 2. model=amazon.titan-embed-text-v1 -> Returns `amazon` + 2. model=amazon.titan-embed-text-v1 -> Returns `amazon` 3. model=us.twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs` 4. model=twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs` """ @@ -349,20 +395,24 @@ class BaseAWSLLM: parts = model.split(".") # Check if the second part (after potential region) is a known provider if len(parts) >= 2: - potential_provider = parts[1] # e.g., "twelvelabs" from "us.twelvelabs.marengo-embed-2-7-v1:0" + potential_provider = parts[ + 1 + ] # e.g., "twelvelabs" from "us.twelvelabs.marengo-embed-2-7-v1:0" if potential_provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL): return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, potential_provider) - + # Check if the first part is a known provider (standard format) - potential_provider = parts[0] # e.g., "cohere" from "cohere.embed-english-v3:0" + potential_provider = parts[ + 0 + ] # e.g., "cohere" from "cohere.embed-english-v3:0" if potential_provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL): return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, potential_provider) - + # Fallback: check if any provider name appears in the model string for provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL): if provider in model: return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, provider) - + return None def _get_aws_region_name( @@ -851,7 +901,7 @@ class BaseAWSLLM: api_base: Optional[str], aws_bedrock_runtime_endpoint: Optional[str], aws_region_name: str, - endpoint_type: Optional[Literal["runtime", "agent"]] = "runtime", + endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]] = "runtime", ) -> Tuple[str, str]: env_aws_bedrock_runtime_endpoint = get_secret("AWS_BEDROCK_RUNTIME_ENDPOINT") if api_base is not None: @@ -885,7 +935,7 @@ class BaseAWSLLM: return endpoint_url, proxy_endpoint_url def _select_default_endpoint_url( - self, endpoint_type: Optional[Literal["runtime", "agent"]], aws_region_name: str + self, endpoint_type: Optional[Literal["runtime", "agent", "agentcore"]], aws_region_name: str ) -> str: """ Select the default endpoint url based on the endpoint type @@ -894,6 +944,8 @@ class BaseAWSLLM: """ if endpoint_type == "agent": return f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com" + elif endpoint_type == "agentcore": + return f"https://bedrock-agentcore.{aws_region_name}.amazonaws.com" else: return f"https://bedrock-runtime.{aws_region_name}.amazonaws.com" @@ -984,11 +1036,23 @@ class BaseAWSLLM: raise ImportError( "Missing boto3 to call bedrock. Run 'pip install boto3'." ) + + # Filter headers for AWS signature calculation + # AWS SigV4 only includes specific headers in signature calculation + aws_signature_headers = self._filter_headers_for_aws_signature(headers) sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name) request = AWSRequest( - method="POST", url=endpoint_url, data=data, headers=headers + method="POST", + url=endpoint_url, + data=data, + headers=aws_signature_headers, ) sigv4.add_auth(request) + + # Add back all original headers (including forwarded ones) after signature calculation + for header_name, header_value in headers.items(): + request.headers[header_name] = header_value + if ( extra_headers is not None and "Authorization" in extra_headers ): # prevent sigv4 from overwriting the auth header @@ -997,9 +1061,39 @@ class BaseAWSLLM: return prepped + def _filter_headers_for_aws_signature(self, headers: dict) -> dict: + """ + Filter headers to only include those that AWS SigV4 includes in signature calculation. + This Fixes forwarded client headers from breaking the signature calculation. + """ + aws_signature_headers = {} + aws_headers = { + "host", + "content-type", + "date", + "x-amz-date", + "x-amz-security-token", + "x-amz-content-sha256", + "x-amz-algorithm", + "x-amz-credential", + "x-amz-signedheaders", + "x-amz-signature", + } + + for header_name, header_value in headers.items(): + header_lower = header_name.lower() + if ( + header_lower in aws_headers + or header_lower.startswith("x-amz-") + or header_lower.startswith("x-amzn-") + ): + aws_signature_headers[header_name] = header_value + + return aws_signature_headers + def _sign_request( self, - service_name: Literal["bedrock", "sagemaker"], + service_name: Literal["bedrock", "sagemaker", "bedrock-agentcore"], headers: dict, optional_params: dict, request_data: dict, diff --git a/litellm/llms/bedrock/chat/agentcore/__init__.py b/litellm/llms/bedrock/chat/agentcore/__init__.py new file mode 100644 index 00000000000..a2f13876203 --- /dev/null +++ b/litellm/llms/bedrock/chat/agentcore/__init__.py @@ -0,0 +1,4 @@ +from .transformation import AmazonAgentCoreConfig + +__all__ = ["AmazonAgentCoreConfig"] + diff --git a/litellm/llms/bedrock/chat/agentcore/sse_iterator.py b/litellm/llms/bedrock/chat/agentcore/sse_iterator.py new file mode 100644 index 00000000000..35407337fdd --- /dev/null +++ b/litellm/llms/bedrock/chat/agentcore/sse_iterator.py @@ -0,0 +1,150 @@ +""" +SSE Stream Iterator for Bedrock AgentCore. + +Handles Server-Sent Events (SSE) streaming responses from AgentCore. +""" + +import json +from typing import TYPE_CHECKING + +import httpx + +from litellm._logging import verbose_logger +from litellm._uuid import uuid +from litellm.types.llms.bedrock_agentcore import AgentCoreUsage +from litellm.types.utils import Delta, ModelResponse, StreamingChoices, Usage + +if TYPE_CHECKING: + pass + + +class AgentCoreSSEStreamIterator: + """Async iterator for AgentCore SSE streaming responses.""" + + def __init__(self, response: httpx.Response, model: str): + self.response = response + self.model = model + self.finished = False + self.line_iterator = self.response.aiter_lines() + + def __aiter__(self): + return self + + async def __anext__(self) -> ModelResponse: + """Parse SSE events and yield ModelResponse chunks.""" + try: + async for line in self.line_iterator: + line = line.strip() + + if not line or not line.startswith('data:'): + continue + + # Extract JSON from SSE line + json_str = line[5:].strip() + if not json_str: + continue + + try: + data = json.loads(json_str) + + # Skip non-dict data + if not isinstance(data, dict): + continue + + # Process content delta events + if "event" in data and isinstance(data["event"], dict): + event_payload = data["event"] + content_block_delta = event_payload.get("contentBlockDelta") + + if content_block_delta: + delta = content_block_delta.get("delta", {}) + text = delta.get("text", "") + + if text: + # Yield chunk with text + chunk = ModelResponse( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=self.model, + object="chat.completion.chunk", + ) + + chunk.choices = [ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content=text, role="assistant"), + ) + ] + + return chunk + + # Check for metadata/usage + metadata = event_payload.get("metadata") + if metadata and "usage" in metadata: + # This is the final chunk with usage + chunk = ModelResponse( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=self.model, + object="chat.completion.chunk", + ) + + chunk.choices = [ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(), + ) + ] + + usage_data: AgentCoreUsage = metadata["usage"] # type: ignore + setattr(chunk, "usage", Usage( + prompt_tokens=usage_data.get("inputTokens", 0), + completion_tokens=usage_data.get("outputTokens", 0), + total_tokens=usage_data.get("totalTokens", 0), + )) + + self.finished = True + return chunk + + # Check for final message (alternative finish signal) + if "message" in data and isinstance(data["message"], dict): + if not self.finished: + chunk = ModelResponse( + id=f"chatcmpl-{uuid.uuid4()}", + created=0, + model=self.model, + object="chat.completion.chunk", + ) + + chunk.choices = [ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(), + ) + ] + + self.finished = True + return chunk + + except json.JSONDecodeError: + verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}") + continue + + # Stream ended naturally + raise StopAsyncIteration + + except StopAsyncIteration: + raise + except httpx.StreamConsumed: + # This is expected when the stream has been fully consumed + raise StopAsyncIteration + except httpx.StreamClosed: + # This is expected when the stream is closed + raise StopAsyncIteration + except Exception as e: + verbose_logger.error(f"Error in AgentCore SSE stream: {str(e)}") + raise StopAsyncIteration + diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py new file mode 100644 index 00000000000..7c65cad94df --- /dev/null +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -0,0 +1,695 @@ +""" +Transformation for Bedrock AgentCore + +https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgentRuntime.html +""" + +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast +from urllib.parse import quote + +import httpx + +from litellm._logging import verbose_logger +from litellm._uuid import uuid +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_content_list_to_str, +) +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.chat.agentcore.sse_iterator import AgentCoreSSEStreamIterator +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.types.llms.bedrock_agentcore import ( + AgentCoreMessage, + AgentCoreParsedResponse, + AgentCoreUsage, +) +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Choices, Message, ModelResponse, Usage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from litellm.utils import CustomStreamWrapper + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + HTTPHandler = Any + AsyncHTTPHandler = Any + CustomStreamWrapper = Any + + +class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): + def __init__(self, **kwargs): + BaseConfig.__init__(self, **kwargs) + BaseAWSLLM.__init__(self, **kwargs) + + def get_supported_openai_params(self, model: str) -> List[str]: + """ + Bedrock AgentCore has 0 OpenAI compatible params + """ + return [] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI params to AgentCore params + """ + return optional_params + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete url for the request + """ + ### SET RUNTIME ENDPOINT ### + aws_bedrock_runtime_endpoint = optional_params.get( + "aws_bedrock_runtime_endpoint", None + ) + + # Extract ARN from model string + agent_runtime_arn = self._get_agent_runtime_arn(model) + + # Parse ARN to get region + region = self._extract_region_from_arn(agent_runtime_arn) + + # Build the base endpoint URL for AgentCore + # Note: We don't use get_runtime_endpoint as AgentCore has its own endpoint structure + if aws_bedrock_runtime_endpoint: + base_url = aws_bedrock_runtime_endpoint + else: + base_url = f"https://bedrock-agentcore.{region}.amazonaws.com" + + # Based on boto3 client.invoke_agent_runtime, the path is: + # /runtimes/{URL-ENCODED-ARN}/invocations?qualifier= + encoded_arn = quote(agent_runtime_arn, safe="") + endpoint_url = f"{base_url}/runtimes/{encoded_arn}/invocations" + + # Add qualifier as query parameter if provided + if "qualifier" in optional_params: + endpoint_url = f"{endpoint_url}?qualifier={optional_params['qualifier']}" + + return endpoint_url + + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ) -> Tuple[dict, Optional[bytes]]: + # Check if api_key (bearer token) is provided for Cognito authentication + jwt_token = optional_params.get("api_key") + if jwt_token: + verbose_logger.debug( + f"AgentCore: Using Bearer token authentication (Cognito/JWT) - token: {jwt_token[:50]}..." + ) + headers["Content-Type"] = "application/json" + headers["Authorization"] = f"Bearer {jwt_token}" + # Return headers with bearer token and JSON-encoded body (not SigV4 signed) + return headers, json.dumps(request_data).encode() + + # Otherwise, use AWS SigV4 authentication + verbose_logger.debug("AgentCore: Using AWS SigV4 authentication (IAM)") + return self._sign_request( + service_name="bedrock-agentcore", + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + model=model, + stream=stream, + fake_stream=fake_stream, + api_key=api_key, + ) + + def _get_agent_runtime_arn(self, model: str) -> str: + """ + Extract ARN from model string + model = "agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC" + returns: "arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC" + """ + parts = model.split("/", 1) + if len(parts) != 2 or parts[0] != "agentcore": + raise ValueError( + "Invalid model format. Expected format: 'model=bedrock/agentcore/arn:aws:bedrock-agentcore:region:account:runtime/runtime_id'" + ) + return parts[1] + + def _extract_region_from_arn(self, arn: str) -> str: + """ + Extract region from ARN + arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/hosted_agent_r9jvp-3ySZuRHjLC + returns: us-west-2 + """ + parts = arn.split(":") + if len(parts) >= 4: + return parts[3] + raise ValueError(f"Invalid ARN format: {arn}") + + def _get_runtime_session_id(self, optional_params: dict) -> str: + """ + Get or generate runtime session ID (must be 33+ chars) + """ + session_id = optional_params.get("runtimeSessionId", None) + if session_id: + verbose_logger.debug(f"Using provided runtimeSessionId: {session_id}") + return session_id + + # Generate a session ID with 33+ characters + generated_id = f"litellm-session-{str(uuid.uuid4())}" + verbose_logger.debug(f"Generated new session ID: {generated_id}") + return generated_id + + def _get_runtime_user_id(self, optional_params: dict) -> Optional[str]: + """ + Get runtime user ID if provided + """ + user_id = optional_params.get("runtimeUserId", None) + if user_id: + verbose_logger.debug(f"Using provided runtimeUserId: {user_id}") + return user_id + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the request to AgentCore format. + + Based on boto3's implementation: + - Session ID goes in header: X-Amzn-Bedrock-AgentCore-Runtime-Session-Id + - User ID goes in header: X-Amzn-Bedrock-AgentCore-Runtime-User-Id + - Qualifier goes as query parameter + - Only the payload goes in the request body + + Returns: + dict: Payload dict containing the prompt + """ + verbose_logger.debug( + f"AgentCore transform_request - optional_params keys: {list(optional_params.keys())}" + ) + + # Use the last message content as the prompt + prompt = convert_content_list_to_str(messages[-1]) + + # Create the payload - this is what goes in the body (raw JSON) + payload: dict = {"prompt": prompt} + + # Get or generate session ID - this goes in the header + runtime_session_id = self._get_runtime_session_id(optional_params) + headers["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id"] = runtime_session_id + + # Get user ID if provided - this goes in the header + runtime_user_id = self._get_runtime_user_id(optional_params) + if runtime_user_id: + headers["X-Amzn-Bedrock-AgentCore-Runtime-User-Id"] = runtime_user_id + + # The request data is the payload dict (will be JSON encoded by the HTTP handler) + # Qualifier will be handled as a query parameter in get_complete_url + + verbose_logger.debug(f"PAYLOAD: {payload}") + return payload + + def _extract_sse_json(self, line: str) -> Optional[Dict]: + """Extract and parse JSON from an SSE data line.""" + if not line.startswith("data:"): + return None + + json_str = line[5:].strip() + if not json_str: + return None + + try: + data = json.loads(json_str) + # Skip non-dict data (some lines contain JSON strings) + return data if isinstance(data, dict) else None + except json.JSONDecodeError: + verbose_logger.debug(f"Skipping non-JSON line: {line[:100]}") + return None + + def _extract_usage_from_event(self, event_data: Dict) -> Optional[AgentCoreUsage]: + """Extract usage information from event metadata.""" + event_payload = event_data.get("event") + if not event_payload: + return None + + metadata = event_payload.get("metadata") + if metadata and "usage" in metadata: + return metadata["usage"] # type: ignore + + return None + + def _extract_content_delta(self, event_data: Dict) -> Optional[str]: + """Extract text content from contentBlockDelta event.""" + event_payload = event_data.get("event") + if not event_payload: + return None + + content_block_delta = event_payload.get("contentBlockDelta") + if not content_block_delta: + return None + + delta = content_block_delta.get("delta", {}) + return delta.get("text") + + def _extract_content_from_message(self, message: AgentCoreMessage) -> str: + """ + Extract text content from message content blocks. + This works for both SSE messages and JSON responses. + """ + content_list = message.get("content", []) + if not isinstance(content_list, list): + return "" + + return "".join( + block["text"] + for block in content_list + if isinstance(block, dict) and "text" in block + ) + + def _calculate_usage( + self, model: str, messages: List[AllMessageValues], content: str + ) -> Optional[Usage]: + """ + Calculate token usage using LiteLLM's token counter. + + Args: + model: The model name + messages: Input messages + content: Response content + + Returns: + Usage object with calculated tokens, or None if calculation fails + """ + try: + from litellm.utils import token_counter + + prompt_tokens = token_counter(model=model, messages=messages) + completion_tokens = token_counter( + model=model, text=content, count_response_tokens=True + ) + total_tokens = prompt_tokens + completion_tokens + + verbose_logger.debug( + f"Calculated usage - prompt: {prompt_tokens}, completion: {completion_tokens}, total: {total_tokens}" + ) + + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ) + except Exception as e: + verbose_logger.warning(f"Failed to calculate token usage: {str(e)}") + return None + + def _parse_json_response(self, response_json: dict) -> AgentCoreParsedResponse: + """ + Parse direct JSON response (non-streaming). + + JSON response structure: + { + "result": { + "role": "assistant", + "content": [{"text": "..."}] + } + } + """ + result = response_json.get("result", {}) + + # Extract content using the same helper as SSE parsing + content = self._extract_content_from_message(result) # type: ignore + + # JSON responses don't include usage data + return AgentCoreParsedResponse( + content=content, + usage=None, + final_message=result, # type: ignore + ) + + def _get_parsed_response( + self, raw_response: httpx.Response + ) -> AgentCoreParsedResponse: + """ + Parse AgentCore response based on content type. + + Args: + raw_response: Raw HTTP response from AgentCore + + Returns: + AgentCoreParsedResponse: Parsed response data + """ + content_type = raw_response.headers.get("content-type", "").lower() + verbose_logger.debug(f"AgentCore response Content-Type: {content_type}") + + # Parse response based on content type + if "application/json" in content_type: + # Direct JSON response + verbose_logger.debug("Parsing JSON response") + response_json = raw_response.json() + verbose_logger.debug(f"Response JSON: {response_json}") + return self._parse_json_response(response_json) + else: + # SSE stream response (text/event-stream or default) + verbose_logger.debug("Parsing SSE stream response") + response_text = raw_response.text + verbose_logger.debug( + f"AgentCore response (first 500 chars): {response_text[:500]}" + ) + return self._parse_sse_stream(response_text) + + def _parse_sse_stream(self, response_text: str) -> AgentCoreParsedResponse: + """ + Parse Server-Sent Events (SSE) stream format. + Each line starts with 'data:' followed by JSON. + + Returns: + AgentCoreParsedResponse: Parsed response with content, usage, and message + """ + final_message: Optional[AgentCoreMessage] = None + usage_data: Optional[AgentCoreUsage] = None + content_blocks: List[str] = [] + + for line in response_text.strip().split("\n"): + line = line.strip() + if not line: + continue + + data = self._extract_sse_json(line) + if not data: + continue + + verbose_logger.debug(f"SSE event keys: {list(data.keys())}") + + # Check for final complete message + if "message" in data and isinstance(data["message"], dict): + final_message = data["message"] # type: ignore + verbose_logger.debug("Found final message") + + # Process event data + if "event" in data and isinstance(data["event"], dict): + event_payload = data["event"] + verbose_logger.debug( + f"Event payload keys: {list(event_payload.keys())}" + ) + + # Extract usage metadata + if usage := self._extract_usage_from_event(data): + usage_data = usage + verbose_logger.debug(f"Found usage data: {usage_data}") + + # Collect content deltas + if text := self._extract_content_delta(data): + content_blocks.append(text) + + # Build final content + content = ( + self._extract_content_from_message(final_message) + if final_message + else "".join(content_blocks) + ) + + verbose_logger.debug(f"Final usage_data: {usage_data}") + + return AgentCoreParsedResponse( + content=content, usage=usage_data, final_message=final_message + ) + + def get_streaming_response( + self, + model: str, + raw_response: httpx.Response, + ) -> AgentCoreSSEStreamIterator: + """ + Return a streaming iterator for SSE responses. + + Args: + model: The model name + raw_response: Raw HTTP response with streaming data + + Returns: + AgentCoreSSEStreamIterator: Iterator that yields ModelResponse chunks + """ + return AgentCoreSSEStreamIterator(response=raw_response, model=model) + + def get_sync_custom_stream_wrapper( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj, + api_base: str, + headers: dict, + data: dict, + messages: list, + client: Optional[Union[HTTPHandler, "AsyncHTTPHandler"]] = None, + json_mode: Optional[bool] = None, + signed_json_body: Optional[bytes] = None, + ) -> CustomStreamWrapper: + """ + Get a CustomStreamWrapper for synchronous streaming. + + This is called when stream=True is passed to completion(). + """ + from litellm.llms.custom_httpx.http_handler import ( + HTTPHandler, + _get_httpx_client, + ) + from litellm.utils import CustomStreamWrapper + + if client is None or not isinstance(client, HTTPHandler): + client = _get_httpx_client(params={}) + + verbose_logger.debug(f"Making sync streaming request to: {api_base}") + + # Make streaming request + response = client.post( + api_base, + headers=headers, + data=signed_json_body if signed_json_body else json.dumps(data), + stream=True, # THIS IS KEY - tells httpx to not buffer + logging_obj=logging_obj, + ) + + if response.status_code != 200: + raise BedrockError( + status_code=response.status_code, message=str(response.read()) + ) + + # Create iterator for SSE stream + completion_stream = self.get_streaming_response( + model=model, raw_response=response + ) + + streaming_response = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response="first stream response received", + additional_args={"complete_input_dict": data}, + ) + + return streaming_response + + async def get_async_custom_stream_wrapper( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj, + api_base: str, + headers: dict, + data: dict, + messages: list, + client: Optional["AsyncHTTPHandler"] = None, + json_mode: Optional[bool] = None, + signed_json_body: Optional[bytes] = None, + ) -> CustomStreamWrapper: + """ + Get a CustomStreamWrapper for asynchronous streaming. + + This is called when stream=True is passed to acompletion(). + """ + from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, + ) + from litellm.utils import CustomStreamWrapper + + if client is None or not isinstance(client, AsyncHTTPHandler): + client = get_async_httpx_client( + llm_provider=cast(Any, "bedrock"), params={} + ) + + verbose_logger.debug(f"Making async streaming request to: {api_base}") + + # Make async streaming request + response = await client.post( + api_base, + headers=headers, + data=signed_json_body if signed_json_body else json.dumps(data), + stream=True, # THIS IS KEY - tells httpx to not buffer + logging_obj=logging_obj, + ) + + if response.status_code != 200: + raise BedrockError( + status_code=response.status_code, message=str(await response.aread()) + ) + + # Create iterator for SSE stream + completion_stream = self.get_streaming_response( + model=model, raw_response=response + ) + + streaming_response = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response="first stream response received", + additional_args={"complete_input_dict": data}, + ) + + return streaming_response + + @property + def has_custom_stream_wrapper(self) -> bool: + """Indicates that this config has custom streaming support.""" + return True + + @property + def supports_stream_param_in_request_body(self) -> bool: + """ + AgentCore does not allow passing `stream` in the request body. + Streaming is automatic based on the response format. + """ + return False + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + """ + Transform the AgentCore response to LiteLLM ModelResponse format. + AgentCore can return either JSON or SSE (Server-Sent Events) stream responses. + + Note: For streaming responses, use get_streaming_response() instead. + """ + try: + # Parse the response based on content type (JSON or SSE) + parsed_data = self._get_parsed_response(raw_response) + + content = parsed_data["content"] + usage_data = parsed_data["usage"] + + verbose_logger.debug(f"Parsed content length: {len(content)}") + verbose_logger.debug(f"Usage data: {usage_data}") + + # Create the message + message = Message(content=content, role="assistant") + + # Create choices + choice = Choices(finish_reason="stop", index=0, message=message) + + # Update model response + model_response.choices = [choice] + model_response.model = model + + # Add usage information if available + # Note: AgentCore JSON responses don't include usage data + # SSE responses may include usage in metadata events + if usage_data: + usage = Usage( + prompt_tokens=usage_data.get("inputTokens", 0), + completion_tokens=usage_data.get("outputTokens", 0), + total_tokens=usage_data.get("totalTokens", 0), + ) + setattr(model_response, "usage", usage) + else: + # Calculate token usage using LiteLLM's token counter + verbose_logger.debug( + "No usage data from AgentCore - calculating tokens" + ) + calculated_usage = self._calculate_usage(model, messages, content) + if calculated_usage: + setattr(model_response, "usage", calculated_usage) + + return model_response + + except Exception as e: + verbose_logger.error( + f"Error processing Bedrock AgentCore response: {str(e)}" + ) + raise BedrockError( + message=f"Error processing response: {str(e)}", + status_code=raw_response.status_code, + ) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + return headers + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return BedrockError(status_code=status_code, message=error_message) + + def should_fake_stream( + self, + model: Optional[str], + stream: Optional[bool], + custom_llm_provider: Optional[str] = None, + ) -> bool: + return True diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 54c603e5960..fd1f6f0c893 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -1,5 +1,4 @@ import json -import urllib from typing import Any, Optional, Union import httpx @@ -84,16 +83,6 @@ class BedrockConverseLLM(BaseAWSLLM): def __init__(self) -> None: super().__init__() - def encode_model_id(self, model_id: str) -> str: - """ - Double encode the model ID to ensure it matches the expected double-encoded format. - Args: - model_id (str): The model ID to encode. - Returns: - str: The double-encoded model ID. - """ - return urllib.parse.quote(model_id, safe="") # type: ignore - async def async_streaming( self, model: str, diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index d099c9813d6..d76a3c31b51 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1439,11 +1439,6 @@ class AmazonConverseConfig(BaseConfig): if stream is True: if model is not None: ################################################################### - # GPT-OSS models do not support streaming - ################################################################### - if "gpt-oss" in model: - return True - ################################################################### # AI21 models do not support streaming ################################################################### if "ai21" in model: diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 71aadffe5bb..53cbafcbe6a 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -3,20 +3,15 @@ TODO: DELETE FILE. Bedrock LLM is no longer used. Goto `litellm/llms/bedrock/cha """ import copy -import json import time import types -import urllib.parse from functools import partial from typing import ( - Any, AsyncIterator, Callable, Iterator, - List, Optional, Tuple, - Union, cast, get_args, ) @@ -672,16 +667,6 @@ class BedrockLLM(BaseAWSLLM): return model_response - def encode_model_id(self, model_id: str) -> str: - """ - Double encode the model ID to ensure it matches the expected double-encoded format. - Args: - model_id (str): The model ID to encode. - Returns: - str: The double-encoded model ID. - """ - return urllib.parse.quote(model_id, safe="") - def completion( # noqa: PLR0915 self, model: str, @@ -1176,33 +1161,6 @@ class BedrockLLM(BaseAWSLLM): return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider) return None - def get_bedrock_model_id( - self, - optional_params: dict, - provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL], - model: str, - ) -> str: - modelId = optional_params.pop("model_id", None) - if modelId is not None: - modelId = self.encode_model_id(model_id=modelId) - else: - modelId = model - - if provider == "llama" and "llama/" in modelId: - modelId = self._get_model_id_for_llama_like_model(modelId) - - return modelId - - def _get_model_id_for_llama_like_model( - self, - model: str, - ) -> str: - """ - Remove `llama` from modelID since `llama` is simply a spec to follow for custom bedrock models - """ - model_id = model.replace("llama/", "") - return self.encode_model_id(model_id=model_id) - def get_response_stream_shape(): global _response_stream_shape_cache diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py new file mode 100644 index 00000000000..b3a957ce0f8 --- /dev/null +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py @@ -0,0 +1,219 @@ +""" +Handles transforming requests for `bedrock/invoke/{qwen3} models` + +Inherits from `AmazonInvokeConfig` + +Qwen3 + Invoke API Tutorial: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html +""" + +from typing import Any, List, Optional + +import httpx + +from litellm.llms.base_llm.chat.transformation import BaseConfig +from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( + AmazonInvokeConfig, + LiteLLMLoggingObj, +) +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ModelResponse + + +class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig): + """ + Config for sending `qwen3` requests to `/bedrock/invoke/` + + Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html + """ + + max_tokens: Optional[int] = None + temperature: Optional[float] = None + top_p: Optional[float] = None + top_k: Optional[int] = None + stop: Optional[List[str]] = None + + def __init__( + self, + max_tokens: Optional[int] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + stop: Optional[List[str]] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + AmazonInvokeConfig.__init__(self) + + def get_supported_openai_params(self, model: str) -> List[str]: + return [ + "max_tokens", + "temperature", + "top_p", + "top_k", + "stop", + "stream", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + for k, v in non_default_params.items(): + if k == "max_tokens": + optional_params["max_tokens"] = v + if k == "temperature": + optional_params["temperature"] = v + if k == "top_p": + optional_params["top_p"] = v + if k == "top_k": + optional_params["top_k"] = v + if k == "stop": + optional_params["stop"] = v + if k == "stream": + optional_params["stream"] = v + return optional_params + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform OpenAI format to Qwen3 Bedrock invoke format + """ + # Convert messages to prompt format + prompt = self._convert_messages_to_prompt(messages) + + # Build the request body + request_body = { + "prompt": prompt, + } + + # Add optional parameters + if "max_tokens" in optional_params: + request_body["max_gen_len"] = optional_params["max_tokens"] + if "temperature" in optional_params: + request_body["temperature"] = optional_params["temperature"] + if "top_p" in optional_params: + request_body["top_p"] = optional_params["top_p"] + if "top_k" in optional_params: + request_body["top_k"] = optional_params["top_k"] + if "stop" in optional_params: + request_body["stop"] = optional_params["stop"] + + return request_body + + def _convert_messages_to_prompt(self, messages: List[AllMessageValues]) -> str: + """ + Convert OpenAI messages format to Qwen3 prompt format + Supports tool calls, multimodal content, and various message types + """ + prompt_parts = [] + + for message in messages: + role = message.get("role", "") + content = message.get("content", "") + tool_calls = message.get("tool_calls", []) + + if role == "system": + prompt_parts.append(f"<|im_start|>system\n{content}<|im_end|>") + elif role == "user": + # Handle multimodal content + if isinstance(content, list): + text_content = [] + for item in content: + if item.get("type") == "text": + text_content.append(item.get("text", "")) + elif item.get("type") == "image_url": + # For Qwen3, we can include image placeholders + text_content.append("<|vision_start|><|image_pad|><|vision_end|>") + content = "".join(text_content) + prompt_parts.append(f"<|im_start|>user\n{content}<|im_end|>") + elif role == "assistant": + if tool_calls and isinstance(tool_calls, list): + # Handle tool calls + for tool_call in tool_calls: + function_name = tool_call.get("function", {}).get("name", "") + function_args = tool_call.get("function", {}).get("arguments", "") + prompt_parts.append(f"<|im_start|>assistant\n\n{{\"name\": \"{function_name}\", \"arguments\": \"{function_args}\"}}\n<|im_end|>") + else: + prompt_parts.append(f"<|im_start|>assistant\n{content}<|im_end|>") + elif role == "tool": + # Handle tool responses + prompt_parts.append(f"<|im_start|>tool\n{content}<|im_end|>") + + # Add assistant start token for response generation + prompt_parts.append("<|im_start|>assistant\n") + + return "\n".join(prompt_parts) + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + """ + Transform Qwen3 Bedrock response to OpenAI format + """ + try: + if hasattr(raw_response, 'json'): + response_data = raw_response.json() + else: + response_data = raw_response + + # Extract the generated text - Qwen3 uses "generation" field + generated_text = response_data.get("generation", "") + + # Clean up the response (remove assistant start token if present) + if generated_text.startswith("<|im_start|>assistant\n"): + generated_text = generated_text[len("<|im_start|>assistant\n"):] + if generated_text.endswith("<|im_end|>"): + generated_text = generated_text[:-len("<|im_end|>")] + + # Set the content in the existing model_response structure + if hasattr(model_response, 'choices') and len(model_response.choices) > 0: + choice = model_response.choices[0] + if hasattr(choice, 'message'): + choice.message.content = generated_text + choice.finish_reason = "stop" + else: + # Handle streaming choices + choice.delta.content = generated_text + choice.finish_reason = "stop" + + # Set usage information if available in response + if "usage" in response_data: + usage_data = response_data["usage"] + if hasattr(model_response, 'usage'): + model_response.usage.prompt_tokens = usage_data.get("prompt_tokens", 0) + model_response.usage.completion_tokens = usage_data.get("completion_tokens", 0) + model_response.usage.total_tokens = usage_data.get("total_tokens", 0) + + return model_response + + except Exception as e: + if logging_obj: + logging_obj.post_call( + input=messages, + api_key=api_key, + original_response=raw_response, + additional_args={"error": str(e)}, + ) + raise e diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 9b13d3df08e..02b8fd57115 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -69,11 +69,19 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): litellm_params: dict, headers: dict, ) -> dict: + # Filter out AWS authentication parameters before passing to Anthropic transformation + # AWS params should only be used for signing requests, not included in request body + filtered_params = { + k: v + for k, v in optional_params.items() + if k not in self.aws_authentication_params + } + _anthropic_request = AnthropicConfig.transform_request( self, model=model, messages=messages, - optional_params=optional_params, + optional_params=filtered_params, litellm_params=litellm_params, headers=headers, ) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 08a0690716b..e6146f1064e 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -1,7 +1,6 @@ import copy import json import time -import urllib.parse from functools import partial from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union, cast, get_args @@ -190,14 +189,16 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): ] = True # cohere requires stream = True in inference params request_data = {"prompt": prompt, **inference_params} elif provider == "anthropic": - transformed_request = litellm.AmazonAnthropicClaudeConfig().transform_request( - model=model, - messages=messages, - optional_params=optional_params, - litellm_params=litellm_params, - headers=headers, + transformed_request = ( + litellm.AmazonAnthropicClaudeConfig().transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) ) - + return transformed_request elif provider == "nova": return litellm.AmazonInvokeNovaConfig().transform_request( @@ -327,7 +328,9 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): elif provider == "meta" or provider == "llama" or provider == "deepseek_r1": outputText = completion_response["generation"] elif provider == "mistral": - outputText = litellm.AmazonMistralConfig.get_outputText(completion_response, model_response) + outputText = litellm.AmazonMistralConfig.get_outputText( + completion_response, model_response + ) else: # amazon titan outputText = completion_response.get("results")[0].get("outputText") except Exception as e: @@ -549,48 +552,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider) return None - def get_bedrock_model_id( - self, - optional_params: dict, - provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL], - model: str, - ) -> str: - modelId = optional_params.pop("model_id", None) - if modelId is not None: - modelId = self.encode_model_id(model_id=modelId) - else: - modelId = model - - modelId = modelId.replace("invoke/", "", 1) - if provider == "llama" and "llama/" in modelId: - modelId = self._get_model_id_from_model_with_spec(modelId, spec="llama") - elif provider == "deepseek_r1" and "deepseek_r1/" in modelId: - modelId = self._get_model_id_from_model_with_spec( - modelId, spec="deepseek_r1" - ) - return modelId - - def _get_model_id_from_model_with_spec( - self, - model: str, - spec: str, - ) -> str: - """ - Remove `llama` from modelID since `llama` is simply a spec to follow for custom bedrock models - """ - model_id = model.replace(spec + "/", "") - return self.encode_model_id(model_id=model_id) - - def encode_model_id(self, model_id: str) -> str: - """ - Double encode the model ID to ensure it matches the expected double-encoded format. - Args: - model_id (str): The model ID to encode. - Returns: - str: The double-encoded model ID. - """ - return urllib.parse.quote(model_id, safe="") - def convert_messages_to_prompt( self, model, messages, provider, custom_prompt_dict ) -> Tuple[str, Optional[list]]: diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 1a599fda59f..baaec996535 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -237,6 +237,7 @@ def init_bedrock_client( "sts", aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, + verify=ssl_verify ) sts_response = sts_client.assume_role( @@ -440,22 +441,23 @@ class BedrockModelInfo(BaseLLMModelInfo): """ Abbreviations of regions AWS Bedrock supports for cross region inference """ - return ["global", "us", "eu", "apac", "jp", "au"] + return ["global", "us", "eu", "apac", "jp", "au", "us-gov"] @staticmethod def get_bedrock_route( model: str, - ) -> Literal["converse", "invoke", "converse_like", "agent", "async_invoke"]: + ) -> Literal["converse", "invoke", "converse_like", "agent", "agentcore", "async_invoke"]: """ Get the bedrock route for the given model. """ route_mappings: Dict[ - str, Literal["invoke", "converse_like", "converse", "agent", "async_invoke"] + str, Literal["invoke", "converse_like", "converse", "agent", "agentcore", "async_invoke"] ] = { "invoke/": "invoke", "converse_like/": "converse_like", "converse/": "converse", "agent/": "agent", + "agentcore/": "agentcore", "async_invoke/": "async_invoke", } @@ -494,6 +496,13 @@ class BedrockModelInfo(BaseLLMModelInfo): """ return "agent/" in model + @staticmethod + def _explicit_agentcore_route(model: str) -> bool: + """ + Check if the model is an explicit agentcore route. + """ + return "agentcore/" in model + @staticmethod def _explicit_converse_like_route(model: str) -> bool: """ @@ -538,6 +547,65 @@ class BedrockModelInfo(BaseLLMModelInfo): return None +def get_bedrock_chat_config(model: str): + """ + Helper function to get the appropriate Bedrock chat config based on model and route. + + Args: + model: The model name/identifier + + Returns: + The appropriate Bedrock config class instance + """ + bedrock_route = BedrockModelInfo.get_bedrock_route(model) + bedrock_invoke_provider = litellm.BedrockLLM.get_bedrock_invoke_provider( + model=model + ) + base_model = BedrockModelInfo.get_base_model(model) + + # Handle explicit routes first + if bedrock_route == "converse" or bedrock_route == "converse_like": + return litellm.AmazonConverseConfig() + elif bedrock_route == "agent": + from litellm.llms.bedrock.chat.invoke_agent.transformation import ( + AmazonInvokeAgentConfig, + ) + return AmazonInvokeAgentConfig() + elif bedrock_route == "agentcore": + from litellm.llms.bedrock.chat.agentcore.transformation import ( + AmazonAgentCoreConfig, + ) + return AmazonAgentCoreConfig() + + # Handle provider-specific configs + if bedrock_invoke_provider == "amazon": + return litellm.AmazonTitanConfig() + elif bedrock_invoke_provider == "anthropic": + if ( + base_model + in litellm.AmazonAnthropicConfig.get_legacy_anthropic_model_names() + ): + return litellm.AmazonAnthropicConfig() + else: + return litellm.AmazonAnthropicClaudeConfig() + elif bedrock_invoke_provider == "meta" or bedrock_invoke_provider == "llama": + return litellm.AmazonLlamaConfig() + elif bedrock_invoke_provider == "ai21": + return litellm.AmazonAI21Config() + elif bedrock_invoke_provider == "cohere": + return litellm.AmazonCohereConfig() + elif bedrock_invoke_provider == "mistral": + return litellm.AmazonMistralConfig() + elif bedrock_invoke_provider == "deepseek_r1": + return litellm.AmazonDeepSeekR1Config() + elif bedrock_invoke_provider == "nova": + return litellm.AmazonInvokeNovaConfig() + elif bedrock_invoke_provider == "qwen3": + return litellm.AmazonQwen3Config() + else: + return litellm.AmazonInvokeConfig() + + class BedrockEventStreamDecoderBase: """ Base class for event stream decoding for Bedrock @@ -826,6 +894,7 @@ class CommonBatchFilesUtils: Tuple of (bucket_name, object_key) """ import time + from litellm._uuid import uuid # Get bucket name diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 3edd6d6741b..fea29935975 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -391,7 +391,7 @@ class BedrockEmbedding(BaseAWSLLM): ) # default to model if not passed modelId = urllib.parse.quote(unencoded_model_id, safe="") aws_region_name = self._get_aws_region_name( - optional_params=optional_params, + optional_params={"aws_region_name": aws_region_name}, model=model, model_id=unencoded_model_id, ) diff --git a/litellm/llms/bedrock/image/amazon_titan_transformation.py b/litellm/llms/bedrock/image/amazon_titan_transformation.py new file mode 100644 index 00000000000..2709f406dfd --- /dev/null +++ b/litellm/llms/bedrock/image/amazon_titan_transformation.py @@ -0,0 +1,160 @@ +""" +Transformation logic for Amazon Titan Image Generation. +""" + +import types +from typing import List, Optional + +from openai.types.image import Image + +from litellm import get_model_info +from litellm.types.llms.bedrock import ( + AmazonNovaCanvasImageGenerationConfig, + AmazonTitanImageGenerationRequestBody, + AmazonTitanTextToImageParams, +) +from litellm.types.utils import ImageResponse + + +class AmazonTitanImageGenerationConfig: + """ + Reference: https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=stability.stable-diffusion-xl-v0 + """ + + cfg_scale: Optional[int] = None + seed: Optional[float] = None + steps: Optional[List[str]] = None + width: Optional[int] = None + height: Optional[int] = None + + def __init__( + self, + cfg_scale: Optional[int] = None, + seed: Optional[float] = None, + steps: Optional[List[str]] = None, + width: Optional[int] = None, + height: Optional[int] = None, + ) -> None: + locals_ = locals().copy() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + @classmethod + def _is_titan_model(cls, model: Optional[str] = None) -> bool: + """ + Returns True if the model is a Titan model + + Titan models follow this pattern: + + """ + if model and "amazon.titan" in model: + return True + return False + + @classmethod + def get_supported_openai_params(cls, model: Optional[str] = None) -> List: + return ["size", "n", "quality"] + + @classmethod + def map_openai_params( + cls, + non_default_params: dict, + optional_params: dict, + ): + from typing import Any, Dict + + image_generation_config: Dict[str, Any] = {} + for k, v in non_default_params.items(): + if k == "size" and v is not None: + width, height = v.split("x") + image_generation_config["width"] = int(width) + image_generation_config["height"] = int(height) + elif k == "n" and v is not None: + image_generation_config["numberOfImages"] = v + elif ( + k == "quality" and v is not None + ): # 'auto', 'hd', 'standard', 'high', 'medium', 'low' + if v in ("hd", "premium", "high"): + image_generation_config["quality"] = "premium" + elif v in ("standard", "medium", "low"): + image_generation_config["quality"] = "standard" + + if image_generation_config: + optional_params["imageGenerationConfig"] = image_generation_config + return optional_params + + @classmethod + def _transform_request( + cls, + input: str, + optional_params: dict, + ) -> AmazonTitanImageGenerationRequestBody: + from typing import Any, Dict + + image_generation_config = optional_params.pop("imageGenerationConfig", {}) + negative_text = optional_params.pop("negativeText", None) + text_to_image_params: Dict[str, Any] = {"text": input} + if negative_text: + text_to_image_params["negativeText"] = negative_text + task_type = optional_params.pop("taskType", "TEXT_IMAGE") + user_specified_image_generation_config = optional_params.pop( + "imageGenerationConfig", {} + ) + image_generation_config = { + **image_generation_config, + **user_specified_image_generation_config, + } + return AmazonTitanImageGenerationRequestBody( + taskType=task_type, + textToImageParams=AmazonTitanTextToImageParams(**text_to_image_params), # type: ignore + imageGenerationConfig=AmazonNovaCanvasImageGenerationConfig( + **image_generation_config + ), + ) + + @classmethod + def transform_response_dict_to_openai_response( + cls, model_response: ImageResponse, response_dict: dict + ) -> ImageResponse: + image_list: List[Image] = [] + for image in response_dict["images"]: + _image = Image(b64_json=image) + image_list.append(_image) + + model_response.data = image_list + + return model_response + + @classmethod + def cost_calculator( + cls, + model: str, + image_response: ImageResponse, + size: Optional[str] = None, + optional_params: Optional[dict] = None, + ) -> float: + model_info = get_model_info(model=model) + output_cost_per_image = model_info.get("output_cost_per_image") or 0.0 + if not image_response.data: + return 0.0 + num_images = len(image_response.data) + return output_cost_per_image * num_images diff --git a/litellm/llms/bedrock/image/cost_calculator.py b/litellm/llms/bedrock/image/cost_calculator.py index a0dc91d7119..9b2ae8782cb 100644 --- a/litellm/llms/bedrock/image/cost_calculator.py +++ b/litellm/llms/bedrock/image/cost_calculator.py @@ -1,6 +1,9 @@ from typing import Optional import litellm +from litellm.llms.bedrock.image.amazon_titan_transformation import ( + AmazonTitanImageGenerationConfig, +) from litellm.types.utils import ImageResponse @@ -17,6 +20,13 @@ def cost_calculator( """ if litellm.AmazonStability3Config()._is_stability_3_model(model=model): pass + elif AmazonTitanImageGenerationConfig._is_titan_model(model=model): + return AmazonTitanImageGenerationConfig.cost_calculator( + model=model, + image_response=image_response, + size=size, + optional_params=optional_params, + ) else: # Stability 1 models optional_params = optional_params or {} diff --git a/litellm/llms/bedrock/image/image_handler.py b/litellm/llms/bedrock/image/image_handler.py index 0103f190d36..313a1dc17bd 100644 --- a/litellm/llms/bedrock/image/image_handler.py +++ b/litellm/llms/bedrock/image/image_handler.py @@ -7,8 +7,18 @@ import httpx from pydantic import BaseModel import litellm +from litellm import BEDROCK_INVOKE_PROVIDERS_LITERAL from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging +from litellm.llms.bedrock.image.amazon_nova_canvas_transformation import ( + AmazonNovaCanvasConfig, +) +from litellm.llms.bedrock.image.amazon_stability3_transformation import ( + AmazonStability3Config, +) +from litellm.llms.bedrock.image.amazon_titan_transformation import ( + AmazonTitanImageGenerationConfig, +) from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -63,7 +73,7 @@ class BedrockImageGeneration(BaseAWSLLM): extra_headers=extra_headers, logging_obj=logging_obj, prompt=prompt, - api_key=api_key + api_key=api_key, ) if aimg_generation is True: @@ -174,8 +184,14 @@ class BedrockImageGeneration(BaseAWSLLM): optional_params, model ) + # Use the existing ARN-aware provider detection method + bedrock_provider = self.get_bedrock_invoke_provider(model) ### SET RUNTIME ENDPOINT ### - modelId = model + modelId = self.get_bedrock_model_id( + model=model, + provider=bedrock_provider, + optional_params=optional_params, + ) _, proxy_endpoint_url = self.get_runtime_endpoint( api_base=api_base, aws_bedrock_runtime_endpoint=boto3_credentials_info.aws_bedrock_runtime_endpoint, @@ -183,14 +199,17 @@ class BedrockImageGeneration(BaseAWSLLM): ) proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke" data = self._get_request_body( - model=model, prompt=prompt, optional_params=optional_params + model=model, + prompt=prompt, + optional_params=optional_params, + bedrock_provider=bedrock_provider, ) # Make POST Request body = json.dumps(data).encode("utf-8") headers = {"Content-Type": "application/json"} if extra_headers is not None: - headers = {"Content-Type": "application/json", **extra_headers} + headers = {"Content-Type": "application/json", **extra_headers} prepped = self.get_request_headers( credentials=boto3_credentials_info.credentials, @@ -201,7 +220,7 @@ class BedrockImageGeneration(BaseAWSLLM): headers=headers, api_key=api_key, ) - + ## LOGGING logging_obj.pre_call( input=prompt, @@ -222,6 +241,7 @@ class BedrockImageGeneration(BaseAWSLLM): def _get_request_body( self, model: str, + bedrock_provider: Optional[BEDROCK_INVOKE_PROVIDERS_LITERAL], prompt: str, optional_params: dict, ) -> dict: @@ -233,9 +253,6 @@ class BedrockImageGeneration(BaseAWSLLM): Returns: dict: The request body to use for the Bedrock Image Generation API """ - # Use the existing ARN-aware provider detection method - bedrock_provider = self.get_bedrock_invoke_provider(model) - if bedrock_provider == "amazon" or bedrock_provider == "nova": # Handle Amazon Nova Canvas models provider = "amazon" @@ -306,15 +323,21 @@ class BedrockImageGeneration(BaseAWSLLM): if response_dict is None: raise ValueError("Error in response object format, got None") - config_class = ( - litellm.AmazonStability3Config - if litellm.AmazonStability3Config._is_stability_3_model(model=model) - else ( - litellm.AmazonNovaCanvasConfig - if litellm.AmazonNovaCanvasConfig._is_nova_model(model=model) - else litellm.AmazonStabilityConfig - ) - ) + config_class: Union[ + type[AmazonTitanImageGenerationConfig], + type[AmazonNovaCanvasConfig], + type[AmazonStability3Config], + type[litellm.AmazonStabilityConfig], + ] + if AmazonTitanImageGenerationConfig._is_titan_model(model=model): + config_class = AmazonTitanImageGenerationConfig + elif AmazonNovaCanvasConfig._is_nova_model(model=model): + config_class = AmazonNovaCanvasConfig + elif AmazonStability3Config._is_stability_3_model(model=model): + config_class = AmazonStability3Config + else: + config_class = litellm.AmazonStabilityConfig + config_class.transform_response_dict_to_openai_response( model_response=model_response, response_dict=response_dict, diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 4fa8517a090..be782d35766 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -111,7 +111,6 @@ class AmazonAnthropicClaudeMessagesConfig( litellm_params=litellm_params, headers=headers, ) - ######################################################### ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### ######################################################### diff --git a/litellm/llms/bedrock/vector_stores/transformation.py b/litellm/llms/bedrock/vector_stores/transformation.py index c05b6ba3fb1..72e1e1470d3 100644 --- a/litellm/llms/bedrock/vector_stores/transformation.py +++ b/litellm/llms/bedrock/vector_stores/transformation.py @@ -13,6 +13,9 @@ from litellm.types.integrations.rag.bedrock_knowledgebase import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_stores import ( + BaseVectorStoreAuthCredentials, + VectorStoreIndexEndpoints, + VECTOR_STORE_OPENAI_PARAMS, VectorStoreResultContent, VectorStoreSearchOptionalRequestParams, VectorStoreSearchResponse, @@ -32,6 +35,145 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): BaseVectorStoreConfig.__init__(self) BaseAWSLLM.__init__(self) + def get_auth_credentials( + self, litellm_params: dict + ) -> BaseVectorStoreAuthCredentials: + return {} + + def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints: + return { + "read": [("POST", "/knowledgebases/{knowledge_base_id}/retrieve")], + "write": [], + } + + def get_supported_openai_params( + self, model: str + ) -> List[VECTOR_STORE_OPENAI_PARAMS]: + return ["filters", "max_num_results", "ranking_options"] + + def _map_operator_to_aws(self, operator: str) -> str: + """ + Map OpenAI-style operators to AWS Bedrock operator names. + + OpenAI uses: eq, ne, gt, gte, lt, lte, in, nin + AWS uses: equals, notEquals, greaterThan, greaterThanOrEquals, lessThan, lessThanOrEquals, in, notIn, startsWith, listContains, stringContains + """ + operator_mapping = { + "eq": "equals", + "ne": "notEquals", + "gt": "greaterThan", + "gte": "greaterThanOrEquals", + "lt": "lessThan", + "lte": "lessThanOrEquals", + "in": "in", + "nin": "notIn", + # AWS-specific operators (pass through) + "equals": "equals", + "notEquals": "notEquals", + "greaterThan": "greaterThan", + "greaterThanOrEquals": "greaterThanOrEquals", + "lessThan": "lessThan", + "lessThanOrEquals": "lessThanOrEquals", + "notIn": "notIn", + "startsWith": "startsWith", + "listContains": "listContains", + "stringContains": "stringContains", + } + return operator_mapping.get(operator, operator) + + def _map_operator_filter(self, filter_dict: dict) -> dict: + """ + Map a single OpenAI operator filter to AWS KB format. + + OpenAI format: {"key": , "value": , "operator": } + AWS KB format: {"operator": {"key": , "value": }} + """ + aws_operator = self._map_operator_to_aws(filter_dict["operator"]) + return { + aws_operator: { + "key": filter_dict["key"], + "value": filter_dict["value"], + } + } + + def _map_and_or_filters(self, value: dict) -> dict: + """ + Map OpenAI and/or filters to AWS KB format. + + OpenAI format: {"and" | "or": [{"key": , "value": , "operator": }]} + AWS KB format: {"andAll" | "orAll": [{"operator": {"key": , "value": }}]} + + Note: AWS requires andAll/orAll to have at least 2 elements. + For single filters, unwrap and return just the operator. + """ + aws_filters = {} + + if "and" in value: + and_filters = value["and"] + # If only 1 filter, return just the operator (AWS requires andAll to have >=2 elements) + if len(and_filters) == 1: + return self._map_operator_filter(and_filters[0]) + + aws_filters["andAll"] = [ + { + self._map_operator_to_aws(and_filters[i]["operator"]): { + "key": and_filters[i]["key"], + "value": and_filters[i]["value"], + } + } + for i in range(len(and_filters)) + ] + + if "or" in value: + or_filters = value["or"] + # If only 1 filter, return just the operator (AWS requires orAll to have >=2 elements) + if len(or_filters) == 1: + return self._map_operator_filter(or_filters[0]) + + aws_filters["orAll"] = [ + { + self._map_operator_to_aws(or_filters[i]["operator"]): { + "key": or_filters[i]["key"], + "value": or_filters[i]["value"], + } + } + for i in range(len(or_filters)) + ] + + return aws_filters + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + drop_params: bool, + ) -> dict: + for param, value in non_default_params.items(): + if param == "max_num_results": + optional_params["numberOfResults"] = value + elif param == "filters" and value is not None: + + # map the openai filters to the aws kb filters format + # openai filters = {"key": , "value": , "operator": } OR {"and" | "or": [{"key": , "value": , "operator": }]} + # aws kb filters = {"operator": {"": }} OR {"andAll | orAll": [{"operator": {"": }}]} + # 1. check if filter is in openai format + # 2. if it is, map it to the aws kb filters format + # 3. if it is not, assume it is in aws kb filters format and add it to the optional_params + aws_filters: Optional[Dict] = None + + if isinstance(value, dict): + if "operator" in value.keys(): + # Single operator - map directly (no wrapping needed) + aws_filters = self._map_operator_filter(value) + elif "and" in value.keys() or "or" in value.keys(): + aws_filters = self._map_and_or_filters(value) + else: + # Assume it's already in AWS KB format + aws_filters = value + optional_params["filters"] = aws_filters + + return optional_params + def validate_environment( self, headers: dict, litellm_params: Optional[GenericLiteLLMParams] ) -> dict: @@ -39,13 +181,13 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): headers.setdefault("Content-Type", "application/json") return headers - def get_complete_url( - self, api_base: Optional[str], litellm_params: dict - ) -> str: + def get_complete_url(self, api_base: Optional[str], litellm_params: dict) -> str: aws_region_name = litellm_params.get("aws_region_name") endpoint_url, _ = self.get_runtime_endpoint( api_base=api_base, - aws_bedrock_runtime_endpoint=litellm_params.get("aws_bedrock_runtime_endpoint"), + aws_bedrock_runtime_endpoint=litellm_params.get( + "aws_bedrock_runtime_endpoint" + ), aws_region_name=self.get_aws_region_name_for_non_llm_api_calls( aws_region_name=aws_region_name ), @@ -86,7 +228,9 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): # Create a properly typed retrieval configuration typed_retrieval_config: BedrockKBRetrievalConfiguration = {} if "vectorSearchConfiguration" in retrieval_config: - typed_retrieval_config["vectorSearchConfiguration"] = retrieval_config["vectorSearchConfiguration"] + typed_retrieval_config["vectorSearchConfiguration"] = retrieval_config[ + "vectorSearchConfiguration" + ] request_body["retrievalConfiguration"] = typed_retrieval_config litellm_logging_obj.model_call_details["query"] = query @@ -117,8 +261,12 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): source_uri = metadata.get("x-amz-bedrock-kb-source-uri", "") if metadata else "" if source_uri: return source_uri - - chunk_id = metadata.get("x-amz-bedrock-kb-chunk-id", "unknown") if metadata else "unknown" + + chunk_id = ( + metadata.get("x-amz-bedrock-kb-chunk-id", "unknown") + if metadata + else "unknown" + ) return f"bedrock-kb-{chunk_id}" def _get_filename_from_metadata(self, metadata: Dict[str, Any]) -> str: @@ -127,18 +275,26 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): Tries to extract filename from source URI, falls back to domain name or data source ID. """ source_uri = metadata.get("x-amz-bedrock-kb-source-uri", "") if metadata else "" - + if source_uri: try: parsed_uri = urlparse(source_uri) - filename = parsed_uri.path.split('/')[-1] if parsed_uri.path and parsed_uri.path != '/' else parsed_uri.netloc - if not filename or filename == '/': + filename = ( + parsed_uri.path.split("/")[-1] + if parsed_uri.path and parsed_uri.path != "/" + else parsed_uri.netloc + ) + if not filename or filename == "/": filename = parsed_uri.netloc return filename except Exception: return source_uri - - data_source_id = metadata.get("x-amz-bedrock-kb-data-source-id", "unknown") if metadata else "unknown" + + data_source_id = ( + metadata.get("x-amz-bedrock-kb-data-source-id", "unknown") + if metadata + else "unknown" + ) return f"bedrock-kb-document-{data_source_id}" def _get_attributes_from_metadata(self, metadata: Dict[str, Any]) -> Dict[str, Any]: @@ -161,13 +317,13 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): text = content.get("text") if content else None if text is None: continue - + # Extract metadata and use helper functions metadata = item.get("metadata", {}) or {} file_id = self._get_file_id_from_metadata(metadata) filename = self._get_filename_from_metadata(metadata) attributes = self._get_attributes_from_metadata(metadata) - + results.append( VectorStoreSearchResult( score=item.get("score"), diff --git a/litellm/llms/clarifai/chat/transformation.py b/litellm/llms/clarifai/chat/transformation.py index 73be89fc6e7..48884ff0139 100644 --- a/litellm/llms/clarifai/chat/transformation.py +++ b/litellm/llms/clarifai/chat/transformation.py @@ -1,262 +1,133 @@ -import json -from typing import TYPE_CHECKING, Any, AsyncIterator, Iterator, List, Optional, Union +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union import httpx -from litellm.litellm_core_utils.prompt_templates.common_utils import ( - convert_content_list_to_str, +from litellm.secret_managers.main import get_secret_str +from litellm.types.utils import ModelResponse +from litellm.types.llms.openai import ( + AllMessageValues, ) -from litellm.llms.base_llm.base_model_iterator import FakeStreamResponseIterator -from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException -from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import ( - ChatCompletionToolCallChunk, - ChatCompletionUsageBlock, - Choices, - GenericStreamingChunk, - Message, - ModelResponse, - Usage, -) -from litellm.utils import token_counter +from litellm.llms.openai.common_utils import OpenAIError +from litellm.llms.base_llm.chat.transformation import BaseLLMException -from ..common_utils import ClarifaiError +from ...openai.chat.gpt_transformation import OpenAIGPTConfig if TYPE_CHECKING: - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - LoggingClass = LiteLLMLoggingObj + LiteLLMLoggingObj = _LiteLLMLoggingObj else: - LoggingClass = Any + LiteLLMLoggingObj = Any -class ClarifaiConfig(BaseConfig): +class ClarifaiConfig(OpenAIGPTConfig): """ - Reference: https://clarifai.com/meta/Llama-2/models/llama2-70b-chat + Configuration class for Clarifai chat completions. + Since Clarifai is OpenAI-compatible, we extend OpenAIGPTConfig. """ - - max_tokens: Optional[int] = None - temperature: Optional[int] = None - top_k: Optional[int] = None - - def __init__( - self, - max_tokens: Optional[int] = None, - temperature: Optional[int] = None, - top_k: Optional[int] = None, - ) -> None: - locals_ = locals().copy() - for key, value in locals_.items(): - if key != "self" and value is not None: - setattr(self.__class__, key, value) - - @classmethod - def get_config(cls): - return super().get_config() - def get_supported_openai_params(self, model: str) -> list: + """ + Get the supported OpenAI params for the given model + """ return [ - "temperature", "max_tokens", + "max_completion_tokens", + "response_format", + "stream", + "temperature", + "top_p", + "tool_choice", + "tools", + "presence_penalty", + "frequency_penalty", + "stream_options", ] - - def map_openai_params( - self, - non_default_params: dict, - optional_params: dict, - model: str, - drop_params: bool, - ) -> dict: - for param, value in non_default_params.items(): - if param == "temperature": - optional_params["temperature"] = value - elif param == "max_tokens": - optional_params["max_tokens"] = value - - return optional_params - - def _completions_to_model(self, prompt: str, optional_params: dict) -> dict: - params = {} - if temperature := optional_params.get("temperature"): - params["temperature"] = temperature - if max_tokens := optional_params.get("max_tokens"): - params["max_tokens"] = max_tokens - return { - "inputs": [{"data": {"text": {"raw": prompt}}}], - "model": {"output_info": {"params": params}}, - } - - def _convert_model_to_url(self, model: str, api_base: str): - user_id, app_id, model_id = model.split(".") - return f"{api_base}/users/{user_id}/apps/{app_id}/models/{model_id}/outputs" - - def transform_request( - self, - model: str, - messages: List[AllMessageValues], - optional_params: dict, - litellm_params: dict, - headers: dict, - ) -> dict: - prompt = " ".join(convert_content_list_to_str(message) for message in messages) - - ## Load Config - config = self.get_config() - for k, v in config.items(): - if k not in optional_params: - optional_params[k] = v - - data = self._completions_to_model( - prompt=prompt, optional_params=optional_params + + @staticmethod + def get_api_key(api_key: Optional[str] = None) -> Optional[str]: + return ( + api_key + or get_secret_str("CLARIFAI_API_KEY") ) + + @staticmethod + def get_api_base(api_base: Optional[str] = None) -> Optional[str]: + return api_base or "https://api.clarifai.com/v2/ext/openai/v1" + + @staticmethod + def get_base_model(model: Optional[str] = None) -> Optional[str]: + if model: + user_id, app_id, model_id = model.split(".") + return f"https://clarifai.com/{user_id}/{app_id}/models/{model_id}" + return None - return data - - def validate_environment( + def _get_openai_compatible_provider_info( self, - headers: dict, - model: str, - messages: List[AllMessageValues], - optional_params: dict, - litellm_params: dict, - api_key: Optional[str] = None, - api_base: Optional[str] = None, - ) -> dict: - headers = { - "accept": "application/json", - "content-type": "application/json", - } - - if api_key: - headers["Authorization"] = f"Bearer {api_key}" - return headers - - def get_error_class( - self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] - ) -> BaseLLMException: - return ClarifaiError(message=error_message, status_code=status_code) - + api_base: Optional[str], + api_key: Optional[str], + ) -> Tuple[Optional[str], Optional[str]]: + """ + Get API base and key for Clarifai provider. + """ + api_base = api_base or "https://api.clarifai.com/v2/ext/openai/v1" + dynamic_api_key = api_key or get_secret_str("CLARIFAI_API_KEY") or "" + return api_base, dynamic_api_key + + def transform_request(self, model, messages, optional_params, litellm_params, headers): + model = self.get_base_model(model) or model + return super().transform_request(model, messages, optional_params, litellm_params, headers) + def transform_response( self, model: str, raw_response: httpx.Response, model_response: ModelResponse, - logging_obj: LoggingClass, + logging_obj: LiteLLMLoggingObj, request_data: dict, messages: List[AllMessageValues], optional_params: dict, litellm_params: dict, - encoding: str, + encoding: Any, api_key: Optional[str] = None, json_mode: Optional[bool] = None, ) -> ModelResponse: + """ + Transform the Clarifai response to a standard ModelResponse. + Since Clarifai is OpenAI-compatible, we use OpenAI response transformation. + """ + ## Logging logging_obj.post_call( input=messages, api_key=api_key, original_response=raw_response.text, additional_args={"complete_input_dict": request_data}, ) - ## RESPONSE OBJECT + ## Reponse try: completion_response = raw_response.json() - except httpx.HTTPStatusError as e: - raise ClarifaiError( - message=str(e), + except Exception as e: + raise OpenAIError( status_code=raw_response.status_code, - ) - except Exception as e: - raise ClarifaiError( - message=str(e), - status_code=422, - ) - # print(completion_response) - try: - choices_list = [] - for idx, item in enumerate(completion_response["outputs"]): - if len(item["data"]["text"]["raw"]) > 0: - message_obj = Message(content=item["data"]["text"]["raw"]) - else: - message_obj = Message(content=None) - choice_obj = Choices( - finish_reason="stop", - index=idx + 1, # check - message=message_obj, - ) - choices_list.append(choice_obj) - model_response.choices = choices_list # type: ignore + message=f"Failed to parse Clarifai response: {str(e)}", + headers=raw_response.headers, + ) from e + + response = ModelResponse(**completion_response) + + if response.model is not None: + response.model = "clarifai/" + model - except Exception as e: - raise ClarifaiError( - message=str(e), - status_code=422, - ) + return response - # Calculate Usage - prompt_tokens = token_counter(model=model, messages=messages) - completion_tokens = len( - encoding.encode(model_response["choices"][0]["message"].get("content")) - ) - model_response.model = model - setattr( - model_response, - "usage", - Usage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ), - ) - return model_response - - def get_model_response_iterator( - self, - streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse], - sync_stream: bool, - json_mode: Optional[bool] = False, - ) -> Any: - return ClarifaiModelResponseIterator( - model_response=streaming_response, - json_mode=json_mode, - ) - - -class ClarifaiModelResponseIterator(FakeStreamResponseIterator): - def __init__( - self, - model_response: Union[Iterator[str], AsyncIterator[str], ModelResponse], - json_mode: Optional[bool] = False, - ): - super().__init__( - model_response=model_response, - json_mode=json_mode, - ) - - def chunk_parser(self, chunk: dict) -> GenericStreamingChunk: - try: - text = "" - tool_use: Optional[ChatCompletionToolCallChunk] = None - is_finished = False - finish_reason = "" - usage: Optional[ChatCompletionUsageBlock] = None - provider_specific_fields = None - - text = ( - chunk.get("outputs", "")[0] - .get("data", "") - .get("text", "") - .get("raw", "") - ) - - index: int = 0 - - return GenericStreamingChunk( - text=text, - tool_use=tool_use, - is_finished=is_finished, - finish_reason=finish_reason, - usage=usage, - index=index, - provider_specific_fields=provider_specific_fields, - ) - except json.JSONDecodeError: - raise ValueError(f"Failed to decode JSON from chunk: {chunk}") + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + """ + Get the appropriate error class for Clarifai errors. + Since Clarifai is OpenAI-compatible, we use OpenAI error handling. + """ + return OpenAIError( + status_code=status_code, + message=error_message, + headers=headers, + ) \ No newline at end of file diff --git a/litellm/llms/clarifai/common_utils.py b/litellm/llms/clarifai/common_utils.py deleted file mode 100644 index 611d2ccf30b..00000000000 --- a/litellm/llms/clarifai/common_utils.py +++ /dev/null @@ -1,6 +0,0 @@ -from litellm.llms.base_llm.chat.transformation import BaseLLMException - - -class ClarifaiError(BaseLLMException): - def __init__(self, status_code: int, message: str): - super().__init__(status_code=status_code, message=message) diff --git a/litellm/llms/cohere/chat/v2_transformation.py b/litellm/llms/cohere/chat/v2_transformation.py index 76948e7f8b9..8f6dde1967c 100644 --- a/litellm/llms/cohere/chat/v2_transformation.py +++ b/litellm/llms/cohere/chat/v2_transformation.py @@ -4,14 +4,19 @@ from typing import TYPE_CHECKING, Any, AsyncIterator, Iterator, List, Optional, import httpx import litellm -from litellm.litellm_core_utils.prompt_templates.factory import cohere_messages_pt_v2 -from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.cohere import CohereV2ChatResponse -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionToolCallChunk, + ChatCompletionAnnotation, + ChatCompletionAnnotationURLCitation, +) +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.utils import ModelResponse, Usage from ..common_utils import CohereError -from ..common_utils import ModelResponseIterator as CohereModelResponseIterator +from ..common_utils import CohereV2ModelResponseIterator from ..common_utils import validate_environment as cohere_validate_environment if TYPE_CHECKING: @@ -22,7 +27,7 @@ else: LiteLLMLoggingObj = Any -class CohereV2ChatConfig(BaseConfig): +class CohereV2ChatConfig(OpenAIGPTConfig): """ Configuration class for Cohere's API interface. @@ -164,32 +169,12 @@ class CohereV2ChatConfig(BaseConfig): litellm_params: dict, headers: dict, ) -> dict: - ## Load Config - for k, v in litellm.CohereChatConfig.get_config().items(): - if ( - k not in optional_params - ): # completion(top_k=3) > cohere_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - most_recent_message, chat_history = cohere_messages_pt_v2( - messages=messages, model=model, llm_provider="cohere_chat" - ) - - ## Handle Tool Calling - if "tools" in optional_params: - _is_function_call = True - cohere_tools = self._construct_cohere_tool(tools=optional_params["tools"]) - optional_params["tools"] = cohere_tools - if isinstance(most_recent_message, dict): - optional_params["tool_results"] = [most_recent_message] - elif isinstance(most_recent_message, str): - optional_params["message"] = most_recent_message - - ## check if chat history message is 'user' and 'tool_results' is given -> force_single_step=True, else cohere api fails - if len(chat_history) > 0 and chat_history[-1]["role"] == "USER": - optional_params["force_single_step"] = True - - return optional_params + """ + Cohere v2 chat api is in openai format, so we can use the openai transform request function to transform the request. + """ + data = super().transform_request(model, messages, optional_params, litellm_params, headers) + + return data def transform_response( self, @@ -227,9 +212,15 @@ class CohereV2ChatConfig(BaseConfig): ] ) - ## ADD CITATIONS - if "citations" in cohere_v2_chat_response: - setattr(model_response, "citations", cohere_v2_chat_response["citations"]) + ## ADD CITATIONS AS ANNOTATIONS + annotations: Optional[List[ChatCompletionAnnotation]] = None + citations = None + + if "message" in cohere_v2_chat_response and "citations" in cohere_v2_chat_response["message"]: + citations = cohere_v2_chat_response["message"]["citations"] + + if citations: + annotations = self._translate_citations_to_openai_annotations(citations) ## Tool calling response cohere_tools_response = cohere_v2_chat_response["message"].get("tool_calls", []) @@ -245,8 +236,13 @@ class CohereV2ChatConfig(BaseConfig): _message = litellm.Message( tool_calls=tool_calls, content=None, + annotations=annotations, ) model_response.choices[0].message = _message # type: ignore + else: + if annotations: + current_message = model_response.choices[0].message # type: ignore + current_message.annotations = annotations ## CALCULATING USAGE - use cohere `billed_units` for returning usage token_usage = cohere_v2_chat_response["usage"].get("tokens", {}) @@ -263,94 +259,99 @@ class CohereV2ChatConfig(BaseConfig): setattr(model_response, "usage", usage) return model_response - def _construct_cohere_tool( - self, - tools: Optional[list] = None, - ): - if tools is None: - tools = [] - cohere_tools = [] - for tool in tools: - cohere_tool = self._translate_openai_tool_to_cohere(tool) - cohere_tools.append(cohere_tool) - return cohere_tools - - def _translate_openai_tool_to_cohere( - self, - openai_tool: dict, - ): - # cohere tools look like this - """ - { - "name": "query_daily_sales_report", - "description": "Connects to a database to retrieve overall sales volumes and sales information for a given day.", - "parameter_definitions": { - "day": { - "description": "Retrieves sales data for this day, formatted as YYYY-MM-DD.", - "type": "str", - "required": True - } - } - } - """ - - # OpenAI tools look like this - """ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - }, - } - """ - cohere_tool = { - "name": openai_tool["function"]["name"], - "description": openai_tool["function"]["description"], - "parameter_definitions": {}, - } - - for param_name, param_def in openai_tool["function"]["parameters"][ - "properties" - ].items(): - required_params = ( - openai_tool.get("function", {}) - .get("parameters", {}) - .get("required", []) - ) - cohere_param_def = { - "description": param_def.get("description", ""), - "type": param_def.get("type", ""), - "required": param_name in required_params, - } - cohere_tool["parameter_definitions"][param_name] = cohere_param_def - - return cohere_tool - def get_model_response_iterator( self, streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse], sync_stream: bool, json_mode: Optional[bool] = False, ): - return CohereModelResponseIterator( + return CohereV2ModelResponseIterator( streaming_response=streaming_response, sync_stream=sync_stream, json_mode=json_mode, ) + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for Cohere v2 chat completion. + The api_base should already include the full path. + """ + if api_base is None: + raise ValueError("api_base is required") + return api_base + def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] ) -> BaseLLMException: return CohereError(status_code=status_code, message=error_message) + + def _translate_citations_to_openai_annotations(self, citations: List[dict]) -> List[ChatCompletionAnnotation]: + """ + Transform Cohere citations to OpenAI annotations format. + + Creates separate annotations for each source in a citation, allowing multiple + annotations with the same start/end index if they reference different sources. + + Args: + citations: List of Cohere citation objects with format: + { + "start": int, + "end": int, + "text": str, + "sources": [ + { + "type": "document", + "document": { + "title": str, + "snippet": str, + ... + }, + "id": str + } + ] + } + + Returns: + List of OpenAI ChatCompletionAnnotation objects (one per source) + """ + annotations: List[ChatCompletionAnnotation] = [] + + for citation in citations: + start_index = citation.get("start", 0) + end_index = citation.get("end", 0) + + # Extract source information - loop through all sources + sources = citation.get("sources", []) + if not sources: + continue + + # Create an annotation for each source + for source in sources: + if source.get("type") == "document" and "document" in source: + document = source["document"] + title = document.get("title", "") + url = source.get("url") or f"source:{source.get('id', 'unknown')}" + + url_citation: ChatCompletionAnnotationURLCitation = { + "start_index": start_index, + "end_index": end_index, + "title": title, + "url": url, + } + + annotation: ChatCompletionAnnotation = { + "type": "url_citation", + "url_citation": url_citation, + } + + annotations.append(annotation) + + return annotations \ No newline at end of file diff --git a/litellm/llms/cohere/common_utils.py b/litellm/llms/cohere/common_utils.py index d194d9556b6..333916fffa3 100644 --- a/litellm/llms/cohere/common_utils.py +++ b/litellm/llms/cohere/common_utils.py @@ -1,12 +1,14 @@ import json -from typing import List, Optional +from typing import List, Optional, Literal, Tuple +from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ( ChatCompletionToolCallChunk, ChatCompletionUsageBlock, GenericStreamingChunk, + ProviderSpecificModelInfo, ) @@ -15,6 +17,74 @@ class CohereError(BaseLLMException): super().__init__(status_code=status_code, message=message) +class CohereModelInfo(BaseLLMModelInfo): + def get_provider_info( + self, + model: str, + ) -> Optional[ProviderSpecificModelInfo]: + """ + Default values all models of this provider support. + """ + return None + + def get_models( + self, api_key: Optional[str] = None, api_base: Optional[str] = None + ) -> List[str]: + """ + Returns a list of models supported by this provider. + """ + return [] + + @staticmethod + def get_api_key(api_key: Optional[str] = None) -> Optional[str]: + return api_key + + @staticmethod + def get_api_base( + api_base: Optional[str] = None, + ) -> Optional[str]: + return api_base + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + return {} + + @staticmethod + def get_base_model(model: str) -> Optional[str]: + """ + Returns the base model name from the given model name. + + Some providers like bedrock - can receive model=`invoke/anthropic.claude-3-opus-20240229-v1:0` or `converse/anthropic.claude-3-opus-20240229-v1:0` + This function will return `anthropic.claude-3-opus-20240229-v1:0` + """ + pass + + @staticmethod + def get_cohere_route(model: str) -> Literal["v1", "v2"]: + """ + Get the Cohere route for the given model. + + Args: + model: The model name (e.g., "cohere_chat/v2/command-r-plus", "command-r-plus") + + Returns: + "v2" for standard Cohere v2 API (default), "v1" for Cohere v1 API + """ + # Check for explicit v1 route + if "v1/" in model: + return "v1" + + # Default to v2 for all other cases + return "v2" + def validate_environment( headers: dict, model: str, @@ -145,3 +215,197 @@ class ModelResponseIterator: raise StopAsyncIteration except ValueError as e: raise RuntimeError(f"Error parsing chunk: {e},\nReceived chunk: {chunk}") + +class CohereV2ModelResponseIterator: + """V2-specific response iterator for Cohere streaming""" + + def __init__( + self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False + ): + self.streaming_response = streaming_response + self.response_iterator = self.streaming_response + self.content_blocks: List = [] + self.tool_index = -1 + self.json_mode = json_mode + + def _parse_content_delta(self, chunk: dict) -> str: + """Parse content-delta chunks to extract text.""" + delta = chunk.get("delta", {}) + message = delta.get("message", {}) + content = message.get("content", {}) + if isinstance(content, dict) and "text" in content: + return content["text"] + elif isinstance(content, str): + return content + return "" + + def _parse_tool_call_delta(self, chunk: dict) -> Optional[ChatCompletionToolCallChunk]: + """Parse tool-call-delta chunks to extract tool calls.""" + delta = chunk.get("delta", {}) + tool_calls = delta.get("tool_calls", []) + if tool_calls: + return { + "id": tool_calls[0].get("id", ""), + "type": "function", + "function": { + "name": tool_calls[0].get("name", ""), + "arguments": tool_calls[0].get("arguments", "") + } + } # type: ignore + return None + + def _parse_tool_plan_delta(self, chunk: dict) -> Optional[dict]: + """Parse tool-plan-delta events to extract tool plan.""" + data = chunk.get("data", {}) + delta = data.get("delta", {}) + message = delta.get("message", {}) + tool_plan = message.get("tool_plan", "") + if tool_plan: + return {"tool_plan": tool_plan} + return None + + def _parse_citation_start(self, chunk: dict) -> Optional[dict]: + """Parse citation-start events to extract citations.""" + data = chunk.get("data", {}) + delta = data.get("delta", {}) + message = delta.get("message", {}) + citations = message.get("citations", {}) + if citations: + citation_data = { + "start": citations.get("start", 0), + "end": citations.get("end", 0), + "text": citations.get("text", ""), + "sources": citations.get("sources", []), + "type": citations.get("type", "TEXT_CONTENT") + } + return {"citations": [citation_data]} + return None + + def _parse_message_end(self, chunk: dict) -> Tuple[bool, str, Optional[ChatCompletionUsageBlock]]: + """Parse message-end events to extract finish info and usage.""" + data = chunk.get("data", {}) + delta = data.get("delta", {}) + is_finished = True + finish_reason = delta.get("finish_reason", "stop") + + usage = None + usage_data = delta.get("usage", {}) + if usage_data: + tokens_data = usage_data.get("tokens", {}) + usage = ChatCompletionUsageBlock( + prompt_tokens=tokens_data.get("input_tokens", 0), + completion_tokens=tokens_data.get("output_tokens", 0), + total_tokens=tokens_data.get("input_tokens", 0) + tokens_data.get("output_tokens", 0) + ) + + return is_finished, finish_reason, usage + + def chunk_parser(self, chunk: dict) -> GenericStreamingChunk: + """ + Parse Cohere v2 streaming chunks. + + v2 format: + - Content: chunk.type == "content-delta" -> chunk.delta.message.content.text + - Tool calls: chunk.type == "tool-call-delta" -> chunk.delta.tool_calls + - Tool plan: chunk.event == "tool-plan-delta" -> chunk.data.delta.message.tool_plan + - Citations: chunk.event == "citation-start" -> chunk.data.delta.message.citations + - Finish: chunk.event == "message-end" -> chunk.data.delta.finish_reason + """ + try: + text = "" + tool_use: Optional[ChatCompletionToolCallChunk] = None + is_finished = False + finish_reason = "" + usage: Optional[ChatCompletionUsageBlock] = None + provider_specific_fields = None + + index = int(chunk.get("index", 0)) + chunk_type = chunk.get("type", "") + event_type = chunk.get("event", "") + + # Handle different chunk types + if chunk_type == "content-delta": + text = self._parse_content_delta(chunk) + elif chunk_type == "tool-call-delta": + tool_use = self._parse_tool_call_delta(chunk) + elif event_type == "tool-plan-delta": + provider_specific_fields = self._parse_tool_plan_delta(chunk) + elif event_type == "citation-start": + provider_specific_fields = self._parse_citation_start(chunk) + elif event_type == "message-end": + is_finished, finish_reason, usage = self._parse_message_end(chunk) + + # Handle citations in any chunk type (fallback) + if "citations" in chunk: + if provider_specific_fields is None: + provider_specific_fields = {} + provider_specific_fields["citations"] = chunk["citations"] + + return GenericStreamingChunk( + text=text, + tool_use=tool_use, + is_finished=is_finished, + finish_reason=finish_reason, + usage=usage, + index=index, + provider_specific_fields=provider_specific_fields, + ) + + except Exception as e: + raise ValueError(f"Failed to parse v2 chunk: {e}, chunk: {chunk}") + + # Sync iterator + def __iter__(self): + return self + + def __next__(self): + try: + chunk = self.response_iterator.__next__() + except StopIteration: + raise StopIteration + except ValueError as e: + raise RuntimeError(f"Error receiving chunk from stream: {e}") + + try: + return self.convert_str_chunk_to_generic_chunk(chunk=chunk) + except StopIteration: + raise StopIteration + except ValueError as e: + raise RuntimeError(f"Error parsing chunk: {e},\nReceived chunk: {chunk}") + + def convert_str_chunk_to_generic_chunk(self, chunk: str) -> GenericStreamingChunk: + """ + Convert a string chunk to a GenericStreamingChunk for v2 + + Note: This is used for Cohere v2 pass through streaming logging + """ + str_line = chunk + if isinstance(chunk, bytes): # Handle binary data + str_line = chunk.decode("utf-8") # Convert bytes to string + index = str_line.find("data:") + if index != -1: + str_line = str_line[index:] + + data_json = json.loads(str_line) + return self.chunk_parser(chunk=data_json) + + # Async iterator + def __aiter__(self): + self.async_response_iterator = self.streaming_response.__aiter__() + return self + + async def __anext__(self): + try: + chunk = await self.async_response_iterator.__anext__() + except StopAsyncIteration: + raise StopAsyncIteration + except ValueError as e: + raise RuntimeError(f"Error receiving chunk from stream: {e}") + + try: + return self.convert_str_chunk_to_generic_chunk(chunk=chunk) + except StopAsyncIteration: + raise StopAsyncIteration + except ValueError as e: + raise RuntimeError(f"Error parsing chunk: {e},\nReceived chunk: {chunk}") + diff --git a/litellm/llms/cohere/embed/v1_transformation.py b/litellm/llms/cohere/embed/v1_transformation.py index e55899a4afa..1a4bc393e84 100644 --- a/litellm/llms/cohere/embed/v1_transformation.py +++ b/litellm/llms/cohere/embed/v1_transformation.py @@ -123,10 +123,23 @@ class CohereEmbeddingConfig: """ embeddings = response_json["embeddings"] output_data = [] - for idx, embedding in enumerate(embeddings): - output_data.append( - {"object": "embedding", "index": idx, "embedding": embedding} - ) + is_embeddings_by_type = response_json.get("response_type") == "embeddings_by_type" + if is_embeddings_by_type: + for embedding_type in embeddings: + for idx, embedding in enumerate(embeddings[embedding_type]): + output_data.append( + { + "object": "embedding", + "index": idx, + "embedding": embedding, + "type": embedding_type, + } + ) + else: + for idx, embedding in enumerate(embeddings): + output_data.append( + {"object": "embedding", "index": idx, "embedding": embedding} + ) model_response.object = "list" model_response.data = output_data model_response.model = model diff --git a/litellm/llms/cohere/rerank/guardrail_translation/README.md b/litellm/llms/cohere/rerank/guardrail_translation/README.md new file mode 100644 index 00000000000..e77e5a74dd0 --- /dev/null +++ b/litellm/llms/cohere/rerank/guardrail_translation/README.md @@ -0,0 +1,229 @@ +# Cohere Rerank Guardrail Translation Handler + +Handler for processing the rerank endpoint (`/v1/rerank`) with guardrails. + +## Overview + +This handler processes rerank requests by: +1. Extracting the query text from the request +2. Applying guardrails to the query +3. Updating the request with the guardrailed query +4. Returning the output unchanged (rankings are not text) + +Note: Documents are not processed by guardrails as they represent the corpus +being searched, not user input. Only the query is guardrailed. + +## Data Format + +### Input Format + +**With String Documents:** +```json +{ + "model": "rerank-english-v3.0", + "query": "What is the capital of France?", + "documents": [ + "Paris is the capital of France.", + "Berlin is the capital of Germany.", + "Madrid is the capital of Spain." + ], + "top_n": 2 +} +``` + +**With Dict Documents:** +```json +{ + "model": "rerank-english-v3.0", + "query": "What is the capital of France?", + "documents": [ + {"text": "Paris is the capital of France.", "id": "doc1"}, + {"text": "Berlin is the capital of Germany.", "id": "doc2"}, + {"text": "Madrid is the capital of Spain.", "id": "doc3"} + ], + "top_n": 2 +} +``` + +### Output Format + +```json +{ + "id": "rerank-abc123", + "results": [ + {"index": 0, "relevance_score": 0.98}, + {"index": 2, "relevance_score": 0.12} + ], + "meta": { + "billed_units": {"search_units": 1} + } +} +``` + +## Usage + +The handler is automatically discovered and applied when guardrails are used with the rerank endpoint. + +### Example: Using Guardrails with Rerank + +```bash +curl -X POST 'http://localhost:4000/v1/rerank' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer your-api-key' \ +-d '{ + "model": "rerank-english-v3.0", + "query": "What is machine learning?", + "documents": [ + "Machine learning is a subset of AI.", + "Deep learning uses neural networks.", + "Python is a programming language." + ], + "guardrails": ["content_filter"], + "top_n": 2 +}' +``` + +The guardrail will be applied to the query only (not the documents). + +### Example: PII Masking in Query + +```bash +curl -X POST 'http://localhost:4000/v1/rerank' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer your-api-key' \ +-d '{ + "model": "rerank-english-v3.0", + "query": "Find documents about John Doe from john@example.com", + "documents": [ + "Document 1 content here.", + "Document 2 content here.", + "Document 3 content here." + ], + "guardrails": ["mask_pii"], + "top_n": 3 +}' +``` + +The query will be masked to: "Find documents about [NAME_REDACTED] from [EMAIL_REDACTED]" + +### Example: Mixed Document Types + +```bash +curl -X POST 'http://localhost:4000/v1/rerank' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer your-api-key' \ +-d '{ + "model": "rerank-english-v3.0", + "query": "Technical documentation", + "documents": [ + {"text": "This is document 1", "metadata": {"source": "wiki"}}, + {"text": "This is document 2", "metadata": {"source": "docs"}}, + "This is document 3 as a plain string" + ], + "guardrails": ["content_moderation"] +}' +``` + +## Implementation Details + +### Input Processing + +- **Query Field**: `query` (string) + - Processing: Apply guardrail to query text + - Result: Updated query + +- **Documents Field**: `documents` (list) + - Processing: Not processed (corpus being searched, not user input) + - Result: Unchanged + +### Output Processing + +- **Processing**: Not applicable (output contains relevance scores, not text) +- **Result**: Response returned unchanged + +## Use Cases + +1. **PII Protection**: Remove PII from queries before reranking +2. **Content Filtering**: Filter inappropriate content from search queries +3. **Compliance**: Ensure queries meet requirements +4. **Data Sanitization**: Clean up query text before semantic search operations + +## Extension + +Override these methods to customize behavior: + +- `process_input_messages()`: Customize how query is processed +- `process_output_response()`: Currently a no-op, but can be overridden if needed + +## Supported Call Types + +- `CallTypes.rerank` - Synchronous rerank +- `CallTypes.arerank` - Asynchronous rerank + +## Notes + +- Only the query is processed by guardrails +- Documents are not processed (they represent the corpus, not user input) +- Output processing is a no-op since rankings don't contain text +- Both sync and async call types use the same handler +- Works with all rerank providers (Cohere, Together AI, etc.) + +## Common Patterns + +### PII Masking in Search + +```python +import litellm + +response = litellm.rerank( + model="rerank-english-v3.0", + query="Find info about john@example.com", + documents=[ + "Document 1 content.", + "Document 2 content.", + "Document 3 content." + ], + guardrails=["mask_pii"], + top_n=2 +) + +# Query will have PII masked +# query becomes: "Find info about [EMAIL_REDACTED]" +print(response.results) +``` + +### Content Filtering + +```python +import litellm + +response = litellm.rerank( + model="rerank-english-v3.0", + query="Search query here", + documents=[ + {"text": "Document 1 content", "id": "doc1"}, + {"text": "Document 2 content", "id": "doc2"}, + ], + guardrails=["content_filter"], +) +``` + +### Async Rerank with Guardrails + +```python +import litellm +import asyncio + +async def rerank_with_guardrails(): + response = await litellm.arerank( + model="rerank-english-v3.0", + query="Technical query", + documents=["Doc 1", "Doc 2", "Doc 3"], + guardrails=["sanitize"], + top_n=2 + ) + return response + +result = asyncio.run(rerank_with_guardrails()) +``` + diff --git a/litellm/llms/cohere/rerank/guardrail_translation/__init__.py b/litellm/llms/cohere/rerank/guardrail_translation/__init__.py new file mode 100644 index 00000000000..70b580facf5 --- /dev/null +++ b/litellm/llms/cohere/rerank/guardrail_translation/__init__.py @@ -0,0 +1,11 @@ +"""Cohere Rerank handler for Unified Guardrails.""" + +from litellm.llms.cohere.rerank.guardrail_translation.handler import CohereRerankHandler +from litellm.types.utils import CallTypes + +guardrail_translation_mappings = { + CallTypes.rerank: CohereRerankHandler, + CallTypes.arerank: CohereRerankHandler, +} + +__all__ = ["guardrail_translation_mappings", "CohereRerankHandler"] diff --git a/litellm/llms/cohere/rerank/guardrail_translation/handler.py b/litellm/llms/cohere/rerank/guardrail_translation/handler.py new file mode 100644 index 00000000000..0c5e50dc41e --- /dev/null +++ b/litellm/llms/cohere/rerank/guardrail_translation/handler.py @@ -0,0 +1,90 @@ +""" +Cohere Rerank Handler for Unified Guardrails + +This module provides guardrail translation support for the rerank endpoint. +The handler processes only the 'query' parameter for guardrails. +""" + +from typing import TYPE_CHECKING, Any + +from litellm._logging import verbose_proxy_logger +from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation + +if TYPE_CHECKING: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.rerank import RerankResponse + + +class CohereRerankHandler(BaseTranslation): + """ + Handler for processing rerank requests with guardrails. + + This class provides methods to: + 1. Process input query (pre-call hook) + 2. Process output response (post-call hook) - not applicable for rerank + + The handler specifically processes: + - The 'query' parameter (string) + + Note: Documents are not processed by guardrails as they are the corpus + being searched, not user input. + """ + + async def process_input_messages( + self, + data: dict, + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process input query by applying guardrails. + + Args: + data: Request data dictionary containing 'query' + guardrail_to_apply: The guardrail instance to apply + + Returns: + Modified data with guardrails applied to query only + """ + # Process query only + query = data.get("query") + if query is not None and isinstance(query, str): + guardrailed_query = await guardrail_to_apply.apply_guardrail(text=query) + data["query"] = guardrailed_query + + verbose_proxy_logger.debug( + "Rerank: Applied guardrail to query. " + "Original length: %d, New length: %d", + len(query), + len(guardrailed_query), + ) + else: + verbose_proxy_logger.debug( + "Rerank: No query to process or query is not a string" + ) + + return data + + async def process_output_response( + self, + response: "RerankResponse", + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process output response - not applicable for rerank. + + Rerank responses contain relevance scores and indices, not text, + so there's nothing to apply guardrails to. This method returns + the response unchanged. + + Args: + response: Rerank response object with rankings + guardrail_to_apply: The guardrail instance (unused) + + Returns: + Unmodified response (rankings don't need text guardrails) + """ + verbose_proxy_logger.debug( + "Rerank: Output processing not applicable " + "(output contains relevance scores, not text)" + ) + return response diff --git a/litellm/llms/cohere/rerank/transformation.py b/litellm/llms/cohere/rerank/transformation.py index f9c979712da..d085cb13c44 100644 --- a/litellm/llms/cohere/rerank/transformation.py +++ b/litellm/llms/cohere/rerank/transformation.py @@ -20,7 +20,12 @@ class CohereRerankConfig(BaseRerankConfig): def __init__(self) -> None: pass - def get_complete_url(self, api_base: Optional[str], model: str) -> str: + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: Optional[dict] = None, + ) -> str: if api_base: # Remove trailing slashes and ensure clean base URL api_base = api_base.rstrip("/") @@ -72,6 +77,7 @@ class CohereRerankConfig(BaseRerankConfig): headers: dict, model: str, api_key: Optional[str] = None, + optional_params: Optional[dict] = None, ) -> dict: if api_key is None: api_key = ( diff --git a/litellm/llms/cohere/rerank_v2/transformation.py b/litellm/llms/cohere/rerank_v2/transformation.py index eb551a8a949..01309d937f9 100644 --- a/litellm/llms/cohere/rerank_v2/transformation.py +++ b/litellm/llms/cohere/rerank_v2/transformation.py @@ -12,7 +12,12 @@ class CohereRerankV2Config(CohereRerankConfig): def __init__(self) -> None: pass - def get_complete_url(self, api_base: Optional[str], model: str) -> str: + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: Optional[dict] = None, + ) -> str: if api_base: # Remove trailing slashes and ensure clean base URL api_base = api_base.rstrip("/") diff --git a/litellm/llms/cometapi/embed/__init__.py b/litellm/llms/cometapi/embed/__init__.py new file mode 100644 index 00000000000..a36647f46c6 --- /dev/null +++ b/litellm/llms/cometapi/embed/__init__.py @@ -0,0 +1,3 @@ +from .transformation import CometAPIEmbeddingConfig + +__all__ = ["CometAPIEmbeddingConfig"] diff --git a/litellm/llms/cometapi/embed/transformation.py b/litellm/llms/cometapi/embed/transformation.py new file mode 100644 index 00000000000..5cfd1253149 --- /dev/null +++ b/litellm/llms/cometapi/embed/transformation.py @@ -0,0 +1,157 @@ +""" +CometAPI Embedding API support - OpenAI compatible +""" + +from typing import List, Optional, Union + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues +from litellm.types.utils import EmbeddingResponse, Usage + +from ..common_utils import CometAPIException + + +class CometAPIEmbeddingConfig(BaseEmbeddingConfig): + """ + Configuration class for CometAPI Embedding API. + + Since CometAPI is OpenAI-compatible, this class provides OpenAI-standard + embedding functionality with CometAPI-specific authentication and endpoints. + """ + + def __init__(self) -> None: + pass + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for the CometAPI embedding endpoint. + """ + api_base = ( + "https://api.cometapi.com/v1" if api_base is None else api_base.rstrip("/") + ) + complete_url = f"{api_base}/embeddings" + return complete_url + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate and set up authentication headers for CometAPI. + """ + if api_key is None: + api_key = get_secret_str("COMETAPI_KEY") + + default_headers = { + "Authorization": f"Bearer {api_key}", + "accept": "application/json", + "Content-Type": "application/json", + } + + if "Authorization" in headers: + default_headers["Authorization"] = headers["Authorization"] + + return {**default_headers, **headers} + + def get_supported_openai_params(self, model: str) -> List[str]: + """ + Get the supported OpenAI parameters for embedding requests. + CometAPI supports standard OpenAI embedding parameters. + """ + return [ + "dimensions", + "encoding_format", + "user", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to CometAPI format. + """ + supported_openai_params = self.get_supported_openai_params(model) + for param, value in non_default_params.items(): + if param in supported_openai_params: + optional_params[param] = value + return optional_params + + def transform_embedding_request( + self, + model: str, + input: AllEmbeddingInputValues, + optional_params: dict, + headers: dict, + ) -> dict: + """ + Transform the embedding request into CometAPI format. + """ + return {"input": input, "model": model, **optional_params} + + def transform_embedding_response( + self, + model: str, + raw_response: httpx.Response, + model_response: EmbeddingResponse, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + request_data: dict, + optional_params: dict, + litellm_params: dict, + ) -> EmbeddingResponse: + """ + Transform CometAPI response into standard EmbeddingResponse format. + """ + try: + raw_response_json = raw_response.json() + except Exception: + raise CometAPIException( + message=raw_response.text, + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + model_response.model = raw_response_json.get("model") + model_response.data = raw_response_json.get("data") + model_response.object = raw_response_json.get("object") + + usage = Usage( + prompt_tokens=raw_response_json.get("usage", {}).get("prompt_tokens", 0), + total_tokens=raw_response_json.get("usage", {}).get("total_tokens", 0), + ) + + model_response.usage = usage + return model_response + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + """ + Get the appropriate error class for CometAPI exceptions. + """ + return CometAPIException( + message=error_message, status_code=status_code, headers=headers + ) diff --git a/litellm/llms/cometapi/image_generation/__init__.py b/litellm/llms/cometapi/image_generation/__init__.py new file mode 100644 index 00000000000..8d7630f2b30 --- /dev/null +++ b/litellm/llms/cometapi/image_generation/__init__.py @@ -0,0 +1,13 @@ +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) + +from .transformation import CometAPIImageGenerationConfig + +__all__ = [ + "CometAPIImageGenerationConfig", +] + + +def get_cometapi_image_generation_config(model: str) -> BaseImageGenerationConfig: + return CometAPIImageGenerationConfig() diff --git a/litellm/llms/cometapi/image_generation/cost_calculator.py b/litellm/llms/cometapi/image_generation/cost_calculator.py new file mode 100644 index 00000000000..b10c9d09087 --- /dev/null +++ b/litellm/llms/cometapi/image_generation/cost_calculator.py @@ -0,0 +1,25 @@ +from typing import Any + +import litellm +from litellm.types.utils import ImageResponse + + +def cost_calculator( + model: str, + image_response: Any, +) -> float: + """ + CometAPI image generation cost calculator + """ + _model_info = litellm.get_model_info( + model=model, + custom_llm_provider=litellm.LlmProviders.COMETAPI.value, + ) + output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0 + num_images: int = 0 + if isinstance(image_response, ImageResponse): + if image_response.data: + num_images = len(image_response.data) + return output_cost_per_image * num_images + else: + raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") diff --git a/litellm/llms/cometapi/image_generation/transformation.py b/litellm/llms/cometapi/image_generation/transformation.py new file mode 100644 index 00000000000..bf1ca9ddde6 --- /dev/null +++ b/litellm/llms/cometapi/image_generation/transformation.py @@ -0,0 +1,170 @@ +from typing import TYPE_CHECKING, Any, List, Optional + +import httpx + +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIImageGenerationOptionalParams, +) +from litellm.types.utils import ImageObject, ImageResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class CometAPIImageGenerationConfig(BaseImageGenerationConfig): + DEFAULT_BASE_URL: str = "https://api.cometapi.com" + IMAGE_GENERATION_ENDPOINT: str = "v1/images/generations" + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + """ + https://api.cometapi.com/v1/images/generations + """ + return [ + "n", + "quality", + "response_format", + "size", + "style", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + + for k in non_default_params.keys(): + if k not in optional_params.keys(): + if k in supported_params: + # CometAPI uses OpenAI-compatible parameters, so we can pass them directly + optional_params[k] = non_default_params[k] + elif drop_params: + pass + else: + raise ValueError( + f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters." + ) + + return optional_params + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete url for the request + """ + complete_url: str = ( + api_base + or get_secret_str("COMETAPI_BASE_URL") + or get_secret_str("COMETAPI_API_BASE") + or self.DEFAULT_BASE_URL + ) + + complete_url = complete_url.rstrip("/") + complete_url = f"{complete_url}/{self.IMAGE_GENERATION_ENDPOINT}" + return complete_url + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + final_api_key: Optional[str] = ( + api_key or + get_secret_str("COMETAPI_KEY") or + get_secret_str("COMETAPI_API_KEY") + ) + if not final_api_key: + raise ValueError("COMETAPI_KEY or COMETAPI_API_KEY is not set") + + headers["Authorization"] = f"Bearer {final_api_key}" + headers["Content-Type"] = "application/json" + return headers + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the image generation request to the CometAPI image generation request body + + https://api.cometapi.com/v1/images/generations + """ + # CometAPI uses OpenAI-compatible format + request_body = { + "prompt": prompt, + "model": model, + **optional_params, + } + return request_body + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + """ + Transform the image generation response to the litellm image response + + https://api.cometapi.com/v1/images/generations + """ + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error transforming image generation response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + if not model_response.data: + model_response.data = [] + + # CometAPI returns OpenAI-compatible format + # Expected format: {"created": timestamp, "data": [{"url": "...", "b64_json": "..."}]} + if "data" in response_data: + for image_data in response_data["data"]: + image_obj = ImageObject( + b64_json=image_data.get("b64_json"), + url=image_data.get("url"), + ) + model_response.data.append(image_obj) + + return model_response diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index 50bbccd6a4b..6997afafd8d 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -18,6 +18,7 @@ from litellm.secret_managers.main import str_to_bool AIOHTTP_EXC_MAP: Dict = { # Order matters here, most specific exception first # Timeout related exceptions + asyncio.TimeoutError: httpx.TimeoutException, aiohttp.ServerTimeoutError: httpx.TimeoutException, aiohttp.ConnectionTimeoutError: httpx.ConnectTimeout, aiohttp.SocketTimeoutError: httpx.ReadTimeout, @@ -95,6 +96,15 @@ class AiohttpResponseStream(httpx.AsyncByteStream): # If the error is due to incomplete transfer encoding, we can still # return what we've received so far, similar to how httpx handles it return + except RuntimeError as e: + # Some providers (e.g., SSE streams) may close the connection + # causing aiohttp StreamReader to raise a generic RuntimeError + # with message "Connection closed.". Treat this as a graceful + # end-of-stream so downstream consumers don't error. + if "Connection closed" in str(e): + verbose_logger.debug("Upstream closed streaming connection; ending iterator gracefully") + return + raise except aiohttp.http_exceptions.TransferEncodingError as e: # Handle transfer encoding errors gracefully verbose_logger.debug(f"Transfer encoding error, but continuing: {e}") @@ -244,6 +254,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport): allow_redirects=False, auto_decompress=False, timeout=ClientTimeout( + total=timeout.get("read"), sock_connect=timeout.get("connect"), sock_read=timeout.get("read"), connect=timeout.get("pool"), diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index a3ad2c67272..37b4af306a1 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1,8 +1,9 @@ import asyncio import os import ssl +import sys import time -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional, Tuple, Union import certifi import httpx @@ -17,7 +18,7 @@ from litellm.constants import ( AIOHTTP_CONNECTOR_LIMIT, AIOHTTP_KEEPALIVE_TIMEOUT, AIOHTTP_TTL_DNS_CACHE, - DEFAULT_SSL_CIPHERS + DEFAULT_SSL_CIPHERS, ) from litellm.litellm_core_utils.logging_utils import track_llm_api_timing from litellm.types.llms.custom_http import * @@ -46,6 +47,46 @@ headers = { _DEFAULT_TIMEOUT = httpx.Timeout(timeout=5.0, connect=5.0) +def _prepare_request_data_and_content( + data: Optional[Union[dict, str, bytes]] = None, + content: Any = None, +) -> Tuple[Optional[Union[dict, Mapping]], Any]: + """ + Helper function to route data/content parameters correctly for httpx requests + + This prevents httpx DeprecationWarnings that cause memory leaks. + + Background: + - httpx shows a DeprecationWarning when you pass bytes/str to `data=` + - It wants you to use `content=` instead for bytes/str + - The warning itself leaks memory when triggered repeatedly + + Solution: + - Move bytes/str from `data=` to `content=` before calling build_request + - Keep dicts in `data=` (that's still the correct parameter for dicts) + + Args: + data: Request data (can be dict, str, or bytes) + content: Request content (raw bytes/str) + + Returns: + Tuple of (request_data, request_content) properly routed for httpx + """ + request_data = None + request_content = content + + if data is not None: + if isinstance(data, (bytes, str)): + # Bytes/strings belong in content= (only if not already provided) + if content is None: + request_content = data + else: + # dict/Mapping stays in data= parameter + request_data = data + + return request_data, request_content + + def get_ssl_configuration( ssl_verify: Optional[VerifyTypes] = None, ) -> Union[bool, str, ssl.SSLContext]: @@ -100,11 +141,11 @@ def get_ssl_configuration( if ssl_verify is not False: custom_ssl_context = ssl.create_default_context(cafile=cafile) - + # Optimize SSL handshake performance # Set minimum TLS version to 1.2 for better performance custom_ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2 - + # Configure cipher suites for optimal performance if ssl_security_level and isinstance(ssl_security_level, str): # User provided custom cipher configuration (e.g., via SSL_SECURITY_LEVEL env var) @@ -114,6 +155,28 @@ def get_ssl_configuration( # but falls back to widely compatible ones custom_ssl_context.set_ciphers(DEFAULT_SSL_CIPHERS) + # Configure ECDH curve for key exchange (e.g., to disable PQC and improve performance) + # Set SSL_ECDH_CURVE env var or litellm.ssl_ecdh_curve to 'X25519' to disable PQC + # Common valid curves: X25519, prime256v1, secp384r1, secp521r1 + ssl_ecdh_curve = os.getenv("SSL_ECDH_CURVE", litellm.ssl_ecdh_curve) + if ssl_ecdh_curve and isinstance(ssl_ecdh_curve, str): + try: + custom_ssl_context.set_ecdh_curve(ssl_ecdh_curve) + verbose_logger.debug(f"SSL ECDH curve set to: {ssl_ecdh_curve}") + except AttributeError: + verbose_logger.warning( + f"SSL ECDH curve configuration not supported. " + f"Python version: {sys.version.split()[0]}, OpenSSL version: {ssl.OPENSSL_VERSION}. " + f"Requested curve: {ssl_ecdh_curve}. Continuing with default curves." + ) + except ValueError as e: + # Invalid curve name + verbose_logger.warning( + f"Invalid SSL ECDH curve name: '{ssl_ecdh_curve}'. {e}. " + f"Common valid curves: X25519, prime256v1, secp384r1, secp521r1. " + f"Continuing with default curves (including PQC)." + ) + # Use our custom SSL context instead of the original ssl_verify value return custom_ssl_context @@ -278,17 +341,20 @@ class AsyncHTTPHandler: if timeout is None: timeout = self.timeout + # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) + request_data, request_content = _prepare_request_data_and_content(data, content) + req = self.client.build_request( "POST", url, - data=data, # type: ignore + data=request_data, json=json, params=params, headers=headers, timeout=timeout, files=files, - content=content, - ) + content=request_content, + ) response = await self.client.send(req, stream=stream) response.raise_for_status() return response @@ -341,19 +407,23 @@ class AsyncHTTPHandler: async def put( self, url: str, - data: Optional[Union[dict, str]] = None, # type: ignore + data: Optional[Union[dict, str, bytes]] = None, # type: ignore json: Optional[dict] = None, params: Optional[dict] = None, headers: Optional[dict] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, stream: bool = False, + content: Any = None, ): try: if timeout is None: timeout = self.timeout + # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) + request_data, request_content = _prepare_request_data_and_content(data, content) + req = self.client.build_request( - "PUT", url, data=data, json=json, params=params, headers=headers, timeout=timeout # type: ignore + "PUT", url, data=request_data, json=json, params=params, headers=headers, timeout=timeout, content=request_content # type: ignore ) response = await self.client.send(req) response.raise_for_status() @@ -401,19 +471,23 @@ class AsyncHTTPHandler: async def patch( self, url: str, - data: Optional[Union[dict, str]] = None, # type: ignore + data: Optional[Union[dict, str, bytes]] = None, # type: ignore json: Optional[dict] = None, params: Optional[dict] = None, headers: Optional[dict] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, stream: bool = False, + content: Any = None, ): try: if timeout is None: timeout = self.timeout + # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) + request_data, request_content = _prepare_request_data_and_content(data, content) + req = self.client.build_request( - "PATCH", url, data=data, json=json, params=params, headers=headers, timeout=timeout # type: ignore + "PATCH", url, data=request_data, json=json, params=params, headers=headers, timeout=timeout, content=request_content # type: ignore ) response = await self.client.send(req) response.raise_for_status() @@ -461,18 +535,23 @@ class AsyncHTTPHandler: async def delete( self, url: str, - data: Optional[Union[dict, str]] = None, # type: ignore + data: Optional[Union[dict, str, bytes]] = None, # type: ignore json: Optional[dict] = None, params: Optional[dict] = None, headers: Optional[dict] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, stream: bool = False, + content: Any = None, ): try: if timeout is None: timeout = self.timeout + + # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) + request_data, request_content = _prepare_request_data_and_content(data, content) + req = self.client.build_request( - "DELETE", url, data=data, json=json, params=params, headers=headers, timeout=timeout # type: ignore + "DELETE", url, data=request_data, json=json, params=params, headers=headers, timeout=timeout, content=request_content # type: ignore ) response = await self.client.send(req, stream=stream) response.raise_for_status() @@ -520,8 +599,11 @@ class AsyncHTTPHandler: Used for retrying connection client errors. """ + # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) + request_data, request_content = _prepare_request_data_and_content(data, content) + req = client.build_request( - "POST", url, data=data, json=json, params=params, headers=headers, content=content # type: ignore + "POST", url, data=request_data, json=json, params=params, headers=headers, content=request_content # type: ignore ) response = await client.send(req, stream=stream) response.raise_for_status() @@ -671,7 +753,7 @@ class AsyncHTTPHandler: keepalive_timeout=AIOHTTP_KEEPALIVE_TIMEOUT, ttl_dns_cache=AIOHTTP_TTL_DNS_CACHE, enable_cleanup_closed=True, - **connector_kwargs + **connector_kwargs, ), trust_env=trust_env, ), @@ -775,21 +857,24 @@ class HTTPHandler: logging_obj: Optional[LiteLLMLoggingObject] = None, ): try: + # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) + request_data, request_content = _prepare_request_data_and_content(data, content) + if timeout is not None: req = self.client.build_request( "POST", url, - data=data, # type: ignore + data=request_data, # type: ignore json=json, params=params, headers=headers, timeout=timeout, files=files, - content=content, # type: ignore + content=request_content, # type: ignore ) else: req = self.client.build_request( - "POST", url, data=data, json=json, params=params, headers=headers, files=files, content=content # type: ignore + "POST", url, data=request_data, json=json, params=params, headers=headers, files=files, content=request_content # type: ignore ) response = self.client.send(req, stream=stream) response.raise_for_status() @@ -817,21 +902,25 @@ class HTTPHandler: def patch( self, url: str, - data: Optional[Union[dict, str]] = None, + data: Optional[Union[dict, str, bytes]] = None, json: Optional[Union[dict, str]] = None, params: Optional[dict] = None, headers: Optional[dict] = None, stream: bool = False, timeout: Optional[Union[float, httpx.Timeout]] = None, + content: Any = None, ): try: + # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) + request_data, request_content = _prepare_request_data_and_content(data, content) + if timeout is not None: req = self.client.build_request( - "PATCH", url, data=data, json=json, params=params, headers=headers, timeout=timeout # type: ignore + "PATCH", url, data=request_data, json=json, params=params, headers=headers, timeout=timeout, content=request_content # type: ignore ) else: req = self.client.build_request( - "PATCH", url, data=data, json=json, params=params, headers=headers # type: ignore + "PATCH", url, data=request_data, json=json, params=params, headers=headers, content=request_content # type: ignore ) response = self.client.send(req, stream=stream) response.raise_for_status() @@ -860,21 +949,25 @@ class HTTPHandler: def put( self, url: str, - data: Optional[Union[dict, str]] = None, + data: Optional[Union[dict, str, bytes]] = None, json: Optional[Union[dict, str]] = None, params: Optional[dict] = None, headers: Optional[dict] = None, stream: bool = False, timeout: Optional[Union[float, httpx.Timeout]] = None, + content: Any = None, ): try: + # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) + request_data, request_content = _prepare_request_data_and_content(data, content) + if timeout is not None: req = self.client.build_request( - "PUT", url, data=data, json=json, params=params, headers=headers, timeout=timeout # type: ignore + "PUT", url, data=request_data, json=json, params=params, headers=headers, timeout=timeout, content=request_content # type: ignore ) else: req = self.client.build_request( - "PUT", url, data=data, json=json, params=params, headers=headers # type: ignore + "PUT", url, data=request_data, json=json, params=params, headers=headers, content=request_content # type: ignore ) response = self.client.send(req, stream=stream) return response @@ -890,21 +983,25 @@ class HTTPHandler: def delete( self, url: str, - data: Optional[Union[dict, str]] = None, # type: ignore + data: Optional[Union[dict, str, bytes]] = None, # type: ignore json: Optional[dict] = None, params: Optional[dict] = None, headers: Optional[dict] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, stream: bool = False, + content: Any = None, ): try: + # Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix) + request_data, request_content = _prepare_request_data_and_content(data, content) + if timeout is not None: req = self.client.build_request( - "DELETE", url, data=data, json=json, params=params, headers=headers, timeout=timeout # type: ignore + "DELETE", url, data=request_data, json=json, params=params, headers=headers, timeout=timeout, content=request_content # type: ignore ) else: req = self.client.build_request( - "DELETE", url, data=data, json=json, params=params, headers=headers # type: ignore + "DELETE", url, data=request_data, json=json, params=params, headers=headers, content=request_content # type: ignore ) response = self.client.send(req, stream=stream) response.raise_for_status() @@ -946,7 +1043,7 @@ class HTTPHandler: if litellm.force_ipv4: return HTTPTransport(local_address="0.0.0.0") else: - return getattr(litellm, 'sync_transport', None) + return getattr(litellm, "sync_transport", None) def get_async_httpx_client( diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 5037f4d8d44..05c640aa580 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -20,6 +20,7 @@ import litellm.litellm_core_utils import litellm.types import litellm.types.utils from litellm._logging import verbose_logger +from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, @@ -30,6 +31,7 @@ from litellm.llms.base_llm.audio_transcription.transformation import ( from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseConfig +from litellm.llms.base_llm.containers.transformation import BaseContainerConfig from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.base_llm.files.transformation import BaseFilesConfig from litellm.llms.base_llm.google_genai.transformation import ( @@ -39,10 +41,14 @@ from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) +from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, OCRResponse from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.base_llm.search.transformation import BaseSearchConfig, SearchResponse +from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeechConfig from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig +from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -55,12 +61,18 @@ from litellm.responses.streaming_iterator import ( ResponsesAPIStreamingIterator, SyncResponsesAPIStreamingIterator, ) +from litellm.types.containers.main import ( + ContainerListResponse, + ContainerObject, + DeleteContainerResult, +) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.llms.openai import ( CreateBatchRequest, CreateFileRequest, + HttpxBinaryResponseContent, OpenAIFileObject, ResponseInputParam, ResponsesAPIResponse, @@ -80,6 +92,7 @@ from litellm.types.vector_stores import ( VectorStoreSearchOptionalRequestParams, VectorStoreSearchResponse, ) +from litellm.types.videos.main import VideoObject from litellm.utils import ( CustomStreamWrapper, ImageResponse, @@ -908,11 +921,13 @@ class BaseLLMHTTPHandler: api_key=api_key, headers=headers or {}, model=model, + optional_params=optional_rerank_params, ) api_base = provider_config.get_complete_url( api_base=api_base, model=model, + optional_params=optional_rerank_params, ) data = provider_config.transform_rerank_request( @@ -1256,6 +1271,482 @@ class BaseLLMHTTPHandler: api_key=api_key, ) + def _prepare_ocr_request( + self, + model: str, + document: Dict[str, str], + optional_params: dict, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + api_base: Optional[str], + headers: Optional[Dict[str, Any]], + provider_config: BaseOCRConfig, + litellm_params: dict, + ) -> Tuple[Dict[str, Any], str, Dict[str, Any], None]: + """ + Shared logic for preparing OCR requests. + Returns: (headers, complete_url, data, files) + """ + from litellm.llms.base_llm.ocr.transformation import OCRRequestData + headers = provider_config.validate_environment( + api_key=api_key, + api_base=api_base, + headers=headers or {}, + model=model, + litellm_params=litellm_params, + ) + + complete_url = provider_config.get_complete_url( + api_base=api_base, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + # Transform the request to get data and files + transformed_result = provider_config.transform_ocr_request( + model=model, + document=document, + optional_params=optional_params, + headers=headers, + ) + + # All providers return OCRRequestData + if not isinstance(transformed_result, OCRRequestData): + raise ValueError( + f"Provider {provider_config.__class__.__name__} must return OCRRequestData" + ) + + # Data is always a dict for Mistral OCR format + if not isinstance(transformed_result.data, dict): + raise ValueError( + f"Expected dict data for OCR request, got {type(transformed_result.data)}" + ) + + data = transformed_result.data + + ## LOGGING + logging_obj.pre_call( + input="OCR document processing", + api_key=api_key, + additional_args={ + "complete_input_dict": data, + "api_base": complete_url, + "headers": headers, + }, + ) + + return headers, complete_url, data, None + + async def _async_prepare_ocr_request( + self, + model: str, + document: Dict[str, str], + optional_params: dict, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + api_base: Optional[str], + headers: Optional[Dict[str, Any]], + provider_config: BaseOCRConfig, + litellm_params: dict, + ) -> Tuple[Dict[str, Any], str, Dict[str, Any], None]: + """ + Async version of _prepare_ocr_request for providers that need async transforms. + Returns: (headers, complete_url, data, files) + """ + from litellm.llms.base_llm.ocr.transformation import OCRRequestData + + headers = provider_config.validate_environment( + api_key=api_key, + api_base=api_base, + headers=headers or {}, + model=model, + litellm_params=litellm_params, + ) + + complete_url = provider_config.get_complete_url( + api_base=api_base, + model=model, + optional_params=optional_params, + litellm_params=litellm_params, + ) + + # Use async transform (providers can override this method if they need async operations) + transformed_result = await provider_config.async_transform_ocr_request( + model=model, + document=document, + optional_params=optional_params, + headers=headers, + ) + + # All providers return OCRRequestData + if not isinstance(transformed_result, OCRRequestData): + raise ValueError( + f"Provider {provider_config.__class__.__name__} must return OCRRequestData" + ) + + # Data is always a dict for Mistral OCR format + if not isinstance(transformed_result.data, dict): + raise ValueError( + f"Expected dict data for OCR request, got {type(transformed_result.data)}" + ) + + data = transformed_result.data + + ## LOGGING + logging_obj.pre_call( + input="OCR document processing", + api_key=api_key, + additional_args={ + "complete_input_dict": data, + "api_base": complete_url, + "headers": headers, + }, + ) + + return headers, complete_url, data, None + + def _transform_ocr_response( + self, + provider_config: BaseOCRConfig, + model: str, + response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> OCRResponse: + """Shared logic for transforming OCR responses.""" + return provider_config.transform_ocr_response( + model=model, + raw_response=response, + logging_obj=logging_obj, + ) + + def ocr( + self, + model: str, + document: Dict[str, str], + optional_params: dict, + timeout: Union[float, httpx.Timeout], + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + api_base: Optional[str], + custom_llm_provider: str, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + aocr: bool = False, + headers: Optional[Dict[str, Any]] = None, + provider_config: Optional[BaseOCRConfig] = None, + litellm_params: Optional[dict] = None, + ) -> Union[OCRResponse, Coroutine[Any, Any, OCRResponse]]: + """ + Sync OCR handler. + """ + if provider_config is None: + raise ValueError( + f"No provider config found for model: {model} and provider: {custom_llm_provider}" + ) + + if litellm_params is None: + litellm_params = {} + + if aocr is True: + return self.async_ocr( + model=model, + document=document, + optional_params=optional_params, + timeout=timeout, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + client=client, + headers=headers, + provider_config=provider_config, + litellm_params=litellm_params, + ) + + # Prepare the request + headers, complete_url, data, files = self._prepare_ocr_request( + model=model, + document=document, + optional_params=optional_params, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, + headers=headers, + provider_config=provider_config, + litellm_params=litellm_params, + ) + + if client is None or not isinstance(client, HTTPHandler): + client = _get_httpx_client() + + try: + # Make the POST request with JSON data (Mistral format) + response = client.post( + url=complete_url, + headers=headers, + json=data, + timeout=timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=provider_config) + + return self._transform_ocr_response( + provider_config=provider_config, + model=model, + response=response, + logging_obj=logging_obj, + ) + + async def async_ocr( + self, + model: str, + document: Dict[str, str], + optional_params: dict, + timeout: Union[float, httpx.Timeout], + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + api_base: Optional[str], + custom_llm_provider: str, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + headers: Optional[Dict[str, Any]] = None, + provider_config: Optional[BaseOCRConfig] = None, + litellm_params: Optional[dict] = None, + ) -> OCRResponse: + """ + Async OCR handler. + """ + if provider_config is None: + raise ValueError( + f"No provider config found for model: {model} and provider: {custom_llm_provider}" + ) + + if litellm_params is None: + litellm_params = {} + + # Prepare the request using async prepare method + headers, complete_url, data, files = await self._async_prepare_ocr_request( + model=model, + document=document, + optional_params=optional_params, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, + headers=headers, + provider_config=provider_config, + litellm_params=litellm_params, + ) + + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + ) + else: + async_httpx_client = client + + try: + # Make the async POST request with JSON data (Mistral format) + response = await async_httpx_client.post( + url=complete_url, + headers=headers, + json=data, + timeout=timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=provider_config) + + # Use async response transform for async operations + return await provider_config.async_transform_ocr_response( + model=model, + raw_response=response, + logging_obj=logging_obj, + ) + + def search( + self, + query: Union[str, List[str]], + optional_params: dict, + timeout: Union[float, httpx.Timeout], + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + api_base: Optional[str], + custom_llm_provider: str, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + asearch: bool = False, + headers: Optional[Dict[str, Any]] = None, + provider_config: Optional[BaseSearchConfig] = None, + ) -> Union[SearchResponse, Coroutine[Any, Any, SearchResponse]]: + """ + Sync Search handler. + """ + if provider_config is None: + raise ValueError( + f"No provider config found for provider: {custom_llm_provider}" + ) + + if asearch is True: + return self.async_search( + query=query, + optional_params=optional_params, + timeout=timeout, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + client=client, + headers=headers, + provider_config=provider_config, + ) + + # Validate environment and get headers + headers = provider_config.validate_environment( + api_key=api_key, + api_base=api_base, + headers=headers or {}, + ) + + # Transform the request + data = provider_config.transform_search_request( + query=query, + optional_params=optional_params, + ) + + # Get complete URL (pass data for providers that need request body for URL construction) + complete_url = provider_config.get_complete_url( + api_base=api_base, + optional_params=optional_params, + data=data, + ) + + ## LOGGING + logging_obj.pre_call( + input=query if isinstance(query, str) else str(query), + api_key=api_key, + additional_args={ + "complete_input_dict": data, + "api_base": complete_url, + "headers": headers, + }, + ) + + if client is None or not isinstance(client, HTTPHandler): + client = _get_httpx_client() + + # Check HTTP method from provider config + http_method = provider_config.get_http_method() + + try: + if http_method == "GET": + # Make GET request (URL already contains query params from get_complete_url) + # Note: timeout is set on the client itself, not per-request for GET + response = client.get( + url=complete_url, + headers=headers, + ) + else: + # Make POST request with JSON data + response = client.post( + url=complete_url, + headers=headers, + json=data, + timeout=timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=provider_config) + + return provider_config.transform_search_response( + raw_response=response, + logging_obj=logging_obj, + ) + + async def async_search( + self, + query: Union[str, List[str]], + optional_params: dict, + timeout: Union[float, httpx.Timeout], + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + api_base: Optional[str], + custom_llm_provider: str, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + headers: Optional[Dict[str, Any]] = None, + provider_config: Optional[BaseSearchConfig] = None, + ) -> SearchResponse: + """ + Async Search handler. + """ + if provider_config is None: + raise ValueError( + f"No provider config found for provider: {custom_llm_provider}" + ) + + # Validate environment and get headers + headers = provider_config.validate_environment( + api_key=api_key, + api_base=api_base, + headers=headers or {}, + ) + + # Transform the request first + data = provider_config.transform_search_request( + query=query, + optional_params=optional_params, + ) + + # Get complete URL (pass data for providers that need request body for URL construction) + complete_url = provider_config.get_complete_url( + api_base=api_base, + optional_params=optional_params, + data=data, + ) + + ## LOGGING + logging_obj.pre_call( + input=query if isinstance(query, str) else str(query), + api_key=api_key, + additional_args={ + "complete_input_dict": data, + "api_base": complete_url, + "headers": headers, + }, + ) + + if client is None or not isinstance(client, AsyncHTTPHandler): + # For search providers, use special Search provider type + from litellm.types.llms.custom_http import httpxSpecialProvider + + async_httpx_client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.Search + ) + else: + async_httpx_client = client + + # Check HTTP method from provider config + http_method = provider_config.get_http_method().upper() + + try: + if http_method == "GET": + # Make async GET request (URL already contains query params from get_complete_url) + # Note: timeout is set on the client itself, not per-request for GET + response = await async_httpx_client.get( + url=complete_url, + headers=headers, + ) + else: + # Make async POST request with JSON data + response = await async_httpx_client.post( + url=complete_url, + headers=headers, + json=data, # type: ignore + timeout=timeout, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=provider_config) + + return provider_config.transform_search_response( + raw_response=response, + logging_obj=logging_obj, + ) + async def async_anthropic_messages_handler( self, model: str, @@ -1293,11 +1784,16 @@ class BaseLLMHTTPHandler: provider_specific_header=provider_specific_header, custom_llm_provider=custom_llm_provider, ) + forwarded_headers = kwargs.get("headers", None) + if forwarded_headers and extra_headers: + merged_headers = {**forwarded_headers, **extra_headers} + else: + merged_headers = forwarded_headers or extra_headers ( headers, api_base, ) = anthropic_messages_provider_config.validate_anthropic_messages_environment( - headers=extra_headers or {}, + headers=merged_headers or {}, model=model, messages=messages, optional_params=anthropic_messages_optional_request_params, @@ -1451,6 +1947,7 @@ class BaseLLMHTTPHandler: _is_async: bool = False, fake_stream: bool = False, litellm_metadata: Optional[Dict[str, Any]] = None, + shared_session: Optional["ClientSession"] = None, ) -> Union[ ResponsesAPIResponse, BaseResponsesAPIStreamingIterator, @@ -1479,6 +1976,7 @@ class BaseLLMHTTPHandler: client=client if isinstance(client, AsyncHTTPHandler) else None, fake_stream=fake_stream, litellm_metadata=litellm_metadata, + shared_session=shared_session, ) if client is None or not isinstance(client, HTTPHandler): @@ -1513,6 +2011,9 @@ class BaseLLMHTTPHandler: headers=headers, ) + if extra_body: + data.update(extra_body) + ## LOGGING logging_obj.pre_call( input=input, @@ -1539,7 +2040,7 @@ class BaseLLMHTTPHandler: headers=headers, json=data, timeout=timeout - or response_api_optional_request_params.get("timeout"), + or float(response_api_optional_request_params.get("timeout", 0)), stream=stream, ) if fake_stream is True: @@ -1567,7 +2068,7 @@ class BaseLLMHTTPHandler: headers=headers, json=data, timeout=timeout - or response_api_optional_request_params.get("timeout"), + or float(response_api_optional_request_params.get("timeout", 0)), ) except Exception as e: raise self._handle_error( @@ -1596,15 +2097,20 @@ class BaseLLMHTTPHandler: client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, fake_stream: bool = False, litellm_metadata: Optional[Dict[str, Any]] = None, + shared_session: Optional["ClientSession"] = None, ) -> Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]: """ Async version of the responses API handler. Uses async HTTP client to make requests. """ if client is None or not isinstance(client, AsyncHTTPHandler): + verbose_logger.debug( + f"Creating HTTP client for responses API with shared_session: {id(shared_session) if shared_session else None}" + ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, ) else: async_httpx_client = client @@ -1634,6 +2140,9 @@ class BaseLLMHTTPHandler: headers=headers, ) + if extra_body: + data.update(extra_body) + ## LOGGING logging_obj.pre_call( input=input, @@ -1660,7 +2169,7 @@ class BaseLLMHTTPHandler: headers=headers, json=data, timeout=timeout - or response_api_optional_request_params.get("timeout"), + or float(response_api_optional_request_params.get("timeout", 0)), stream=stream, ) @@ -1690,7 +2199,7 @@ class BaseLLMHTTPHandler: headers=headers, json=data, timeout=timeout - or response_api_optional_request_params.get("timeout"), + or float(response_api_optional_request_params.get("timeout", 0)), ) except Exception as e: @@ -1717,15 +2226,20 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, ) -> DeleteResponseResult: """ Async version of the delete response API handler. Uses async HTTP client to make requests. """ if client is None or not isinstance(client, AsyncHTTPHandler): + verbose_logger.debug( + f"Creating HTTP client for delete_response with shared_session: {id(shared_session) if shared_session else None}" + ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, ) else: async_httpx_client = client @@ -1788,6 +2302,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, ) -> Union[DeleteResponseResult, Coroutine[Any, Any, DeleteResponseResult]]: """ Async version of the responses API handler. @@ -1804,6 +2319,7 @@ class BaseLLMHTTPHandler: extra_body=extra_body, timeout=timeout, client=client, + shared_session=shared_session, ) if client is None or not isinstance(client, HTTPHandler): sync_httpx_client = _get_httpx_client( @@ -1870,6 +2386,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, ) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]: """ Get a response by ID @@ -1886,6 +2403,7 @@ class BaseLLMHTTPHandler: extra_body=extra_body, timeout=timeout, client=client, + shared_session=shared_session, ) if client is None or not isinstance(client, HTTPHandler): @@ -1949,14 +2467,19 @@ class BaseLLMHTTPHandler: extra_body: Optional[Dict[str, Any]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + shared_session: Optional["ClientSession"] = None, ) -> ResponsesAPIResponse: """ Async version of get_responses """ if client is None or not isinstance(client, AsyncHTTPHandler): + verbose_logger.debug( + f"Creating HTTP client for get_responses with shared_session: {id(shared_session) if shared_session else None}" + ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, ) else: async_httpx_client = client @@ -2027,6 +2550,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, ) -> Union[Dict, Coroutine[Any, Any, Dict]]: if _is_async: return self.async_list_responses_input_items( @@ -2043,6 +2567,7 @@ class BaseLLMHTTPHandler: extra_headers=extra_headers, timeout=timeout, client=client, + shared_session=shared_session, ) if client is None or not isinstance(client, HTTPHandler): @@ -2111,11 +2636,16 @@ class BaseLLMHTTPHandler: extra_headers: Optional[Dict[str, Any]] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + shared_session: Optional["ClientSession"] = None, ) -> Dict: if client is None or not isinstance(client, AsyncHTTPHandler): + verbose_logger.debug( + f"Creating HTTP client for list_input_items with shared_session: {id(shared_session) if shared_session else None}" + ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, ) else: async_httpx_client = client @@ -2802,6 +3332,7 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, ) -> Union[ResponsesAPIResponse, Coroutine[Any, Any, ResponsesAPIResponse]]: """ Async version of the responses API handler. @@ -2818,6 +3349,7 @@ class BaseLLMHTTPHandler: extra_body=extra_body, timeout=timeout, client=client, + shared_session=shared_session, ) if client is None or not isinstance(client, HTTPHandler): sync_httpx_client = _get_httpx_client( @@ -2884,15 +3416,20 @@ class BaseLLMHTTPHandler: timeout: Optional[Union[float, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, _is_async: bool = False, + shared_session: Optional["ClientSession"] = None, ) -> ResponsesAPIResponse: """ Async version of the cancel response API handler. Uses async HTTP client to make requests. """ if client is None or not isinstance(client, AsyncHTTPHandler): + verbose_logger.debug( + f"Creating HTTP client for cancel_response with shared_session: {id(shared_session) if shared_session else None}" + ) async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + shared_session=shared_session, ) else: async_httpx_client = client @@ -2995,7 +3532,12 @@ class BaseLLMHTTPHandler: BaseGoogleGenAIGenerateContentConfig, BaseAnthropicMessagesConfig, BaseBatchesConfig, + BaseOCRConfig, + BaseVideoConfig, + BaseSearchConfig, + BaseTextToSpeechConfig, "BasePassthroughConfig", + "BaseContainerConfig", ], ): status_code = getattr(e, "status_code", 500) @@ -3054,7 +3596,9 @@ class BaseLLMHTTPHandler: try: async with websockets.connect( # type: ignore - url, extra_headers=headers + url, + extra_headers=headers, + max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ) as backend_ws: realtime_streaming = RealTimeStreaming( websocket, @@ -3497,6 +4041,1640 @@ class BaseLLMHTTPHandler: return model_response + ###### VIDEO GENERATION HANDLER ###### + def video_generation_handler( + self, + model: str, + prompt: str, + video_generation_provider_config: BaseVideoConfig, + video_generation_optional_request_params: Dict, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + timeout: Union[float, httpx.Timeout], + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + _is_async: bool = False, + fake_stream: bool = False, + litellm_metadata: Optional[Dict[str, Any]] = None, + api_key: Optional[str] = None, + ) -> Union[ + VideoObject, + Coroutine[Any, Any, VideoObject], + ]: + """ + Handles video generation requests. + When _is_async=True, returns a coroutine instead of making the call directly. + """ + if _is_async: + # Return the async coroutine if called with _is_async=True + return self.async_video_generation_handler( + model=model, + prompt=prompt, + video_generation_provider_config=video_generation_provider_config, + video_generation_optional_request_params=video_generation_optional_request_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + client=client if isinstance(client, AsyncHTTPHandler) else None, + fake_stream=fake_stream, + litellm_metadata=litellm_metadata, + api_key=api_key, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = video_generation_provider_config.validate_environment( + api_key=api_key, + headers=video_generation_optional_request_params.get("extra_headers", {}) + or {}, + model=model, + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = video_generation_provider_config.get_complete_url( + model=model, + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + data, files, api_base = video_generation_provider_config.transform_video_create_request( + model=model, + prompt=prompt, + video_create_optional_request_params=video_generation_optional_request_params, + litellm_params=litellm_params, + headers=headers, + api_base=api_base, + ) + + ## LOGGING + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + # Use JSON when no files, otherwise use form data with files + if files and len(files) > 0: + # Use multipart/form-data when files are present + response = sync_httpx_client.post( + url=api_base, + headers=headers, + data=data, + files=files, + timeout=timeout, + ) + + else: + # Use JSON content type for POST requests without files + response = sync_httpx_client.post( + url=api_base, + headers=headers, + json=data, + timeout=timeout, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=video_generation_provider_config, + ) + + return video_generation_provider_config.transform_video_create_response( + model=model, + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + request_data=data, + ) + + async def async_video_generation_handler( + self, + model: str, + prompt: str, + video_generation_provider_config: "BaseVideoConfig", + video_generation_optional_request_params: Dict, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + timeout: Union[float, httpx.Timeout], + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + fake_stream: bool = False, + litellm_metadata: Optional[Dict[str, Any]] = None, + api_key: Optional[str] = None, + ) -> VideoObject: + """ + Async version of the video generation handler. + Uses async HTTP client to make requests. + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = video_generation_provider_config.validate_environment( + api_key=api_key, + headers=video_generation_optional_request_params.get("extra_headers", {}) + or {}, + model=model, + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = video_generation_provider_config.get_complete_url( + model=model, + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + data, files, api_base = video_generation_provider_config.transform_video_create_request( + model=model, + prompt=prompt, + api_base=api_base, + video_create_optional_request_params=video_generation_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + #Use JSON when no files, otherwise use form data with files + if files is None or len(files) == 0: + response = await async_httpx_client.post( + url=api_base, + headers=headers, + json=data, + timeout=timeout, + ) + else: + response = await async_httpx_client.post( + url=api_base, + headers=headers, + data=data, + files=files, + timeout=timeout, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=video_generation_provider_config, + ) + + return video_generation_provider_config.transform_video_create_response( + model=model, + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + request_data=data, + ) + + ###### VIDEO CONTENT HANDLER ###### + def video_content_handler( + self, + video_id: str, + video_content_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + timeout: Union[float, httpx.Timeout], + extra_headers: Optional[Dict[str, Any]] = None, + api_key: Optional[str] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + _is_async: bool = False, + ) -> Union[bytes, Coroutine[Any, Any, bytes]]: + """ + Handle video content download requests. + """ + if _is_async: + return self.async_video_content_handler( + video_id=video_id, + video_content_provider_config=video_content_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + timeout=timeout, + extra_headers=extra_headers, + api_key=api_key, + client=client, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = video_content_provider_config.validate_environment( + headers=extra_headers or {}, + model="", + api_key=api_key, + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = video_content_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, data = video_content_provider_config.transform_video_content_request( + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + try: + # Use POST if params contains data (e.g., Vertex AI fetchPredictOperation) + # Otherwise use GET (e.g., OpenAI video content download) + if data: + response = sync_httpx_client.post( + url=url, + headers=headers, + json=data, + ) + else: + # Otherwise it's a GET request with query params + response = sync_httpx_client.get( + url=url, + headers=headers, + params=data, + ) + + # Transform the response using the provider config + return video_content_provider_config.transform_video_content_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=video_content_provider_config, + ) + + async def async_video_content_handler( + self, + video_id: str, + video_content_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params: GenericLiteLLMParams, + logging_obj: LiteLLMLoggingObj, + timeout: Union[float, httpx.Timeout], + extra_headers: Optional[Dict[str, Any]] = None, + api_key: Optional[str] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> bytes: + """ + Async version of the video content download handler. + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = video_content_provider_config.validate_environment( + headers=extra_headers or {}, + model="", + api_key=api_key, + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = video_content_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, data = video_content_provider_config.transform_video_content_request( + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + try: + # Use POST if params contains data (e.g., Vertex AI fetchPredictOperation) + # Otherwise use GET (e.g., OpenAI video content download) + if data: + response = await async_httpx_client.post( + url=url, + headers=headers, + json=data, + ) + else: + # Otherwise it's a GET request with query params + response = await async_httpx_client.get( + url=url, + headers=headers, + params=data, + ) + + # Transform the response using the provider config + return await video_content_provider_config.async_transform_video_content_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=video_content_provider_config, + ) + + def video_remix_handler( + self, + video_id: str, + prompt: str, + video_remix_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + _is_async: bool = False, + client=None, + api_key: Optional[str] = None, + ): + """ + Handler for video remix requests. + When _is_async=True, returns a coroutine instead of making the call directly. + """ + if _is_async: + # Return the async coroutine if called with _is_async=True + return self.async_video_remix_handler( + video_id=video_id, + prompt=prompt, + video_remix_provider_config=video_remix_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + client=client, + api_key=api_key, + ) + + # For sync calls, use sync HTTP client directly (like video_generation does) + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = video_remix_provider_config.validate_environment( + api_key=api_key, + headers=extra_headers or {}, + model="", + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = video_remix_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, data = video_remix_provider_config.transform_video_remix_request( + video_id=video_id, + prompt=prompt, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + ) + + ## LOGGING + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + + try: + response = sync_httpx_client.post( + url=url, + headers=headers, + json=data, + timeout=timeout, + ) + + return video_remix_provider_config.transform_video_remix_response( + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=video_remix_provider_config, + ) + + async def async_video_remix_handler( + self, + video_id: str, + prompt: str, + video_remix_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + client=None, + api_key: Optional[str] = None, + ): + """ + Async version of the video remix handler. + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = video_remix_provider_config.validate_environment( + api_key=api_key, + headers=extra_headers or {}, + model="", + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = video_remix_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, data = video_remix_provider_config.transform_video_remix_request( + video_id=video_id, + prompt=prompt, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + ) + + ## LOGGING + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + + try: + response = await async_httpx_client.post( + url=url, + headers=headers, + json=data, + timeout=timeout, + ) + + return video_remix_provider_config.transform_video_remix_response( + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=video_remix_provider_config, + ) + + def video_list_handler( + self, + after: Optional[str], + limit: Optional[int], + order: Optional[str], + video_list_provider_config, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + _is_async: bool = False, + client=None, + api_key: Optional[str] = None, + ): + """ + Handler for video list requests. + """ + if _is_async: + return self.async_video_list_handler( + after=after, + limit=limit, + order=order, + video_list_provider_config=video_list_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_query=extra_query, + timeout=timeout, + client=client, + api_key=api_key, + ) + else: + # For sync calls, we'll use the async handler in a sync context + import asyncio + + return asyncio.run( + self.async_video_list_handler( + after=after, + limit=limit, + order=order, + video_list_provider_config=video_list_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_query=extra_query, + timeout=timeout, + client=client, + ) + ) + + async def async_video_list_handler( + self, + after: Optional[str], + limit: Optional[int], + order: Optional[str], + video_list_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + client=None, + api_key: Optional[str] = None, + ): + """ + Async version of the video list handler. + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = video_list_provider_config.validate_environment( + api_key=api_key, + headers=extra_headers or {}, + model="", + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = video_list_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, params = video_list_provider_config.transform_video_list_request( + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + after=after, + limit=limit, + order=order, + extra_query=extra_query, + ) + + ## LOGGING + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "params": params, + }, + ) + + try: + response = await async_httpx_client.get( + url=url, + headers=headers, + params=params, + ) + + return video_list_provider_config.transform_video_list_response( + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=video_list_provider_config, + ) + + async def async_video_delete_handler( + self, + video_id: str, + video_delete_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + client=None, + api_key: Optional[str] = None, + ): + """ + Async version of the video delete handler. + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = video_delete_provider_config.validate_environment( + api_key=api_key, + headers=extra_headers or {}, + model="", + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = video_delete_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, data = video_delete_provider_config.transform_video_delete_request( + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + + try: + response = await async_httpx_client.delete( + url=url, + headers=headers, + timeout=timeout, + ) + + return video_delete_provider_config.transform_video_delete_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=video_delete_provider_config, + ) + + def video_status_handler( + self, + video_id: str, + video_status_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + _is_async: bool = False, + client=None, + api_key: Optional[str] = None, + ): + """ + Handler for video status requests. + When _is_async=True, returns a coroutine instead of making the call directly. + """ + if _is_async: + # Return the async coroutine if called with _is_async=True + return self.async_video_status_handler( + video_id=video_id, + video_status_provider_config=video_status_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + client=client, + api_key=api_key, + ) + + # For sync calls, use sync HTTP client directly (like video_generation does) + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = video_status_provider_config.validate_environment( + api_key=api_key, + headers=extra_headers or {}, + model="", + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = video_status_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, data = video_status_provider_config.transform_video_status_retrieve_request( + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "video_id": video_id, + "data": data, + }, + ) + + try: + # Use POST if data is provided (e.g., Vertex AI fetchPredictOperation) + # Otherwise use GET (e.g., OpenAI video status) + if data: + response = sync_httpx_client.post( + url=url, + headers=headers, + json=data, + ) + else: + response = sync_httpx_client.get( + url=url, + headers=headers, + ) + + return video_status_provider_config.transform_video_status_retrieve_response( + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=video_status_provider_config, + ) + + async def async_video_status_handler( + self, + video_id: str, + video_status_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + client=None, + api_key: Optional[str] = None, + ): + """ + Async version of the video status handler. + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = video_status_provider_config.validate_environment( + api_key=api_key, + headers=extra_headers or {}, + model="", + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = video_status_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, data = video_status_provider_config.transform_video_status_retrieve_request( + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "video_id": video_id, + "data": data, + }, + ) + + try: + # Use POST if data is provided (e.g., Vertex AI fetchPredictOperation) + # Otherwise use GET (e.g., OpenAI video status) + if data: + response = await async_httpx_client.post( + url=url, + headers=headers, + json=data, + ) + else: + response = await async_httpx_client.get( + url=url, + headers=headers, + ) + return video_status_provider_config.transform_video_status_retrieve_response( + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=video_status_provider_config, + ) + + ###### CONTAINER HANDLER ###### + def container_create_handler( + self, + name: str, + container_create_request_params: Dict, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + _is_async: bool = False, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> Union["ContainerObject", Coroutine[Any, Any, "ContainerObject"]]: + if _is_async: + # Return the async coroutine if called with _is_async=True + return self.async_container_create_handler( + name=name, + container_create_request_params=container_create_request_params, + container_provider_config=container_provider_config, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + timeout=timeout, + client=client, + ) + + # For sync calls, use sync HTTP client + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + # Validate environment and get headers + headers = container_provider_config.validate_environment( + headers=extra_headers or {}, + api_key=litellm_params.get("api_key", None), + ) + + # Add Content-Type header for JSON requests + headers["Content-Type"] = "application/json" + + if extra_headers: + headers.update(extra_headers) + + # Get the complete URL for the request + api_base = container_provider_config.get_complete_url( + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + data = container_provider_config.transform_container_create_request( + name=name, + container_create_optional_request_params=container_create_request_params, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=name, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + response = sync_httpx_client.post( + url=api_base, + headers=headers, + json=data, + timeout=timeout, + ) + + return container_provider_config.transform_container_create_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=container_provider_config, + ) + + async def async_container_create_handler( + self, + name: str, + container_create_request_params: Dict, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> "ContainerObject": + # For async calls, use async HTTP client + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.OPENAI, + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + # Validate environment and get headers + headers = container_provider_config.validate_environment( + headers=extra_headers or {}, + api_key=litellm_params.get("api_key", None), + ) + + # Add Content-Type header for JSON requests + headers["Content-Type"] = "application/json" + + if extra_headers: + headers.update(extra_headers) + + # Get the complete URL for the request + api_base = container_provider_config.get_complete_url( + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + data = container_provider_config.transform_container_create_request( + name=name, + container_create_optional_request_params=container_create_request_params, + litellm_params=litellm_params, + headers=headers, + ) + + ## LOGGING + logging_obj.pre_call( + input=name, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + response = await async_httpx_client.post( + url=api_base, + headers=headers, + json=data, + timeout=timeout, + ) + + return container_provider_config.transform_container_create_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=container_provider_config, + ) + + def container_list_handler( + self, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + after: Optional[str] = None, + limit: Optional[int] = None, + order: Optional[str] = None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + _is_async: bool = False, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> Union["ContainerListResponse", Coroutine[Any, Any, "ContainerListResponse"]]: + if _is_async: + # Return the async coroutine if called with _is_async=True + return self.async_container_list_handler( + container_provider_config=container_provider_config, + litellm_params=litellm_params, + logging_obj=logging_obj, + after=after, + limit=limit, + order=order, + extra_headers=extra_headers, + extra_query=extra_query, + timeout=timeout, + client=client, + ) + + # For sync calls, use sync HTTP client + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + # Validate environment and get headers + headers = container_provider_config.validate_environment( + headers=extra_headers or {}, + api_key=litellm_params.get("api_key", None), + ) + + if extra_headers: + headers.update(extra_headers) + + # Get the complete URL for the request + api_base = container_provider_config.get_complete_url( + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, params = container_provider_config.transform_container_list_request( + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + after=after, + limit=limit, + order=order, + extra_query=extra_query, + ) + + ## LOGGING + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "params": params, + }, + ) + + try: + response = sync_httpx_client.get( + url=url, + headers=headers, + params=params, + ) + + return container_provider_config.transform_container_list_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=container_provider_config, + ) + + async def async_container_list_handler( + self, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + after: Optional[str] = None, + limit: Optional[int] = None, + order: Optional[str] = None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> "ContainerListResponse": + # For async calls, use async HTTP client + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.OPENAI, + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + # Validate environment and get headers + headers = container_provider_config.validate_environment( + headers=extra_headers or {}, + api_key=litellm_params.get("api_key", None), + ) + + if extra_headers: + headers.update(extra_headers) + + # Get the complete URL for the request + api_base = container_provider_config.get_complete_url( + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, params = container_provider_config.transform_container_list_request( + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + after=after, + limit=limit, + order=order, + extra_query=extra_query, + ) + + ## LOGGING + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "params": params, + }, + ) + + try: + response = await async_httpx_client.get( + url=url, + headers=headers, + params=params, + ) + + return container_provider_config.transform_container_list_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=container_provider_config, + ) + + def container_retrieve_handler( + self, + container_id: str, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + _is_async: bool = False, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> Union["ContainerObject", Coroutine[Any, Any, "ContainerObject"]]: + if _is_async: + # Return the async coroutine if called with _is_async=True + return self.async_container_retrieve_handler( + container_id=container_id, + container_provider_config=container_provider_config, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_query=extra_query, + timeout=timeout, + client=client, + ) + + # For sync calls, use sync HTTP client + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + # Validate environment and get headers + headers = container_provider_config.validate_environment( + headers=extra_headers or {}, + api_key=litellm_params.get("api_key", None), + ) + + if extra_headers: + headers.update(extra_headers) + + # Get the complete URL for the request + api_base = container_provider_config.get_complete_url( + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, params = container_provider_config.transform_container_retrieve_request( + container_id=container_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + # Add any extra query parameters + if extra_query: + params.update(extra_query) + + ## LOGGING + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "params": params, + "container_id": container_id, + }, + ) + + try: + response = sync_httpx_client.get( + url=url, + headers=headers, + params=params, + ) + + return container_provider_config.transform_container_retrieve_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=container_provider_config, + ) + + async def async_container_retrieve_handler( + self, + container_id: str, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> "ContainerObject": + # For async calls, use async HTTP client + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.OPENAI, + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + # Validate environment and get headers + headers = container_provider_config.validate_environment( + headers=extra_headers or {}, + api_key=litellm_params.get("api_key", None), + ) + + if extra_headers: + headers.update(extra_headers) + + # Get the complete URL for the request + api_base = container_provider_config.get_complete_url( + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, params = container_provider_config.transform_container_retrieve_request( + container_id=container_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + # Add any extra query parameters + if extra_query: + params.update(extra_query) + + ## LOGGING + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "params": params, + "container_id": container_id, + }, + ) + + try: + response = await async_httpx_client.get( + url=url, + headers=headers, + params=params, + ) + + return container_provider_config.transform_container_retrieve_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=container_provider_config, + ) + + def container_delete_handler( + self, + container_id: str, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + _is_async: bool = False, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> Union["DeleteContainerResult", Coroutine[Any, Any, "DeleteContainerResult"]]: + if _is_async: + # Return the async coroutine if called with _is_async=True + return self.async_container_delete_handler( + container_id=container_id, + container_provider_config=container_provider_config, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_query=extra_query, + timeout=timeout, + client=client, + ) + + # For sync calls, use sync HTTP client + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + # Validate environment and get headers + headers = container_provider_config.validate_environment( + headers=extra_headers or {}, + api_key=litellm_params.get("api_key", None), + ) + + if extra_headers: + headers.update(extra_headers) + + # Get the complete URL for the request + api_base = container_provider_config.get_complete_url( + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, params = container_provider_config.transform_container_delete_request( + container_id=container_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + # Add any extra query parameters + if extra_query: + params.update(extra_query) + + ## LOGGING + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "params": params, + "container_id": container_id, + }, + ) + + try: + response = sync_httpx_client.delete( + url=url, + headers=headers, + params=params, + ) + + return container_provider_config.transform_container_delete_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=container_provider_config, + ) + + async def async_container_delete_handler( + self, + container_id: str, + container_provider_config: "BaseContainerConfig", + litellm_params: GenericLiteLLMParams, + logging_obj: "LiteLLMLoggingObj", + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + timeout: Union[float, httpx.Timeout] = 600, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> "DeleteContainerResult": + # For async calls, use async HTTP client + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.OPENAI, + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + # Validate environment and get headers + headers = container_provider_config.validate_environment( + headers=extra_headers or {}, + api_key=litellm_params.get("api_key", None), + ) + + if extra_headers: + headers.update(extra_headers) + + # Get the complete URL for the request + api_base = container_provider_config.get_complete_url( + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + # Transform the request using the provider config + url, params = container_provider_config.transform_container_delete_request( + container_id=container_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + # Add any extra query parameters + if extra_query: + params.update(extra_query) + + ## LOGGING + logging_obj.pre_call( + input="", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "params": params, + "container_id": container_id, + }, + ) + + try: + response = await async_httpx_client.delete( + url=url, + headers=headers, + params=params, + ) + + return container_provider_config.transform_container_delete_response( + raw_response=response, + logging_obj=logging_obj, + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=container_provider_config, + ) + ###### VECTOR STORE HANDLER ###### async def async_vector_store_search_handler( self, @@ -3568,6 +5746,7 @@ class BaseLLMHTTPHandler: ) try: + response = await async_httpx_client.post( url=url, headers=headers, @@ -4055,3 +6234,222 @@ class BaseLLMHTTPHandler: raw_response=response, logging_obj=logging_obj, ) + + ##################################################################### + ################ TEXT TO SPEECH HANDLER ########################### + ##################################################################### + def text_to_speech_handler( + self, + model: str, + input: str, + voice: Optional[str], + text_to_speech_provider_config: BaseTextToSpeechConfig, + text_to_speech_optional_params: Dict, + custom_llm_provider: str, + litellm_params: Dict, + logging_obj: LiteLLMLoggingObj, + timeout: Union[float, httpx.Timeout], + extra_headers: Optional[Dict[str, Any]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + _is_async: bool = False, + ) -> Union[ + "HttpxBinaryResponseContent", + Coroutine[Any, Any, "HttpxBinaryResponseContent"], + ]: + """ + Handles text-to-speech requests. + When _is_async=True, returns a coroutine instead of making the call directly. + """ + if _is_async: + return self.async_text_to_speech_handler( + model=model, + input=input, + voice=voice, + text_to_speech_provider_config=text_to_speech_provider_config, + text_to_speech_optional_params=text_to_speech_optional_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + timeout=timeout, + client=client if isinstance(client, AsyncHTTPHandler) else None, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = text_to_speech_provider_config.validate_environment( + api_key=litellm_params.get("api_key"), + headers=extra_headers or {}, + model=model, + api_base=litellm_params.get("api_base"), + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = text_to_speech_provider_config.get_complete_url( + model=model, + api_base=litellm_params.get("api_base"), + litellm_params=litellm_params, + ) + + request_data = text_to_speech_provider_config.transform_text_to_speech_request( + model=model, + input=input, + voice=voice, + optional_params=text_to_speech_optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + # Merge provider-specific headers + if "headers" in request_data: + headers.update(request_data["headers"]) + + ## LOGGING + logging_obj.pre_call( + input=input, + api_key="", + additional_args={ + "complete_input_dict": request_data, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + # Determine request body type and send appropriately + if "dict_body" in request_data: + response = sync_httpx_client.post( + url=api_base, + headers=headers, + json=request_data["dict_body"], + timeout=timeout, + ) + elif "ssml_body" in request_data: + response = sync_httpx_client.post( + url=api_base, + headers=headers, + data=request_data["ssml_body"], + timeout=timeout, + ) + else: + raise ValueError( + "No body found in request_data. Must provide one of: dict_body, ssml_body, text_body, binary_body" + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=text_to_speech_provider_config, + ) + + return text_to_speech_provider_config.transform_text_to_speech_response( + model=model, + raw_response=response, + logging_obj=logging_obj, + ) + + async def async_text_to_speech_handler( + self, + model: str, + input: str, + voice: Optional[str], + text_to_speech_provider_config: BaseTextToSpeechConfig, + text_to_speech_optional_params: Dict, + custom_llm_provider: str, + litellm_params: Dict, + logging_obj: LiteLLMLoggingObj, + timeout: Union[float, httpx.Timeout], + extra_headers: Optional[Dict[str, Any]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + ) -> "HttpxBinaryResponseContent": + """ + Async version of the text-to-speech handler. + Uses async HTTP client to make requests. + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = text_to_speech_provider_config.validate_environment( + api_key=litellm_params.get("api_key"), + headers=extra_headers or {}, + model=model, + api_base=litellm_params.get("api_base"), + ) + + if extra_headers: + headers.update(extra_headers) + + api_base = text_to_speech_provider_config.get_complete_url( + model=model, + api_base=litellm_params.get("api_base"), + litellm_params=litellm_params, + ) + + request_data = text_to_speech_provider_config.transform_text_to_speech_request( + model=model, + input=input, + voice=voice, + optional_params=text_to_speech_optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + # Merge provider-specific headers + if "headers" in request_data: + headers.update(request_data["headers"]) + + ## LOGGING + logging_obj.pre_call( + input=input, + api_key="", + additional_args={ + "complete_input_dict": request_data, + "api_base": api_base, + "headers": headers, + }, + ) + + try: + # Determine request body type and send appropriately + if "dict_body" in request_data: + response = await async_httpx_client.post( + url=api_base, + headers=headers, + json=request_data["dict_body"], + timeout=timeout, + ) + elif "ssml_body" in request_data: + response = await async_httpx_client.post( + url=api_base, + headers=headers, + data=request_data["ssml_body"], + timeout=timeout, + ) + else: + raise ValueError( + "No body found in request_data. Must provide one of: dict_body, ssml_body, text_body, binary_body" + ) + + except Exception as e: + raise self._handle_error( + e=e, + provider_config=text_to_speech_provider_config, + ) + + return text_to_speech_provider_config.transform_text_to_speech_response( + model=model, + raw_response=response, + logging_obj=logging_obj, + ) \ No newline at end of file diff --git a/litellm/llms/dashscope/cost_calculator.py b/litellm/llms/dashscope/cost_calculator.py index 107eb7f5adf..9b3e3851162 100644 --- a/litellm/llms/dashscope/cost_calculator.py +++ b/litellm/llms/dashscope/cost_calculator.py @@ -14,6 +14,7 @@ from litellm.utils import get_model_info @dataclass class TokenBreakdown: """Token breakdown for cost calculation.""" + text_tokens: int cached_tokens: int completion_tokens: int @@ -23,133 +24,194 @@ class TokenBreakdown: def _extract_token_breakdown(usage: Usage) -> TokenBreakdown: """Extract token counts from usage, handling cached and reasoning tokens.""" cached_tokens = 0 - if usage.prompt_tokens_details and hasattr(usage.prompt_tokens_details, "cached_tokens"): + if usage.prompt_tokens_details and hasattr( + usage.prompt_tokens_details, "cached_tokens" + ): cached_tokens = usage.prompt_tokens_details.cached_tokens or 0 - + text_tokens = usage.prompt_tokens - cached_tokens - + reasoning_tokens = 0 - if (hasattr(usage, "completion_tokens_details") and - usage.completion_tokens_details and - hasattr(usage.completion_tokens_details, "reasoning_tokens")): + if ( + hasattr(usage, "completion_tokens_details") + and usage.completion_tokens_details + and hasattr(usage.completion_tokens_details, "reasoning_tokens") + ): reasoning_tokens = usage.completion_tokens_details.reasoning_tokens or 0 - + completion_tokens = (usage.completion_tokens or 0) - reasoning_tokens - - return TokenBreakdown(text_tokens, cached_tokens, completion_tokens, reasoning_tokens) + + return TokenBreakdown( + text_tokens, cached_tokens, completion_tokens, reasoning_tokens + ) def _calculate_tiered_cost( - tokens: int, - tiered_pricing: List[dict], + tokens: int, + tiered_pricing: List[dict], cost_key: str, - fallback_cost_key: Optional[str] = None + fallback_cost_key: Optional[str] = None, ) -> float: - """Calculate cost using tiered pricing structure. - - Finds the appropriate tier based on token count and applies that tier's rate to all tokens. + """ + Calculate cost for a given number of tokens based on a true tiered pricing structure. + + This function iterates through sorted pricing tiers, calculates the cost for the + number of tokens that fall into each tier's range, and sums them up to get the total cost. + + Args: + tokens (int): The total number of tokens to calculate the cost for. + tiered_pricing (List[dict]): A list of dictionaries, where each dictionary + represents a pricing tier. + cost_key (str): The key in the tier dictionary that holds the per-token cost + (e.g., 'input_cost_per_token'). + fallback_cost_key (Optional[str], optional): A fallback key to use if the + primary `cost_key` is not found in a tier. Defaults to None. + + Returns: + float: The total calculated cost for the given tokens. + + Example: + >>> tiered_pricing = [ + ... {"range": [0, 100000], "input_cost_per_token": 0.0001}, + ... {"range": [100000, 500000], "input_cost_per_token": 0.00005}, + ... ] + + Calculating cost for 150,000 tokens: + (100,000 * 0.0001) + (50,000 * 0.00005) = $12.5 """ if not tiered_pricing or tokens <= 0: return 0.0 - - # Find the appropriate tier for the token count - for tier in tiered_pricing: + + total_cost = 0.0 + tokens_processed = 0 + + sorted_tiers = sorted(tiered_pricing, key=lambda x: x.get("range", [0, 0])[0]) + + for tier in sorted_tiers: + if tokens_processed >= tokens: + break + tier_range = tier.get("range", []) if len(tier_range) != 2: continue - + range_start, range_end = tier_range - - # Check if tokens fall within this tier's range - if range_start <= tokens <= range_end: + + if tokens <= range_start: + continue + + tier_start = max(range_start, tokens_processed) + tier_end = min(range_end, tokens) + + if tier_end > tier_start: + tokens_in_tier = tier_end - tier_start cost_per_token = tier.get(cost_key) or tier.get(fallback_cost_key, 0) - return tokens * cost_per_token - - # If no tier matches, use the last tier (highest tier) - if tiered_pricing: - last_tier = tiered_pricing[-1] + total_cost += tokens_in_tier * cost_per_token + tokens_processed = tier_end + + # After loop, check if any tokens remain (i.e., tokens > highest tier's end range) + # and charge them at the last tier's rate. + if tokens_processed < tokens and sorted_tiers: + last_tier = sorted_tiers[-1] + remaining_tokens = tokens - tokens_processed cost_per_token = last_tier.get(cost_key) or last_tier.get(fallback_cost_key, 0) - return tokens * cost_per_token - - return 0.0 + total_cost += remaining_tokens * cost_per_token + + return total_cost -def _calculate_flat_cost(tokens: int, cost_per_token: float) -> float: - """Calculate cost using flat pricing.""" - return tokens * cost_per_token - - -def _calculate_prompt_cost(breakdown: TokenBreakdown, model_info: ModelInfo, tiered_pricing: Optional[List[dict]]) -> float: +def _calculate_prompt_cost( + breakdown: TokenBreakdown, + model_info: ModelInfo, + tiered_pricing: Optional[List[dict]], +) -> float: """Calculate total prompt cost including cached tokens.""" if tiered_pricing: text_cost = _calculate_tiered_cost( - tokens=breakdown.text_tokens, - tiered_pricing=tiered_pricing, - cost_key="input_cost_per_token" + tokens=breakdown.text_tokens, + tiered_pricing=tiered_pricing, + cost_key="input_cost_per_token", ) cache_cost = _calculate_tiered_cost( - tokens=breakdown.cached_tokens, - tiered_pricing=tiered_pricing, - cost_key="cache_read_input_token_cost" + tokens=breakdown.cached_tokens, + tiered_pricing=tiered_pricing, + cost_key="cache_read_input_token_cost", + fallback_cost_key="input_cost_per_token", ) return text_cost + cache_cost - - input_cost = model_info.get("input_cost_per_token", 0.0) - cache_cost = model_info.get("cache_read_input_token_cost", input_cost) or input_cost - - return (_calculate_flat_cost(tokens=breakdown.text_tokens, cost_per_token=input_cost) + - _calculate_flat_cost(tokens=breakdown.cached_tokens, cost_per_token=cache_cost)) + + input_cost = float(model_info.get("input_cost_per_token") or 0.0) + + # For cache_cost, first try the specific key, then fall back to input_cost. + cache_cost_val = model_info.get("cache_read_input_token_cost") + if cache_cost_val is None: + cache_cost = input_cost + else: + cache_cost = float(cache_cost_val) + + return (breakdown.text_tokens * input_cost) + (breakdown.cached_tokens * cache_cost) -def _calculate_completion_cost(breakdown: TokenBreakdown, model_info: ModelInfo, tiered_pricing: Optional[List[dict]]) -> float: +def _calculate_completion_cost( + breakdown: TokenBreakdown, + model_info: ModelInfo, + tiered_pricing: Optional[List[dict]], +) -> float: """Calculate total completion cost including reasoning tokens.""" if tiered_pricing: completion_cost = _calculate_tiered_cost( - tokens=breakdown.completion_tokens, - tiered_pricing=tiered_pricing, - cost_key="output_cost_per_token" + tokens=breakdown.completion_tokens, + tiered_pricing=tiered_pricing, + cost_key="output_cost_per_token", ) reasoning_cost = _calculate_tiered_cost( - tokens=breakdown.reasoning_tokens, - tiered_pricing=tiered_pricing, + tokens=breakdown.reasoning_tokens, + tiered_pricing=tiered_pricing, cost_key="output_cost_per_reasoning_token", - fallback_cost_key="output_cost_per_token" + fallback_cost_key="output_cost_per_token", ) return completion_cost + reasoning_cost - - output_cost = model_info.get("output_cost_per_token", 0.0) - reasoning_cost = model_info.get("output_cost_per_reasoning_token", output_cost) or output_cost - - return (_calculate_flat_cost(tokens=breakdown.completion_tokens, cost_per_token=output_cost) + - _calculate_flat_cost(tokens=breakdown.reasoning_tokens, cost_per_token=reasoning_cost)) + + output_cost = float(model_info.get("output_cost_per_token") or 0.0) + + # For reasoning_cost, first try the specific key, then fall back to output_cost. + reasoning_cost_val = model_info.get("output_cost_per_reasoning_token") + if reasoning_cost_val is None: + reasoning_cost = output_cost + else: + reasoning_cost = float(reasoning_cost_val) + + return (breakdown.completion_tokens * output_cost) + ( + breakdown.reasoning_tokens * reasoning_cost + ) def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]: """ Calculate cost per token for Dashscope models. - + Supports both tiered and flat pricing with cached and reasoning tokens. - + Args: model: Model name without provider prefix usage: LiteLLM Usage block - + Returns: Tuple[float, float] - (prompt_cost_in_usd, completion_cost_in_usd) """ model_info = get_model_info(model=model, custom_llm_provider="dashscope") breakdown = _extract_token_breakdown(usage) - tiered_pricing = model_info.get("tiered_pricing") if isinstance(model_info.get("tiered_pricing"), list) else None - + tiered_pricing = ( + model_info.get("tiered_pricing") + if isinstance(model_info.get("tiered_pricing"), list) + else None + ) + prompt_cost = _calculate_prompt_cost( - breakdown=breakdown, - model_info=model_info, - tiered_pricing=tiered_pricing + breakdown=breakdown, model_info=model_info, tiered_pricing=tiered_pricing ) completion_cost = _calculate_completion_cost( - breakdown=breakdown, - model_info=model_info, - tiered_pricing=tiered_pricing + breakdown=breakdown, model_info=model_info, tiered_pricing=tiered_pricing ) - + return prompt_cost, completion_cost diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index a1370074238..ac3be0c3518 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -26,7 +26,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo _should_convert_tool_call_to_json_mode, ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( - strip_name_from_messages, + strip_name_from_message ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.types.llms.anthropic import AllAnthropicToolsValues @@ -43,6 +43,7 @@ from litellm.types.llms.openai import ( ChatCompletionThinkingBlock, ChatCompletionToolChoiceFunctionParam, ChatCompletionToolChoiceObjectParam, + ChatCompletionToolParam, ) from litellm.types.utils import ( ChatCompletionMessageToolCall, @@ -217,6 +218,21 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): databricks_tool = self.convert_anthropic_tool_to_databricks_tool(tool) return databricks_tool + def remove_cache_control_flag_from_messages_and_tools( + self, + model: str, # allows overrides to selectively run this + messages: List[AllMessageValues], + tools: Optional[List["ChatCompletionToolParam"]] = None, + ) -> Tuple[List[AllMessageValues], Optional[List["ChatCompletionToolParam"]]]: + """ + Override the parent class method to preserve cache_control for models on Databricks. + Databricks supports Anthropic-style cache control for Claude models. + Databricks ignores the cache_control flag with other models. + """ + # TODO: Think about how to best design the request transformation so that + # every request doesn't have to be transformed for to OpenAI and Anthropic request formats. + return messages, tools + def map_openai_params( self, non_default_params: dict, @@ -316,8 +332,11 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): _message = message.model_dump(exclude_none=True) else: _message = message + _message = strip_name_from_message(_message, allowed_name_roles=["user"]) + # Move message-level cache_control into a content block when content is a string. + if "cache_control" in _message and isinstance(_message.get("content"), str): + _message = self._move_cache_control_into_string_content_block(_message) new_messages.append(_message) - new_messages = strip_name_from_messages(new_messages) if is_async: return super()._transform_messages( @@ -328,6 +347,32 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): messages=new_messages, model=model, is_async=cast(Literal[False], False) ) + def _move_cache_control_into_string_content_block(self, message: AllMessageValues) -> AllMessageValues: + """ + Moves message-level cache_control into a content block when content is a string. + + Transforms: + {"role": "user", "content": "text", "cache_control": {...}} + Into: + {"role": "user", "content": [{"type": "text", "text": "text", "cache_control": {...}}]} + + This is required for Anthropic's prompt caching API when cache_control is specified + at the message level but content is a simple string (not already an array of content blocks). + """ + content = message.get("content") + # Create new message with cache_control moved into content block + transformed_message = cast(dict[str, Any], message.copy()) + cache_control = transformed_message.pop("cache_control") + transformed_message["content"] = [ + { + "type": "text", + "text": content, + "cache_control": cache_control, + } + ] + return cast(AllMessageValues, transformed_message) + + @staticmethod def extract_content_str( content: Optional[AllDatabricksContentValues], @@ -595,7 +640,7 @@ class DatabricksChatResponseIterator(BaseModelResponseIterator): for _tc in tool_calls: if _tc.get("function", {}).get("arguments") == "{}": _tc["function"]["arguments"] = "" # avoid invalid json - if isinstance(choice["delta"]["content"], list) and ( + if isinstance(choice["delta"].get("content"), list) and ( content := choice["delta"]["content"] ): if citations := content[0].get("citations"): diff --git a/litellm/llms/dataforseo/search/__init__.py b/litellm/llms/dataforseo/search/__init__.py new file mode 100644 index 00000000000..28990c1af3e --- /dev/null +++ b/litellm/llms/dataforseo/search/__init__.py @@ -0,0 +1,11 @@ +""" +DataForSEO Search Module + +This module provides search functionality using DataForSEO's SERP API. +DataForSEO offers comprehensive search engine data with high accuracy. +""" + +from .transformation import DataForSEOSearchConfig + +__all__ = ["DataForSEOSearchConfig"] + diff --git a/litellm/llms/dataforseo/search/transformation.py b/litellm/llms/dataforseo/search/transformation.py new file mode 100644 index 00000000000..86b472f61b8 --- /dev/null +++ b/litellm/llms/dataforseo/search/transformation.py @@ -0,0 +1,209 @@ +""" +Calls DataForSEO SERP API to search the web. + +DataForSEO API Reference: https://docs.dataforseo.com/v3/serp/google/organic/live/advanced/?bash +""" +from typing import Any, Dict, List, Literal, Optional, Union + +import httpx + +from litellm.constants import DEFAULT_DATAFORSEO_LOCATION_CODE +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) +from litellm.secret_managers.main import get_secret_str + + +class DataForSEOSearchConfig(BaseSearchConfig): + """ + Configuration for DataForSEO SERP API search. + + DataForSEO uses HTTP Basic Auth with login:password credentials. + API endpoint: https://api.dataforseo.com/v3/serp/google/organic/live/advanced + """ + + DATAFORSEO_API_BASE = "https://api.dataforseo.com/v3/serp/google/organic/live/advanced" + + @staticmethod + def ui_friendly_name() -> str: + return "DataForSEO" + + def get_http_method(self) -> Literal["GET", "POST"]: + """ + DataForSEO uses POST requests with JSON body. + """ + return "POST" + + def validate_environment( + self, + headers: Dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + **kwargs, + ) -> Dict: + """ + Validate DataForSEO environment and set up authentication. + + DataForSEO uses HTTP Basic Auth with login:password format. + The credentials should be in DATAFORSEO_LOGIN and DATAFORSEO_PASSWORD env vars, + or passed as api_key in "login:password" format. + """ + import base64 + + # Get login and password + login = get_secret_str("DATAFORSEO_LOGIN") + password = get_secret_str("DATAFORSEO_PASSWORD") + + # If api_key is provided in "login:password" format, use it + if api_key and ":" in api_key: + login, password = api_key.split(":", 1) + + if not login: + raise ValueError("DATAFORSEO_LOGIN is not set. Set `DATAFORSEO_LOGIN` environment variable or pass credentials in api_key parameter.") + + if not password: + raise ValueError("DATAFORSEO_PASSWORD is not set. Set `DATAFORSEO_PASSWORD` environment variable or pass credentials in api_key parameter.") + + # Create Basic Auth header + credentials = f"{login}:{password}" + encoded_credentials = base64.b64encode(credentials.encode()).decode() + headers["Authorization"] = f"Basic {encoded_credentials}" + headers["Content-Type"] = "application/json" + + return headers + + def get_complete_url( + self, + api_base: Optional[str], + optional_params: dict, + data: Optional[Union[Dict, List[Dict]]] = None, + **kwargs, + ) -> str: + """ + Get complete URL for DataForSEO SERP API endpoint. + + DataForSEO uses POST requests, so no query parameters in URL. + """ + return api_base or get_secret_str("DATAFORSEO_API_BASE") or self.DATAFORSEO_API_BASE + + def transform_search_request( + self, + query: Union[str, List[str]], + optional_params: dict, + api_key: Optional[str] = None, + **kwargs, + ) -> Union[Dict, List[Dict]]: + """ + Transform Search request to DataForSEO SERP API format. + + Args: + query: Search query (string or list of strings). DataForSEO supports single string queries. + optional_params: Optional parameters for the request + - max_results: Maximum number of search results → maps to `depth` (max 700) + - country: Country name → maps to `location_name` + - search_domain_filter: Domain to filter results → maps to `domain` + - Plus any DataForSEO-specific parameters (location_code, language_code, device, os, etc.) + api_key: DataForSEO credentials (login:password format) + + Returns: + List[Dict]: Request body for DataForSEO API (array of task objects as required by API) + """ + # DataForSEO expects an array of task objects + task: Dict[str, Any] = {} + + # Convert query to string if it's a list + if isinstance(query, list): + query = query[0] if query else "" + + # Required field: keyword + task["keyword"] = query + + # Map unified parameters to DataForSEO parameters + if "max_results" in optional_params and optional_params["max_results"]: + # DataForSEO uses 'depth' for number of results (max 700) + depth = min(int(optional_params["max_results"]), 700) + task["depth"] = depth + + if "country" in optional_params and optional_params["country"]: + # DataForSEO uses location_code (e.g., 2840 for USA) + # For simplicity, we'll use location_name which accepts country names + task["location_name"] = optional_params["country"] + + if "search_domain_filter" in optional_params and optional_params["search_domain_filter"]: + # DataForSEO uses 'domain' parameter to filter by domain + task["domain"] = optional_params["search_domain_filter"] + + # Add defaults if not specified + if "language_code" not in task and "language_name" not in task: + task["language_code"] = "en" + + # DataForSEO requires a location - use default from constants if not specified + if "location_code" not in task and "location_name" not in task: + task["location_code"] = DEFAULT_DATAFORSEO_LOCATION_CODE + + # Pass through all other parameters as-is + for param, value in optional_params.items(): + if param not in self.get_supported_perplexity_optional_params() and param not in task: + task[param] = value + + # DataForSEO API expects an array of tasks + return [task] + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs, + ) -> SearchResponse: + """ + Transform DataForSEO SERP API response to LiteLLM unified SearchResponse format. + + DataForSEO → LiteLLM mappings: + - tasks[0].result[*].items[*].title → SearchResult.title + - tasks[0].result[*].items[*].url → SearchResult.url + - tasks[0].result[*].items[*].description → SearchResult.snippet + - No date/last_updated fields in standard response (set to None) + + Args: + raw_response: Raw httpx response from DataForSEO API + logging_obj: Logging object for tracking + + Returns: + SearchResponse with standardized format + """ + response_json = raw_response.json() + + # Transform results to SearchResult objects + results = [] + + # DataForSEO wraps results in tasks array + if "tasks" in response_json and len(response_json["tasks"]) > 0: + task = response_json["tasks"][0] + + # Check if task was successful + if task.get("status_code") == 20000 and "result" in task: + # Result is an array, take first element + if len(task["result"]) > 0: + result = task["result"][0] + + # Items contain the actual search results + for item in result.get("items", []): + # Only process organic search results + if item.get("type") == "organic": + search_result = SearchResult( + title=item.get("title", ""), + url=item.get("url", ""), + snippet=item.get("description", ""), + date=None, # DataForSEO doesn't provide date in standard response + last_updated=None, + ) + results.append(search_result) + + return SearchResponse( + results=results, + object="search", + ) + diff --git a/litellm/llms/deepgram/audio_transcription/transformation.py b/litellm/llms/deepgram/audio_transcription/transformation.py index 0cdfd734de7..6a540d72778 100644 --- a/litellm/llms/deepgram/audio_transcription/transformation.py +++ b/litellm/llms/deepgram/audio_transcription/transformation.py @@ -58,7 +58,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): ) -> AudioTranscriptionRequestData: """ Processes the audio file input based on its type and returns AudioTranscriptionRequestData. - + For Deepgram, the binary audio data is sent directly as the request body. Args: @@ -69,12 +69,11 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): """ # Use common utility to process the audio file processed_audio = process_audio_file(audio_file) - + # Return structured data with binary content and no files # For Deepgram, we send binary data directly as request body return AudioTranscriptionRequestData( - data=processed_audio.file_content, - files=None + data=processed_audio.file_content, files=None ) def transform_audio_transcription_response( @@ -91,17 +90,32 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): first_channel = response_json["results"]["channels"][0] first_alternative = first_channel["alternatives"][0] - # Extract the full transcript - text = first_alternative["transcript"] + # Detect if diarization is active by checking if words have 'speaker' field + has_diarization = False + if "words" in first_alternative and len(first_alternative["words"]) > 0: + has_diarization = "speaker" in first_alternative["words"][0] + + # Extract the transcript based on diarization mode + if not has_diarization: + # No diarization: use the standard transcript + text = first_alternative["transcript"] + elif "paragraphs" in first_alternative: + # Diarization with paragraphs: use the pre-formatted diarized transcript + text = first_alternative["paragraphs"]["transcript"] + else: + # Diarization without paragraphs: reconstruct from words + text = self._reconstruct_diarized_transcript(first_alternative["words"]) # Create TranscriptionResponse object response = TranscriptionResponse(text=text) # Add additional metadata matching OpenAI format response["task"] = "transcribe" - response["language"] = ( - "english" # Deepgram auto-detects but doesn't return language - ) + + # Use detected_language if available, otherwise default to "en" + detected_language = first_channel.get("detected_language") + response["language"] = detected_language if detected_language else "en" + response["duration"] = response_json["metadata"]["duration"] # Transform words to match OpenAI format @@ -121,6 +135,46 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): f"Error transforming Deepgram response: {str(e)}\nResponse: {raw_response.text}" ) + def _reconstruct_diarized_transcript(self, words: list) -> str: + """ + Reconstructs a diarized transcript from words with speaker information. + + Args: + words: List of word objects with speaker, word, and optionally punctuated_word + + Returns: + Formatted transcript with speaker labels + """ + if not words: + return "" + + segments = [] + current_speaker = None + current_words: list[str] = [] + + for word_obj in words: + speaker = word_obj.get("speaker") + # Use punctuated_word if available, otherwise fall back to word + word_text = word_obj.get("punctuated_word", word_obj.get("word", "")) + + if speaker != current_speaker: + # New speaker: save previous segment and start new one + if current_words: + segments.append( + f"Speaker {current_speaker}: {' '.join(current_words)}" + ) + current_speaker = speaker + current_words = [word_text] + else: + # Same speaker: add word to current segment + current_words.append(word_text) + + # Add the last segment + if current_words: + segments.append(f"\nSpeaker {current_speaker}: {' '.join(current_words)}\n") + + return "\n".join(segments) + def get_complete_url( self, api_base: Optional[str], @@ -150,7 +204,6 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): return url - def _format_param_value(self, value) -> str: """ Formats a parameter value for use in query string. @@ -180,7 +233,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): provider_specific_params = self.get_provider_specific_params( optional_params=optional_params, model=model, - openai_params=self.get_supported_openai_params(model) + openai_params=self.get_supported_openai_params(model), ) for key, value in provider_specific_params.items(): diff --git a/litellm/llms/deepinfra/rerank/transformation.py b/litellm/llms/deepinfra/rerank/transformation.py index 69c7dabebd8..47f47418cb2 100644 --- a/litellm/llms/deepinfra/rerank/transformation.py +++ b/litellm/llms/deepinfra/rerank/transformation.py @@ -28,7 +28,12 @@ class DeepinfraRerankConfig(BaseRerankConfig): Deepinfra Rerank - Follows the same Spec as Cohere Rerank """ - def get_complete_url(self, api_base: Optional[str], model: str) -> str: + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: Optional[dict] = None, + ) -> str: """ Constructs the complete DeepInfra inference endpoint URL for rerank. @@ -63,6 +68,7 @@ class DeepinfraRerankConfig(BaseRerankConfig): headers: dict, model: str, api_key: Optional[str] = None, + optional_params: Optional[dict] = None, ) -> dict: if api_key is None: api_key = get_secret_str("DEEPINFRA_API_KEY") diff --git a/litellm/llms/exa_ai/search/__init__.py b/litellm/llms/exa_ai/search/__init__.py new file mode 100644 index 00000000000..b647d2cd80f --- /dev/null +++ b/litellm/llms/exa_ai/search/__init__.py @@ -0,0 +1,7 @@ +""" +Exa AI Search API module. +""" +from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig + +__all__ = ["ExaAISearchConfig"] + diff --git a/litellm/llms/exa_ai/search/transformation.py b/litellm/llms/exa_ai/search/transformation.py new file mode 100644 index 00000000000..6b51c6cf25d --- /dev/null +++ b/litellm/llms/exa_ai/search/transformation.py @@ -0,0 +1,188 @@ +""" +Calls Exa AI's /search endpoint to search the web. + +Exa AI API Reference: https://docs.exa.ai/reference/search +""" +from typing import Dict, List, Optional, TypedDict, Union + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) +from litellm.secret_managers.main import get_secret_str + + +class _ExaAISearchRequestRequired(TypedDict): + """Required fields for Exa AI Search API request.""" + query: str # Required - search query + + +class ExaAISearchRequest(_ExaAISearchRequestRequired, total=False): + """ + Exa AI Search API request format. + Based on: https://docs.exa.ai/reference/search + """ + type: str # Optional - search type ('keyword', 'neural', 'fast', 'auto'), default 'auto' + category: str # Optional - data category ('company', 'research paper', 'news', 'pdf', 'github', 'tweet', 'personal site', 'linkedin profile', 'financial report') + userLocation: str # Optional - two-letter ISO country code + numResults: int # Optional - number of results (max 100), default 10 + includeDomains: List[str] # Optional - list of domains to include + excludeDomains: List[str] # Optional - list of domains to exclude + startCrawlDate: str # Optional - crawl date filter (ISO 8601 format) + endCrawlDate: str # Optional - crawl date filter (ISO 8601 format) + startPublishedDate: str # Optional - published date filter (ISO 8601 format) + endPublishedDate: str # Optional - published date filter (ISO 8601 format) + includeText: List[str] # Optional - strings that must be present in webpage text + excludeText: List[str] # Optional - strings that must not be present in webpage text + context: Union[bool, dict] # Optional - format results for LLMs + moderation: bool # Optional - enable content moderation, default false + contents: dict # Optional - content retrieval options + + +class ExaAISearchConfig(BaseSearchConfig): + EXA_AI_API_BASE = "https://api.exa.ai" + + @staticmethod + def ui_friendly_name() -> str: + return "Exa AI" + + def validate_environment( + self, + headers: Dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + **kwargs, + ) -> Dict: + """ + Validate environment and return headers. + """ + api_key = api_key or get_secret_str("EXA_API_KEY") + if not api_key: + raise ValueError("EXA_API_KEY is not set. Set `EXA_API_KEY` environment variable.") + headers["x-api-key"] = api_key + headers["Content-Type"] = "application/json" + return headers + + def get_complete_url( + self, + api_base: Optional[str], + optional_params: dict, + data: Optional[Union[Dict, List[Dict]]] = None, + **kwargs, + ) -> str: + """ + Get complete URL for Search endpoint. + """ + api_base = api_base or get_secret_str("EXA_API_BASE") or self.EXA_AI_API_BASE + + # Append "/search" to the api base if it's not already there + if not api_base.endswith("/search"): + api_base = f"{api_base}/search" + + return api_base + + + def transform_search_request( + self, + query: Union[str, List[str]], + optional_params: dict, + **kwargs, + ) -> Dict: + """ + Transform Search request to Exa AI API format. + + Transforms Perplexity unified spec parameters: + - query → query (same) + - max_results → numResults + - search_domain_filter → includeDomains + - country → userLocation + - max_tokens_per_page → (not applicable, ignored) + + All other Exa-specific parameters are passed through as-is. + + Args: + query: Search query (string or list of strings). Exa AI only supports single string queries. + optional_params: Optional parameters for the request + + Returns: + Dict with typed request data following ExaAISearchRequest spec + """ + if isinstance(query, list): + # Exa AI only supports single string queries, join with spaces + query = " ".join(query) + + request_data: ExaAISearchRequest = { + "query": query, + } + + # Transform Perplexity unified spec parameters to Exa format + if "max_results" in optional_params: + request_data["numResults"] = optional_params["max_results"] + + if "search_domain_filter" in optional_params: + request_data["includeDomains"] = optional_params["search_domain_filter"] + + if "country" in optional_params: + request_data["userLocation"] = optional_params["country"] + + # Convert to dict before dynamic key assignments + result_data = dict(request_data) + + # pass through all other parameters as-is + for param, value in optional_params.items(): + if param not in self.get_supported_perplexity_optional_params() and param not in result_data: + result_data[param] = value + + # By default, request text content if not explicitly specified + # Exa AI doesn't return content/text unless explicitly requested + if "contents" not in result_data: + result_data["contents"] = {"text": True} + + return result_data + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs, + ) -> SearchResponse: + """ + Transform Exa AI API response to LiteLLM unified SearchResponse format. + + Exa AI → LiteLLM mappings: + - results[].title → SearchResult.title + - results[].url → SearchResult.url + - results[].text → SearchResult.snippet + - results[].publishedDate → SearchResult.date + - No last_updated field in Exa AI response (set to None) + + Args: + raw_response: Raw httpx response from Exa AI API + logging_obj: Logging object for tracking + + Returns: + SearchResponse with standardized format + """ + response_json = raw_response.json() + + # Transform results to SearchResult objects + results = [] + for result in response_json.get("results", []): + search_result = SearchResult( + title=result.get("title", ""), + url=result.get("url", ""), + snippet=result.get("text", ""), # Exa AI uses "text" for content + date=result.get("publishedDate"), # ISO 8601 datetime string + last_updated=None, # Exa AI doesn't provide last_updated in response + ) + results.append(search_result) + + return SearchResponse( + results=results, + object="search", + ) + diff --git a/litellm/llms/fal_ai/__init__.py b/litellm/llms/fal_ai/__init__.py new file mode 100644 index 00000000000..1f4cbe0e9ce --- /dev/null +++ b/litellm/llms/fal_ai/__init__.py @@ -0,0 +1,26 @@ +from .cost_calculator import cost_calculator +from .image_generation import ( + FalAIBaseConfig, + FalAIBriaConfig, + FalAIFluxProV11UltraConfig, + FalAIFluxSchnellConfig, + FalAIImageGenerationConfig, + FalAIImagen4Config, + FalAIRecraftV3Config, + FalAIStableDiffusionConfig, + get_fal_ai_image_generation_config, +) + +__all__ = [ + "cost_calculator", + "FalAIBaseConfig", + "FalAIImageGenerationConfig", + "FalAIImagen4Config", + "FalAIRecraftV3Config", + "FalAIBriaConfig", + "FalAIFluxProV11UltraConfig", + "FalAIFluxSchnellConfig", + "FalAIStableDiffusionConfig", + "get_fal_ai_image_generation_config", +] + diff --git a/litellm/llms/fal_ai/cost_calculator.py b/litellm/llms/fal_ai/cost_calculator.py new file mode 100644 index 00000000000..b7caae3834f --- /dev/null +++ b/litellm/llms/fal_ai/cost_calculator.py @@ -0,0 +1,26 @@ +from typing import Any + +import litellm +from litellm.types.utils import ImageResponse + + +def cost_calculator( + model: str, + image_response: Any, +) -> float: + """ + fal.ai image generation cost calculator + """ + _model_info = litellm.get_model_info( + model=model, + custom_llm_provider=litellm.LlmProviders.FAL_AI.value, + ) + output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0 + num_images: int = 0 + if isinstance(image_response, ImageResponse): + if image_response.data: + num_images = len(image_response.data) + return output_cost_per_image * num_images + else: + raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}") + diff --git a/litellm/llms/fal_ai/image_generation/__init__.py b/litellm/llms/fal_ai/image_generation/__init__.py new file mode 100644 index 00000000000..b4ae6734c64 --- /dev/null +++ b/litellm/llms/fal_ai/image_generation/__init__.py @@ -0,0 +1,53 @@ +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) + +from .bria_transformation import FalAIBriaConfig +from .flux_pro_v11_ultra_transformation import FalAIFluxProV11UltraConfig +from .flux_schnell_transformation import FalAIFluxSchnellConfig +from .imagen4_transformation import FalAIImagen4Config +from .recraft_v3_transformation import FalAIRecraftV3Config +from .stable_diffusion_transformation import FalAIStableDiffusionConfig +from .transformation import FalAIBaseConfig, FalAIImageGenerationConfig + +__all__ = [ + "FalAIBaseConfig", + "FalAIImageGenerationConfig", + "FalAIImagen4Config", + "FalAIRecraftV3Config", + "FalAIBriaConfig", + "FalAIFluxProV11UltraConfig", + "FalAIFluxSchnellConfig", + "FalAIStableDiffusionConfig", +] + + +def get_fal_ai_image_generation_config(model: str) -> BaseImageGenerationConfig: + """ + Get the appropriate Fal AI image generation configuration based on the model. + + Args: + model: The Fal AI model name (e.g., "fal-ai/imagen4/preview", "fal-ai/recraft/v3/text-to-image") + + Returns: + The appropriate configuration class for the specified model + """ + model_lower = model.lower() + + # Map model names to their corresponding configuration classes + if "imagen4" in model_lower or "imagen-4" in model_lower: + return FalAIImagen4Config() + elif "recraft" in model_lower: + return FalAIRecraftV3Config() + elif "bria" in model_lower: + return FalAIBriaConfig() + elif "flux-pro" in model_lower and "ultra" in model_lower: + return FalAIFluxProV11UltraConfig() + elif "flux/schnell" in model_lower or "flux-schnell" in model_lower or "schnell" in model_lower: + return FalAIFluxSchnellConfig() + elif "stable-diffusion" in model_lower: + return FalAIStableDiffusionConfig() + + # Default to generic Fal AI configuration + return FalAIImageGenerationConfig() + diff --git a/litellm/llms/fal_ai/image_generation/bria_transformation.py b/litellm/llms/fal_ai/image_generation/bria_transformation.py new file mode 100644 index 00000000000..cb5aa6b761d --- /dev/null +++ b/litellm/llms/fal_ai/image_generation/bria_transformation.py @@ -0,0 +1,231 @@ +from typing import TYPE_CHECKING, Any, List, Optional + +import httpx + +from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams +from litellm.types.utils import ImageObject, ImageResponse + +from .transformation import FalAIBaseConfig + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class FalAIBriaConfig(FalAIBaseConfig): + """ + Configuration for Bria Text-to-Image 3.2 model. + + Bria 3.2 is a commercial-grade text-to-image model with prompt enhancement + and multiple aspect ratio options. + + Model endpoint: bria/text-to-image/3.2 + Documentation: https://fal.ai/models/bria/text-to-image/3.2 + """ + IMAGE_GENERATION_ENDPOINT: str = "bria/text-to-image/3.2" + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + """ + Get supported OpenAI parameters for Bria 3.2. + """ + return [ + "n", + "response_format", + "size", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to Bria 3.2 parameters. + + Mappings: + - size -> aspect_ratio (1:1, 2:3, 3:2, 3:4, 4:3, 4:5, 5:4, 9:16, 16:9) + - response_format -> ignored (Bria returns URLs) + - n -> ignored (Bria doesn't support multiple images in one call) + """ + supported_params = self.get_supported_openai_params(model) + + # Map OpenAI params to Bria params + param_mapping = { + "size": "aspect_ratio", + } + + for k in non_default_params.keys(): + if k not in optional_params.keys(): + if k in supported_params: + # Use mapped parameter name if exists + mapped_key = param_mapping.get(k, k) + mapped_value = non_default_params[k] + + # Transform specific parameters + if k == "response_format": + # Bria always returns URLs, so we can ignore this + continue + elif k == "n": + # Bria doesn't support multiple images, ignore + continue + elif k == "size": + # Map OpenAI size format to Bria aspect ratio + mapped_value = self._map_aspect_ratio(mapped_value) + + optional_params[mapped_key] = mapped_value + elif drop_params: + pass + else: + raise ValueError( + f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters." + ) + + return optional_params + + def _map_aspect_ratio(self, size: str) -> str: + """ + Map OpenAI size format to Bria aspect ratio format. + + OpenAI format: "1024x1024", "1792x1024", etc. + Bria format: "1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9" + """ + # Map common OpenAI sizes to Bria aspect ratios + size_to_aspect_ratio = { + "1024x1024": "1:1", + "512x512": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1024x768": "4:3", + "768x1024": "3:4", + "1280x960": "4:3", + "960x1280": "3:4", + } + + if size in size_to_aspect_ratio: + return size_to_aspect_ratio[size] + + # Parse custom size format "WIDTHxHEIGHT" and calculate aspect ratio + if "x" in size: + try: + width_str, height_str = size.split("x") + width = int(width_str) + height = int(height_str) + + # Calculate aspect ratio and find closest match + ratio = width / height + + # Map to closest supported aspect ratio + if 0.95 <= ratio <= 1.05: # Close to 1:1 + return "1:1" + elif ratio >= 1.7: # Close to 16:9 + return "16:9" + elif ratio <= 0.6: # Close to 9:16 + return "9:16" + elif 1.3 <= ratio <= 1.4: # Close to 4:3 + return "4:3" + elif 0.7 <= ratio <= 0.8: # Close to 3:4 + return "3:4" + elif 1.45 <= ratio <= 1.55: # Close to 3:2 + return "3:2" + elif 0.65 <= ratio <= 0.7: # Close to 2:3 + return "2:3" + elif 1.2 <= ratio <= 1.3: # Close to 5:4 + return "5:4" + elif 0.75 <= ratio <= 0.85: # Close to 4:5 + return "4:5" + except (ValueError, AttributeError, ZeroDivisionError): + pass + + # Default to 1:1 + return "1:1" + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the image generation request to Bria 3.2 request body. + + Required parameters: + - prompt: Prompt for image generation + + Optional parameters: + - aspect_ratio: "1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9" (default: "1:1") + - prompt_enhancer: Improve the prompt (default: true) + - sync_mode: Return image directly in response (default: false) + - truncate_prompt: Truncate the prompt (default: true) + - guidance_scale: Guidance scale 1-10 (default: 5) + - num_inference_steps: Inference steps 20-50 (default: 30) + - seed: Random seed for reproducibility (default: 5555) + - negative_prompt: Negative prompt string + """ + bria_request_body = { + "prompt": prompt, + **optional_params, + } + + return bria_request_body + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + """ + Transform the Bria 3.2 response to litellm ImageResponse format. + + Expected response format: + { + "image": { + "url": "https://...", + "content_type": "image/png", + "file_name": "...", + "file_size": 123456, + "width": 1024, + "height": 1024 + } + } + """ + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error transforming image generation response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + if not model_response.data: + model_response.data = [] + + # Handle Bria response format - uses "image" (singular) not "images" + image_data = response_data.get("image") + if image_data and isinstance(image_data, dict): + model_response.data.append( + ImageObject( + url=image_data.get("url", None), + b64_json=None, # Bria returns URLs only + ) + ) + + return model_response + diff --git a/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py new file mode 100644 index 00000000000..664f11d40dc --- /dev/null +++ b/litellm/llms/fal_ai/image_generation/flux_pro_v11_ultra_transformation.py @@ -0,0 +1,263 @@ +from typing import TYPE_CHECKING, Any, List, Optional + +import httpx + +from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams +from litellm.types.utils import ImageObject, ImageResponse + +from .transformation import FalAIBaseConfig + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class FalAIFluxProV11UltraConfig(FalAIBaseConfig): + """ + Configuration for Fal AI Flux Pro v1.1-ultra model. + + FLUX Pro v1.1-ultra is a high-quality text-to-image model with enhanced detail + and support for image prompts. + + Model endpoint: fal-ai/flux-pro/v1.1-ultra + Documentation: https://fal.ai/models/fal-ai/flux-pro/v1.1-ultra + """ + IMAGE_GENERATION_ENDPOINT: str = "fal-ai/flux-pro/v1.1-ultra" + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + """ + Get supported OpenAI parameters for Flux Pro v1.1-ultra. + """ + return [ + "n", + "response_format", + "size", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to Flux Pro v1.1-ultra parameters. + + Mappings: + - n -> num_images (1-4, default 1) + - response_format -> output_format (jpeg or png) + - size -> aspect_ratio (21:9, 16:9, 4:3, 3:2, 1:1, 2:3, 3:4, 9:16, 9:21) + """ + supported_params = self.get_supported_openai_params(model) + + # Map OpenAI params to Flux Pro v1.1-ultra params + param_mapping = { + "n": "num_images", + "response_format": "output_format", + "size": "aspect_ratio", + } + + for k in non_default_params.keys(): + if k not in optional_params.keys(): + if k in supported_params: + # Use mapped parameter name if exists + mapped_key = param_mapping.get(k, k) + mapped_value = non_default_params[k] + + # Transform specific parameters + if k == "response_format": + # Map OpenAI response formats to image formats + if mapped_value in ["b64_json", "url"]: + mapped_value = "jpeg" + elif k == "size": + # Map OpenAI size format to Flux aspect ratio + mapped_value = self._map_aspect_ratio(mapped_value) + + optional_params[mapped_key] = mapped_value + elif drop_params: + pass + else: + raise ValueError( + f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters." + ) + + return optional_params + + def _map_aspect_ratio(self, size: str) -> str: + """ + Map OpenAI size format to Flux Pro aspect ratio format. + + OpenAI format: "1024x1024", "1792x1024", etc. + Flux format: "21:9", "16:9", "4:3", "3:2", "1:1", "2:3", "3:4", "9:16", "9:21" + + Default: "16:9" + """ + # Map common OpenAI sizes to Flux aspect ratios + size_to_aspect_ratio = { + "1024x1024": "1:1", + "512x512": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1024x768": "4:3", + "768x1024": "3:4", + "1536x1024": "3:2", + "1024x1536": "2:3", + "2048x876": "21:9", + "876x2048": "9:21", + } + + if size in size_to_aspect_ratio: + return size_to_aspect_ratio[size] + + # Parse custom size format "WIDTHxHEIGHT" and calculate aspect ratio + if "x" in size: + try: + width_str, height_str = size.split("x") + width = int(width_str) + height = int(height_str) + + # Calculate aspect ratio and find closest match + ratio = width / height + + # Map to closest supported aspect ratio + if 0.95 <= ratio <= 1.05: # Close to 1:1 + return "1:1" + elif ratio >= 2.3: # Close to 21:9 + return "21:9" + elif 1.7 <= ratio < 2.3: # Close to 16:9 + return "16:9" + elif 1.3 <= ratio < 1.7: # Close to 4:3 + return "4:3" + elif 1.4 <= ratio < 1.6: # Close to 3:2 + return "3:2" + elif 0.6 <= ratio < 0.7: # Close to 3:4 + return "3:4" + elif 0.65 <= ratio < 0.75: # Close to 2:3 + return "2:3" + elif 0.5 <= ratio < 0.6: # Close to 9:16 + return "9:16" + elif ratio < 0.5: # Close to 9:21 + return "9:21" + except (ValueError, AttributeError, ZeroDivisionError): + pass + + # Default to 16:9 + return "16:9" + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the image generation request to Flux Pro v1.1-ultra request body. + + Required parameters: + - prompt: The prompt to generate an image from + + Optional parameters: + - num_images: Number of images (1-4, default: 1) + - aspect_ratio: Aspect ratio (default: "16:9") + - raw: Generate less processed images (default: false) + - output_format: "jpeg" or "png" (default: "jpeg") + - image_url: Image URL for image-to-image generation + - sync_mode: Return data URI (default: false) + - safety_tolerance: Safety level "1"-"6" (default: "2") + - enable_safety_checker: Enable safety checker (default: true) + - seed: Random seed for reproducibility + - image_prompt_strength: Strength of image prompt 0-1 (default: 0.1) + - enhance_prompt: Enhance prompt for better results (default: false) + """ + flux_pro_request_body = { + "prompt": prompt, + **optional_params, + } + + return flux_pro_request_body + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + """ + Transform the Flux Pro v1.1-ultra response to litellm ImageResponse format. + + Expected response format: + { + "images": [ + { + "url": "https://...", + "width": 1024, + "height": 768, + "content_type": "image/jpeg" + } + ], + "timings": {"inference": 2.5, ...}, + "seed": 42, + "has_nsfw_concepts": [false], + "prompt": "original prompt" + } + """ + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error transforming image generation response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + if not model_response.data: + model_response.data = [] + + # Handle Flux Pro v1.1-ultra response format + images = response_data.get("images", []) + if isinstance(images, list): + for image_data in images: + if isinstance(image_data, dict): + model_response.data.append( + ImageObject( + url=image_data.get("url", None), + b64_json=None, # Flux Pro returns URLs only + ) + ) + elif isinstance(image_data, str): + # If images is just a list of URLs + model_response.data.append( + ImageObject( + url=image_data, + b64_json=None, + ) + ) + + # Add additional metadata from Flux Pro response + if hasattr(model_response, "_hidden_params"): + if "seed" in response_data: + model_response._hidden_params["seed"] = response_data["seed"] + if "timings" in response_data: + model_response._hidden_params["timings"] = response_data["timings"] + if "has_nsfw_concepts" in response_data: + model_response._hidden_params["has_nsfw_concepts"] = response_data[ + "has_nsfw_concepts" + ] + + return model_response + diff --git a/litellm/llms/fal_ai/image_generation/flux_schnell_transformation.py b/litellm/llms/fal_ai/image_generation/flux_schnell_transformation.py new file mode 100644 index 00000000000..ed6ed37fb44 --- /dev/null +++ b/litellm/llms/fal_ai/image_generation/flux_schnell_transformation.py @@ -0,0 +1,88 @@ +from typing import Any + +from .flux_pro_v11_ultra_transformation import FalAIFluxProV11UltraConfig + + +class FalAIFluxSchnellConfig(FalAIFluxProV11UltraConfig): + """ + Configuration for Fal AI Flux Schnell model. + + Flux Schnell shares the same response format as Flux Pro models but expects + the OpenAI `size` parameter to be translated into Fal AI's `image_size` + enum/object. + + Model endpoint: fal-ai/flux/schnell + Documentation: https://fal.ai/models/fal-ai/flux/schnell + """ + + IMAGE_GENERATION_ENDPOINT: str = "fal-ai/flux/schnell" + + _OPENAI_SIZE_TO_IMAGE_SIZE = { + "1024x1024": "square_hd", + "512x512": "square", + "1792x1024": "landscape_16_9", + "1024x1792": "portrait_16_9", + "1024x768": "landscape_4_3", + "768x1024": "portrait_4_3", + } + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + + param_mapping = { + "n": "num_images", + "response_format": "output_format", + "size": "image_size", + } + + for k in non_default_params.keys(): + if k not in optional_params.keys(): + if k in supported_params: + mapped_key = param_mapping.get(k, k) + mapped_value = non_default_params[k] + + if k == "response_format": + if mapped_value in ["b64_json", "url"]: + mapped_value = "jpeg" + elif k == "size": + mapped_value = self._map_image_size(mapped_value) + + optional_params[mapped_key] = mapped_value + elif drop_params: + continue + else: + raise ValueError( + f"Parameter {k} is not supported for model {model}. " + f"Supported parameters are {supported_params}. " + "Set drop_params=True to drop unsupported parameters." + ) + + return optional_params + + def _map_image_size(self, size: Any) -> Any: + if isinstance(size, dict): + return size + + if not isinstance(size, str): + return size + + if size in self._OPENAI_SIZE_TO_IMAGE_SIZE: + return self._OPENAI_SIZE_TO_IMAGE_SIZE[size] + + if "x" in size: + try: + width_str, height_str = size.split("x") + width = int(width_str) + height = int(height_str) + return {"width": width, "height": height} + except (ValueError, AttributeError, ZeroDivisionError): + pass + + return "landscape_4_3" + diff --git a/litellm/llms/fal_ai/image_generation/imagen4_transformation.py b/litellm/llms/fal_ai/image_generation/imagen4_transformation.py new file mode 100644 index 00000000000..4e7708c9f40 --- /dev/null +++ b/litellm/llms/fal_ai/image_generation/imagen4_transformation.py @@ -0,0 +1,242 @@ +from typing import TYPE_CHECKING, Any, List, Optional + +import httpx + +from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams +from litellm.types.utils import ImageObject, ImageResponse + +from .transformation import FalAIBaseConfig + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class FalAIImagen4Config(FalAIBaseConfig): + """ + Configuration for Fal AI Imagen4 model. + + Google's highest quality image generation model available through Fal AI. + + Model variants: + - fal-ai/imagen4/preview (Standard): $0.05 per image + - fal-ai/imagen4/preview/fast (Fast): $0.02 per image + - fal-ai/imagen4/preview/ultra (Ultra): $0.06 per image + + Documentation: https://fal.ai/models/fal-ai/imagen4/preview + """ + IMAGE_GENERATION_ENDPOINT: str = "fal-ai/imagen4/preview" + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + """ + Get supported OpenAI parameters for Imagen4. + """ + return [ + "n", + "response_format", + "size", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to Imagen4 parameters. + + Mappings: + - n -> num_images (1-4, default 1) + - size -> aspect_ratio (1:1, 16:9, 9:16, 3:4, 4:3) + - response_format -> ignored (Imagen4 returns URLs) + """ + supported_params = self.get_supported_openai_params(model) + + # Map OpenAI params to Imagen4 params + param_mapping = { + "n": "num_images", + "size": "aspect_ratio", + } + + for k in non_default_params.keys(): + if k not in optional_params.keys(): + if k in supported_params: + # Use mapped parameter name if exists + mapped_key = param_mapping.get(k, k) + mapped_value = non_default_params[k] + + # Transform specific parameters + if k == "response_format": + # Imagen4 always returns URLs, so we can ignore this + continue + elif k == "size": + # Map OpenAI size format to Imagen4 aspect ratio + mapped_value = self._map_aspect_ratio(mapped_value) + + optional_params[mapped_key] = mapped_value + elif drop_params: + pass + else: + raise ValueError( + f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters." + ) + + return optional_params + + def _map_aspect_ratio(self, size: str) -> str: + """ + Map OpenAI size format to Imagen4 aspect ratio format. + + OpenAI format: "1024x1024", "1792x1024", etc. + Imagen4 format: "1:1", "16:9", "9:16", "3:4", "4:3" + + Available aspect ratios: + - 1:1 (default) + - 16:9 + - 9:16 + - 3:4 + - 4:3 + """ + # Map common OpenAI sizes to Imagen4 aspect ratios + size_to_aspect_ratio = { + "1024x1024": "1:1", + "512x512": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1024x768": "4:3", + "768x1024": "3:4", + } + + if size in size_to_aspect_ratio: + return size_to_aspect_ratio[size] + + # Parse custom size format "WIDTHxHEIGHT" and calculate aspect ratio + if "x" in size: + try: + width_str, height_str = size.split("x") + width = int(width_str) + height = int(height_str) + + # Calculate aspect ratio and find closest match + ratio = width / height + + # Map to closest supported aspect ratio + if 0.95 <= ratio <= 1.05: # Close to 1:1 + return "1:1" + elif ratio >= 1.7: # Close to 16:9 + return "16:9" + elif ratio <= 0.6: # Close to 9:16 + return "9:16" + elif ratio >= 1.2: # Close to 4:3 + return "4:3" + elif ratio <= 0.8: # Close to 3:4 + return "3:4" + except (ValueError, AttributeError, ZeroDivisionError): + pass + + # Default to 1:1 + return "1:1" + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the image generation request to Imagen4 request body. + + Required parameters: + - prompt: The text prompt describing what you want to see + + Optional parameters: + - aspect_ratio: "1:1", "16:9", "9:16", "3:4", "4:3" (default: "1:1") + - num_images: Number of images (1-4, default: 1) + - resolution: "1K" or "2K" (default: "1K") + - seed: Random seed for reproducibility + - negative_prompt: Description of what to discourage (default: "") + """ + imagen4_request_body = { + "prompt": prompt, + **optional_params, + } + + return imagen4_request_body + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + """ + Transform the Imagen4 response to litellm ImageResponse format. + + Expected response format: + { + "images": [ + { + "url": "https://...", + "content_type": "image/png", + "file_name": "z9RV14K95DvU.png", + "file_size": 4404019 + } + ], + "seed": 42 + } + """ + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error transforming image generation response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + if not model_response.data: + model_response.data = [] + + # Handle Imagen4 response format + images = response_data.get("images", []) + if isinstance(images, list): + for image_data in images: + if isinstance(image_data, dict): + model_response.data.append( + ImageObject( + url=image_data.get("url", None), + b64_json=None, # Imagen4 returns URLs only + ) + ) + elif isinstance(image_data, str): + # If images is just a list of URLs + model_response.data.append( + ImageObject( + url=image_data, + b64_json=None, + ) + ) + + # Add seed metadata from Imagen4 response + if hasattr(model_response, "_hidden_params"): + if "seed" in response_data: + model_response._hidden_params["seed"] = response_data["seed"] + + return model_response + diff --git a/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py new file mode 100644 index 00000000000..572a8a0f1c3 --- /dev/null +++ b/litellm/llms/fal_ai/image_generation/recraft_v3_transformation.py @@ -0,0 +1,226 @@ +from typing import TYPE_CHECKING, Any, List, Optional + +import httpx + +from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams +from litellm.types.utils import ImageObject, ImageResponse + +from .transformation import FalAIBaseConfig + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class FalAIRecraftV3Config(FalAIBaseConfig): + """ + Configuration for Fal AI Recraft v3 Text-to-Image model. + + Recraft v3 is a text-to-image model with multiple style options including + realistic images, digital illustrations, and vector illustrations. + + Model endpoint: fal-ai/recraft/v3/text-to-image + Documentation: https://fal.ai/models/fal-ai/recraft/v3/text-to-image + """ + IMAGE_GENERATION_ENDPOINT: str = "fal-ai/recraft/v3/text-to-image" + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + """ + Get supported OpenAI parameters for Recraft v3. + """ + return [ + "n", + "response_format", + "size", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to Recraft v3 parameters. + + Mappings: + - size -> image_size (can be preset or custom width/height) + - response_format -> ignored (Recraft returns URLs) + - n -> ignored (Recraft doesn't support multiple images) + """ + supported_params = self.get_supported_openai_params(model) + + # Map OpenAI params to Recraft v3 params + param_mapping = { + "size": "image_size", + } + + for k in non_default_params.keys(): + if k not in optional_params.keys(): + if k in supported_params: + # Use mapped parameter name if exists + mapped_key = param_mapping.get(k, k) + mapped_value = non_default_params[k] + + # Transform specific parameters + if k == "response_format": + # Recraft always returns URLs, so we can ignore this + continue + elif k == "n": + # Recraft doesn't support multiple images, ignore + continue + elif k == "size": + # Map OpenAI size format to Recraft image_size + mapped_value = self._map_image_size(mapped_value) + + optional_params[mapped_key] = mapped_value + elif drop_params: + pass + else: + raise ValueError( + f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters." + ) + + return optional_params + + def _map_image_size(self, size: str) -> Any: + """ + Map OpenAI size format to Recraft v3 image_size format. + + OpenAI format: "1024x1024", "1792x1024", etc. + Recraft format: Can be preset strings or {"width": int, "height": int} + + Available presets: + - square_hd (default) + - square + - portrait_4_3 + - portrait_16_9 + - landscape_4_3 + - landscape_16_9 + """ + # Map common OpenAI sizes to Recraft presets + size_mapping = { + "1024x1024": "square_hd", + "512x512": "square", + "768x1024": "portrait_4_3", + "576x1024": "portrait_16_9", + "1024x768": "landscape_4_3", + "1024x576": "landscape_16_9", + } + + if size in size_mapping: + return size_mapping[size] + + # Parse custom size format "WIDTHxHEIGHT" + if "x" in size: + try: + width, height = size.split("x") + return { + "width": int(width), + "height": int(height), + } + except (ValueError, AttributeError): + pass + + # Default to square_hd + return "square_hd" + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the image generation request to Recraft v3 request body. + + Required parameters: + - prompt: Text prompt (max 1000 characters) + + Optional parameters: + - image_size: Preset or {"width": int, "height": int} (default: "square_hd") + - style: Style preset (default: "realistic_image") + Options: "any", "realistic_image", "digital_illustration", "vector_illustration", etc. + - colors: Array of RGB color objects [{"r": 0-255, "g": 0-255, "b": 0-255}] + - enable_safety_checker: Enable safety checker (default: false) + - style_id: UUID for custom style reference + + Note: Vector illustrations cost 2X as much. + """ + recraft_request_body = { + "prompt": prompt, + **optional_params, + } + + return recraft_request_body + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + """ + Transform the Recraft v3 response to litellm ImageResponse format. + + Expected response format: + { + "images": [ + { + "url": "https://...", + "content_type": "image/webp", + "file_name": "...", + "file_size": 123456 + } + ] + } + """ + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error transforming image generation response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + if not model_response.data: + model_response.data = [] + + # Handle Recraft v3 response format + images = response_data.get("images", []) + if isinstance(images, list): + for image_data in images: + if isinstance(image_data, dict): + model_response.data.append( + ImageObject( + url=image_data.get("url", None), + b64_json=None, # Recraft returns URLs only + ) + ) + elif isinstance(image_data, str): + # If images is just a list of URLs + model_response.data.append( + ImageObject( + url=image_data, + b64_json=None, + ) + ) + + return model_response + diff --git a/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py new file mode 100644 index 00000000000..10e2c6b4161 --- /dev/null +++ b/litellm/llms/fal_ai/image_generation/stable_diffusion_transformation.py @@ -0,0 +1,281 @@ +from typing import TYPE_CHECKING, Any, List, Optional + +import httpx + +from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams +from litellm.types.utils import ImageObject, ImageResponse + +from .transformation import FalAIBaseConfig + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class FalAIStableDiffusionConfig(FalAIBaseConfig): + """ + Configuration for Fal AI Stable Diffusion models. + + Supports Stable Diffusion v3.5 variants and other Stable Diffusion models on Fal AI. + + Example models: + - fal-ai/stable-diffusion-v35-medium + - fal-ai/stable-diffusion-v35-large + + Documentation: https://fal.ai/models/fal-ai/stable-diffusion-v35-medium + """ + IMAGE_GENERATION_ENDPOINT: str = "" # Will be set from model name + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete url for the request. + + For Stable Diffusion models, extract the endpoint from the model name. + """ + from litellm.secret_managers.main import get_secret_str + + complete_url: str = ( + api_base + or get_secret_str("FAL_AI_API_BASE") + or self.DEFAULT_BASE_URL + ) + + complete_url = complete_url.rstrip("/") + + # Extract endpoint from model name + # e.g., "fal-ai/stable-diffusion-v35-medium" or "stable-diffusion-v35-medium" + endpoint = model + if "/" in model and not model.startswith("fal-ai/"): + # If model is like "custom/stable-diffusion-v35-medium", use full path + endpoint = model + elif not model.startswith("fal-ai/"): + # If model is just "stable-diffusion-v35-medium", prepend fal-ai + endpoint = f"fal-ai/{model}" + + complete_url = f"{complete_url}/{endpoint}" + return complete_url + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + """ + Get supported OpenAI parameters for Stable Diffusion models. + """ + return [ + "n", + "response_format", + "size", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to Stable Diffusion parameters. + + Mappings: + - n -> num_images (1-4, default 1) + - response_format -> output_format (jpeg or png) + - size -> image_size (can be preset or custom width/height) + """ + supported_params = self.get_supported_openai_params(model) + + # Map OpenAI params to Stable Diffusion params + param_mapping = { + "n": "num_images", + "response_format": "output_format", + "size": "image_size", + } + + for k in non_default_params.keys(): + if k not in optional_params.keys(): + if k in supported_params: + # Use mapped parameter name if exists + mapped_key = param_mapping.get(k, k) + mapped_value = non_default_params[k] + + # Transform specific parameters + if k == "response_format": + # Map OpenAI response formats to image formats + if mapped_value in ["b64_json", "url"]: + mapped_value = "jpeg" + elif k == "size": + # Map OpenAI size format to Stable Diffusion image_size + mapped_value = self._map_image_size(mapped_value) + + optional_params[mapped_key] = mapped_value + elif drop_params: + pass + else: + raise ValueError( + f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters." + ) + + return optional_params + + def _map_image_size(self, size: str) -> Any: + """ + Map OpenAI size format to Stable Diffusion image_size format. + + OpenAI format: "1024x1024", "1792x1024", etc. + Stable Diffusion format: Can be preset strings or {"width": int, "height": int} + + Available presets: + - square_hd + - square + - portrait_4_3 + - portrait_16_9 + - landscape_4_3 (default) + - landscape_16_9 + """ + # Map common OpenAI sizes to Stable Diffusion presets + size_mapping = { + "1024x1024": "square_hd", + "512x512": "square", + "768x1024": "portrait_4_3", + "576x1024": "portrait_16_9", + "1024x768": "landscape_4_3", + "1024x576": "landscape_16_9", + } + + if size in size_mapping: + return size_mapping[size] + + # Parse custom size format "WIDTHxHEIGHT" + if "x" in size: + try: + width, height = size.split("x") + return { + "width": int(width), + "height": int(height), + } + except (ValueError, AttributeError): + pass + + # Default to landscape_4_3 + return "landscape_4_3" + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the image generation request to Stable Diffusion request body. + + Required parameters: + - prompt: The prompt to generate an image from + + Optional parameters: + - num_images: Number of images (1-4, default: 1) + - image_size: Size preset or {"width": int, "height": int} (default: landscape_4_3) + - output_format: "jpeg" or "png" (default: jpeg) + - sync_mode: Wait for image upload before returning (default: false) + - guidance_scale: CFG scale 0-20 (default: 4.5) + - num_inference_steps: Inference steps 1-50 (default: 40) + - seed: Random seed for reproducibility + - negative_prompt: Negative prompt string (default: "") + - enable_safety_checker: Enable safety checker (default: true) + """ + stable_diffusion_request_body = { + "prompt": prompt, + **optional_params, + } + + return stable_diffusion_request_body + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + """ + Transform the Stable Diffusion response to litellm ImageResponse format. + + Expected response format: + { + "images": [ + { + "url": "https://...", + "width": 1024, + "height": 768, + "content_type": "image/jpeg" + } + ], + "timings": {"inference": 2.5, ...}, + "seed": 42, + "has_nsfw_concepts": [false], + "prompt": "original prompt" + } + """ + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error transforming image generation response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + if not model_response.data: + model_response.data = [] + + # Handle Stable Diffusion response format + images = response_data.get("images", []) + if isinstance(images, list): + for image_data in images: + if isinstance(image_data, dict): + model_response.data.append( + ImageObject( + url=image_data.get("url", None), + b64_json=None, # Stable Diffusion returns URLs only + ) + ) + elif isinstance(image_data, str): + # If images is just a list of URLs + model_response.data.append( + ImageObject( + url=image_data, + b64_json=None, + ) + ) + + # Add additional metadata from Stable Diffusion response + if hasattr(model_response, "_hidden_params"): + if "seed" in response_data: + model_response._hidden_params["seed"] = response_data["seed"] + if "timings" in response_data: + model_response._hidden_params["timings"] = response_data["timings"] + if "has_nsfw_concepts" in response_data: + model_response._hidden_params["has_nsfw_concepts"] = response_data[ + "has_nsfw_concepts" + ] + + return model_response + diff --git a/litellm/llms/fal_ai/image_generation/transformation.py b/litellm/llms/fal_ai/image_generation/transformation.py new file mode 100644 index 00000000000..04b7b167523 --- /dev/null +++ b/litellm/llms/fal_ai/image_generation/transformation.py @@ -0,0 +1,176 @@ +from typing import TYPE_CHECKING, Any, List, Optional + +import httpx + +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIImageGenerationOptionalParams, +) +from litellm.types.utils import ImageObject, ImageResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class FalAIBaseConfig(BaseImageGenerationConfig): + """ + Base configuration for Fal AI image generation models. + Handles common functionality like URL construction and authentication. + """ + DEFAULT_BASE_URL: str = "https://fal.run" + IMAGE_GENERATION_ENDPOINT: str = "" + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete url for the request + + Some providers need `model` in `api_base` + """ + complete_url: str = ( + api_base + or get_secret_str("FAL_AI_API_BASE") + or self.DEFAULT_BASE_URL + ) + + complete_url = complete_url.rstrip("/") + if self.IMAGE_GENERATION_ENDPOINT: + complete_url = f"{complete_url}/{self.IMAGE_GENERATION_ENDPOINT}" + return complete_url + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + final_api_key: Optional[str] = ( + api_key or + get_secret_str("FAL_AI_API_KEY") + ) + if not final_api_key: + raise ValueError("FAL_AI_API_KEY is not set") + + headers["Authorization"] = f"Key {final_api_key}" + return headers + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + """ + Transform the image generation response to the litellm image response + """ + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error transforming image generation response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + if not model_response.data: + model_response.data = [] + + # Handle fal.ai response format + images = response_data.get("images", []) + if isinstance(images, list): + for image_data in images: + if isinstance(image_data, dict): + model_response.data.append(ImageObject( + url=image_data.get("url", None), + b64_json=image_data.get("b64_json", None), + )) + elif isinstance(image_data, str): + # If images is just a list of URLs + model_response.data.append(ImageObject( + url=image_data, + b64_json=None, + )) + + return model_response + + +class FalAIImageGenerationConfig(FalAIBaseConfig): + """ + Default Fal AI image generation configuration for generic models. + """ + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + """ + Get supported OpenAI parameters for fal.ai image generation + """ + return [ + "n", + "response_format", + "size", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + for k in non_default_params.keys(): + if k not in optional_params.keys(): + if k in supported_params: + optional_params[k] = non_default_params[k] + elif drop_params: + pass + else: + raise ValueError( + f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters." + ) + + return optional_params + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the image generation request to the fal.ai image generation request body + """ + fal_ai_image_generation_request_body = { + "prompt": prompt, + **optional_params, + } + return fal_ai_image_generation_request_body + diff --git a/litellm/llms/firecrawl/__init__.py b/litellm/llms/firecrawl/__init__.py new file mode 100644 index 00000000000..bacf1eac070 --- /dev/null +++ b/litellm/llms/firecrawl/__init__.py @@ -0,0 +1,7 @@ +""" +Firecrawl API integration module. +""" +from litellm.llms.firecrawl.search.transformation import FirecrawlSearchConfig + +__all__ = ["FirecrawlSearchConfig"] + diff --git a/litellm/llms/firecrawl/search/__init__.py b/litellm/llms/firecrawl/search/__init__.py new file mode 100644 index 00000000000..999dce655d5 --- /dev/null +++ b/litellm/llms/firecrawl/search/__init__.py @@ -0,0 +1,7 @@ +""" +Firecrawl Search API module. +""" +from litellm.llms.firecrawl.search.transformation import FirecrawlSearchConfig + +__all__ = ["FirecrawlSearchConfig"] + diff --git a/litellm/llms/firecrawl/search/transformation.py b/litellm/llms/firecrawl/search/transformation.py new file mode 100644 index 00000000000..af501a8eac0 --- /dev/null +++ b/litellm/llms/firecrawl/search/transformation.py @@ -0,0 +1,207 @@ +""" +Calls Firecrawl's /search endpoint to search the web. + +Firecrawl API Reference: https://docs.firecrawl.dev/api-reference/endpoint/search +""" +from typing import Dict, List, Optional, TypedDict, Union + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) +from litellm.secret_managers.main import get_secret_str + + +class _FirecrawlSearchRequestRequired(TypedDict): + """Required fields for Firecrawl Search API request.""" + query: str # Required - search query + + +class FirecrawlSearchRequest(_FirecrawlSearchRequestRequired, total=False): + """ + Firecrawl Search API request format. + Based on: https://docs.firecrawl.dev/api-reference/endpoint/search + """ + limit: int # Optional - maximum number of results to return (default 5, max 100) + sources: List[str] # Optional - sources to search ('web', 'images', 'news'), default ['web'] + categories: List[Dict[str, str]] # Optional - categories to filter by (github, research, pdf) + tbs: str # Optional - time-based search parameter + location: str # Optional - location parameter for geo-targeting + country: str # Optional - ISO country code (default 'US') + timeout: int # Optional - timeout in milliseconds (default 60000) + ignoreInvalidURLs: bool # Optional - exclude invalid URLs (default false) + scrapeOptions: Dict # Optional - options for scraping search results + + +class FirecrawlSearchConfig(BaseSearchConfig): + FIRECRAWL_API_BASE = "https://api.firecrawl.dev/v2" + + @staticmethod + def ui_friendly_name() -> str: + return "Firecrawl" + + def validate_environment( + self, + headers: Dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + **kwargs, + ) -> Dict: + """ + Validate environment and return headers. + """ + api_key = api_key or get_secret_str("FIRECRAWL_API_KEY") + if not api_key: + raise ValueError("FIRECRAWL_API_KEY is not set. Set `FIRECRAWL_API_KEY` environment variable.") + headers["Authorization"] = f"Bearer {api_key}" + headers["Content-Type"] = "application/json" + return headers + + def get_complete_url( + self, + api_base: Optional[str], + optional_params: dict, + data: Optional[Union[Dict, List[Dict]]] = None, + **kwargs, + ) -> str: + """ + Get complete URL for Search endpoint. + """ + api_base = api_base or get_secret_str("FIRECRAWL_API_BASE") or self.FIRECRAWL_API_BASE + + # Append "/search" to the api base if it's not already there + if not api_base.endswith("/search"): + api_base = f"{api_base}/search" + + return api_base + + + def transform_search_request( + self, + query: Union[str, List[str]], + optional_params: dict, + **kwargs, + ) -> Dict: + """ + Transform Search request to Firecrawl API format. + + Transforms Perplexity unified spec parameters: + - query → query (same) + - max_results → limit + - search_domain_filter → (not directly supported, can use scrapeOptions) + - country → country + - max_tokens_per_page → (not applicable, ignored) + + All other Firecrawl-specific parameters are passed through as-is. + + Args: + query: Search query (string or list of strings). Firecrawl only supports single string queries. + optional_params: Optional parameters for the request + + Returns: + Dict with typed request data following FirecrawlSearchRequest spec + """ + if isinstance(query, list): + # Firecrawl only supports single string queries, join with spaces + query = " ".join(query) + + request_data: FirecrawlSearchRequest = { + "query": query, + } + + # Transform Perplexity unified spec parameters to Firecrawl format + if "max_results" in optional_params: + request_data["limit"] = optional_params["max_results"] + + if "country" in optional_params: + request_data["country"] = optional_params["country"] + + # Convert to dict before dynamic key assignments + result_data = dict(request_data) + + # pass through all other parameters as-is + for param, value in optional_params.items(): + if param not in self.get_supported_perplexity_optional_params() and param not in result_data: + result_data[param] = value + + # By default, request markdown content if not explicitly specified + # Firecrawl doesn't return content unless explicitly requested via scrapeOptions + if "scrapeOptions" not in result_data: + result_data["scrapeOptions"] = { + "formats": ["markdown"], + "onlyMainContent": True + } + + return result_data + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs, + ) -> SearchResponse: + """ + Transform Firecrawl API response to LiteLLM unified SearchResponse format. + + Firecrawl → LiteLLM mappings: + - data.web[].title → SearchResult.title + - data.web[].url → SearchResult.url + - data.web[].description OR data.web[].markdown → SearchResult.snippet + - No date field in web results (set to None) + - No last_updated field in Firecrawl response (set to None) + + Note: Firecrawl v2 returns results organized by source type (web, images, news). + We primarily use web results for the unified format. + + Args: + raw_response: Raw httpx response from Firecrawl API + logging_obj: Logging object for tracking + + Returns: + SearchResponse with standardized format + """ + response_json = raw_response.json() + + # Transform results to SearchResult objects + results = [] + + # Process web results (primary source) + data = response_json.get("data", {}) + web_results = data.get("web", []) + + for result in web_results: + # Use markdown if available, otherwise fall back to description + snippet = result.get("markdown") or result.get("description", "") + + search_result = SearchResult( + title=result.get("title", ""), + url=result.get("url", ""), + snippet=snippet, + date=None, # Web results don't include date + last_updated=None, # Firecrawl doesn't provide last_updated in response + ) + results.append(search_result) + + # Process news results if available (they have date field) + news_results = data.get("news", []) + for result in news_results: + snippet = result.get("markdown") or result.get("snippet", "") + + search_result = SearchResult( + title=result.get("title", ""), + url=result.get("url", ""), + snippet=snippet, + date=result.get("date"), # News results include date + last_updated=None, + ) + results.append(search_result) + + return SearchResponse( + results=results, + object="search", + ) + diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 524b1c97145..a65eaf38845 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -1,10 +1,10 @@ import json -from litellm._uuid import uuid from typing import Any, List, Literal, Optional, Tuple, Union, cast import httpx import litellm +from litellm._uuid import uuid from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_response_utils.get_headers import ( @@ -102,15 +102,15 @@ class FireworksAIConfig(OpenAIGPTConfig): "prompt_truncate_length", "context_length_exceeded_behavior", ] - + # Only add tools for models that support function calling if supports_function_calling(model=model, custom_llm_provider="fireworks_ai"): supported_params.append("tools") - + # Only add tool_choice for models that explicitly support it if supports_tool_choice(model=model, custom_llm_provider="fireworks_ai"): supported_params.append("tool_choice") - + return supported_params def map_openai_params( @@ -246,7 +246,7 @@ class FireworksAIConfig(OpenAIGPTConfig): litellm_params: dict, headers: dict, ) -> dict: - if not model.startswith("accounts/"): + if not model.startswith("accounts/") and "#" not in model: model = f"accounts/fireworks/models/{model}" messages = self._transform_messages_helper( messages=messages, model=model, litellm_params=litellm_params diff --git a/litellm/llms/fireworks_ai/completion/transformation.py b/litellm/llms/fireworks_ai/completion/transformation.py index 607e709c425..3ac77288c70 100644 --- a/litellm/llms/fireworks_ai/completion/transformation.py +++ b/litellm/llms/fireworks_ai/completion/transformation.py @@ -50,7 +50,7 @@ class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig ) -> dict: prompt = _transform_prompt(messages=messages) - if not model.startswith("accounts/"): + if not model.startswith("accounts/") and "#" not in model: model = f"accounts/fireworks/models/{model}" data = { diff --git a/litellm/llms/gemini/google_genai/transformation.py b/litellm/llms/gemini/google_genai/transformation.py index 94dfea5f58a..2d585769029 100644 --- a/litellm/llms/gemini/google_genai/transformation.py +++ b/litellm/llms/gemini/google_genai/transformation.py @@ -317,5 +317,20 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): ) logging_obj.model_call_details["httpx_response"] = raw_response + response = self.convert_citation_sources_to_citations(response) return GenerateContentResponse(**response) + + def convert_citation_sources_to_citations(self, response: Dict) -> Dict: + """ + Convert citation sources to citations. + API's camelCase citationSources becomes the SDK's snake_case citations + """ + if "candidates" in response: + for candidate in response["candidates"]: + if "citationMetadata" in candidate and isinstance(candidate["citationMetadata"], dict): + citation_metadata = candidate["citationMetadata"] + # Transform citationSources to citations to match expected schema + if "citationSources" in citation_metadata: + citation_metadata["citations"] = citation_metadata.pop("citationSources") + return response \ No newline at end of file diff --git a/litellm/llms/gemini/image_edit/__init__.py b/litellm/llms/gemini/image_edit/__init__.py new file mode 100644 index 00000000000..6181015b811 --- /dev/null +++ b/litellm/llms/gemini/image_edit/__init__.py @@ -0,0 +1,11 @@ +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig + +from .transformation import GeminiImageEditConfig +from .cost_calculator import cost_calculator + +__all__ = ["GeminiImageEditConfig", "get_gemini_image_edit_config", "cost_calculator"] + + +def get_gemini_image_edit_config(model: str) -> BaseImageEditConfig: + return GeminiImageEditConfig() + diff --git a/litellm/llms/gemini/image_edit/cost_calculator.py b/litellm/llms/gemini/image_edit/cost_calculator.py new file mode 100644 index 00000000000..31f35345d84 --- /dev/null +++ b/litellm/llms/gemini/image_edit/cost_calculator.py @@ -0,0 +1,35 @@ +""" +Gemini Image Edit Cost Calculator +""" + +from typing import Any + +import litellm +from litellm.types.utils import ImageResponse + + +def cost_calculator( + model: str, + image_response: Any, +) -> float: + """ + Gemini image edit cost calculator. + + Mirrors image generation pricing: charge per returned image based on + model metadata (`output_cost_per_image`). + """ + model_info = litellm.get_model_info( + model=model, + custom_llm_provider="gemini", + ) + + output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0 + + if not isinstance(image_response, ImageResponse): + raise ValueError( + f"image_response must be of type ImageResponse got type={type(image_response)}" + ) + + num_images = len(image_response.data or []) + return output_cost_per_image * num_images + diff --git a/litellm/llms/gemini/image_edit/transformation.py b/litellm/llms/gemini/image_edit/transformation.py new file mode 100644 index 00000000000..830c58a0062 --- /dev/null +++ b/litellm/llms/gemini/image_edit/transformation.py @@ -0,0 +1,197 @@ +import base64 +from io import BufferedReader, BytesIO +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast + +import httpx +from httpx._types import RequestFiles + +from litellm.images.utils import ImageEditRequestUtils +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import FileTypes, ImageObject, ImageResponse, OpenAIImage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class GeminiImageEditConfig(BaseImageEditConfig): + DEFAULT_BASE_URL: str = "https://generativelanguage.googleapis.com/v1beta" + SUPPORTED_PARAMS: List[str] = ["size"] + + def get_supported_openai_params(self, model: str) -> List[str]: + return list(self.SUPPORTED_PARAMS) + + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict[str, Any]: + supported_params = self.get_supported_openai_params(model) + filtered_params = { + key: value + for key, value in image_edit_optional_params.items() + if key in supported_params + } + + mapped_params: Dict[str, Any] = {} + + if "size" in filtered_params: + mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio( + filtered_params["size"] # type: ignore[arg-type] + ) + + return mapped_params + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + final_api_key: Optional[str] = api_key or get_secret_str("GEMINI_API_KEY") + if not final_api_key: + raise ValueError("GEMINI_API_KEY is not set") + + headers["x-goog-api-key"] = final_api_key + headers["Content-Type"] = "application/json" + return headers + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + base_url = api_base or get_secret_str("GEMINI_API_BASE") or self.DEFAULT_BASE_URL + base_url = base_url.rstrip("/") + return f"{base_url}/models/{model}:generateContent" + + def transform_image_edit_request( # type: ignore[override] + self, + model: str, + prompt: str, + image: FileTypes, + image_edit_optional_request_params: Dict[str, Any], + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[Dict[str, Any], Optional[RequestFiles]]: + inline_parts = self._prepare_inline_image_parts(image) + if not inline_parts: + raise ValueError("Gemini image edit requires at least one image.") + + contents = [ + { + "parts": inline_parts + [{"text": prompt}], + } + ] + + request_body: Dict[str, Any] = {"contents": contents} + + generation_config: Dict[str, Any] = {} + + if "aspectRatio" in image_edit_optional_request_params: + generation_config["aspectRatio"] = image_edit_optional_request_params[ + "aspectRatio" + ] + + if generation_config: + request_body["generationConfig"] = generation_config + + empty_files = cast(RequestFiles, []) + return request_body, empty_files + + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: Any, + ) -> ImageResponse: + model_response = ImageResponse() + try: + response_json = raw_response.json() + except Exception as exc: + raise self.get_error_class( + error_message=f"Error transforming image edit response: {exc}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + candidates = response_json.get("candidates", []) + data_list: List[ImageObject] = [] + + for candidate in candidates: + content = candidate.get("content", {}) + parts = content.get("parts", []) + for part in parts: + inline_data = part.get("inlineData") + if inline_data and inline_data.get("data"): + data_list.append( + ImageObject( + b64_json=inline_data["data"], + url=None, + ) + ) + + model_response.data = cast(List[OpenAIImage], data_list) + return model_response + + def _map_size_to_aspect_ratio(self, size: str) -> str: + aspect_ratio_map = { + "1024x1024": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1280x896": "4:3", + "896x1280": "3:4", + } + return aspect_ratio_map.get(size, "1:1") + + def _prepare_inline_image_parts( + self, image: Union[FileTypes, List[FileTypes]] + ) -> List[Dict[str, Any]]: + images: List[FileTypes] + if isinstance(image, list): + images = image + else: + images = [image] + + inline_parts: List[Dict[str, Any]] = [] + for img in images: + if img is None: + continue + + mime_type = ImageEditRequestUtils.get_image_content_type(img) + image_bytes = self._read_all_bytes(img) + inline_parts.append( + { + "inlineData": { + "mimeType": mime_type, + "data": base64.b64encode(image_bytes).decode("utf-8"), + } + } + ) + + return inline_parts + + def _read_all_bytes(self, image: FileTypes) -> bytes: + if isinstance(image, bytes): + return image + if isinstance(image, BytesIO): + current_pos = image.tell() + image.seek(0) + data = image.read() + image.seek(current_pos) + return data + if isinstance(image, BufferedReader): + current_pos = image.tell() + image.seek(0) + data = image.read() + image.seek(current_pos) + return data + raise ValueError("Unsupported image type for Gemini image edit.") \ No newline at end of file diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index f136bd0a404..d47759d0e82 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -21,6 +21,11 @@ else: LiteLLMLoggingObj = Any +FLASH_IMAGE_PREVIEW_MODEL_IDENTIFIERS = ( + "2.0-flash-preview-image", + "2.0-flash-preview-image-generation", + "2.5-flash-image-preview", +) class GoogleImageGenConfig(BaseImageGenerationConfig): DEFAULT_BASE_URL: str = "https://generativelanguage.googleapis.com/v1beta" @@ -97,8 +102,8 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): complete_url = complete_url.rstrip("/") - # Gemini 2.5 Flash Image Preview uses generateContent endpoint - if "2.5-flash-image-preview" in model: + # Gemini Flash Image Preview models use generateContent endpoint + if any(identifier in model for identifier in FLASH_IMAGE_PREVIEW_MODEL_IDENTIFIERS): complete_url = f"{complete_url}/models/{model}:generateContent" else: # All other Imagen models use predict endpoint @@ -152,8 +157,8 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): } } """ - # For Gemini 2.5 Flash Image Preview, use standard Gemini format - if "2.5-flash-image-preview" in model: + # For Gemini Flash Image Preview models, use standard Gemini format + if any(identifier in model for identifier in FLASH_IMAGE_PREVIEW_MODEL_IDENTIFIERS): request_body: dict = { "contents": [ { @@ -212,8 +217,8 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): model_response.data = [] # Handle different response formats based on model - if "2.5-flash-image-preview" in model: - # Gemini 2.5 Flash Image Preview returns in candidates format + if any(identifier in model for identifier in FLASH_IMAGE_PREVIEW_MODEL_IDENTIFIERS): + # Gemini Flash Image Preview models return in candidates format candidates = response_data.get("candidates", []) for candidate in candidates: content = candidate.get("content", {}) diff --git a/litellm/llms/gemini/videos/__init__.py b/litellm/llms/gemini/videos/__init__.py new file mode 100644 index 00000000000..c5aed2db2d0 --- /dev/null +++ b/litellm/llms/gemini/videos/__init__.py @@ -0,0 +1,5 @@ +# Gemini Video Generation Support +from .transformation import GeminiVideoConfig + +__all__ = ["GeminiVideoConfig"] + diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py new file mode 100644 index 00000000000..d1ae47af269 --- /dev/null +++ b/litellm/llms/gemini/videos/transformation.py @@ -0,0 +1,523 @@ +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union +import base64 + +import httpx +from httpx._types import RequestFiles + +from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject +from litellm.types.router import GenericLiteLLMParams +from litellm.secret_managers.main import get_secret_str +from litellm.types.videos.utils import ( + encode_video_id_with_provider, + extract_original_video_id, +) +from litellm.images.utils import ImageEditRequestUtils +import litellm +from litellm.types.llms.gemini import GeminiLongRunningOperationResponse, GeminiVideoGenerationInstance, GeminiVideoGenerationParameters, GeminiVideoGenerationRequest +from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from ...base_llm.videos.transformation import BaseVideoConfig as _BaseVideoConfig + from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException + + LiteLLMLoggingObj = _LiteLLMLoggingObj + BaseVideoConfig = _BaseVideoConfig + BaseLLMException = _BaseLLMException +else: + LiteLLMLoggingObj = Any + BaseVideoConfig = Any + BaseLLMException = Any + + +def _convert_image_to_gemini_format(image_file) -> Dict[str, str]: + """ + Convert image file to Gemini format with base64 encoding and MIME type. + + Args: + image_file: File-like object opened in binary mode (e.g., open("path", "rb")) + + Returns: + Dict with bytesBase64Encoded and mimeType + """ + mime_type = ImageEditRequestUtils.get_image_content_type(image_file) + + if hasattr(image_file, 'seek'): + image_file.seek(0) + image_bytes = image_file.read() + base64_encoded = base64.b64encode(image_bytes).decode("utf-8") + + return { + "bytesBase64Encoded": base64_encoded, + "mimeType": mime_type + } + + +class GeminiVideoConfig(BaseVideoConfig): + """ + Configuration class for Gemini (Veo) video generation. + + Veo uses a long-running operation model: + 1. POST to :predictLongRunning returns operation name + 2. Poll operation until done=true + 3. Extract video URI from response + 4. Download video using file API + """ + + def __init__(self): + super().__init__() + + def get_supported_openai_params(self, model: str) -> list: + """ + Get the list of supported OpenAI parameters for Veo video generation. + Veo supports minimal parameters compared to OpenAI. + """ + return [ + "model", + "prompt", + "input_reference", + "seconds", + "size" + ] + + def map_openai_params( + self, + video_create_optional_params: VideoCreateOptionalRequestParams, + model: str, + drop_params: bool, + ) -> Dict[str, Any]: + """ + Map OpenAI-style parameters to Veo format. + + Mappings: + - prompt → prompt + - input_reference → image + - size → aspectRatio (e.g., "1280x720" → "16:9") + - seconds → durationSeconds (defaults to 4 seconds if not provided) + + All other params are passed through as-is to support Gemini-specific parameters. + """ + mapped_params: Dict[str, Any] = {} + + # Get supported OpenAI params (exclude "model" and "prompt" which are handled separately) + supported_openai_params = self.get_supported_openai_params(model) + openai_params_to_map = { + param for param in supported_openai_params + if param not in {"model", "prompt"} + } + + # Map input_reference to image + if "input_reference" in video_create_optional_params: + mapped_params["image"] = video_create_optional_params["input_reference"] + + # Map size to aspectRatio + if "size" in video_create_optional_params: + size = video_create_optional_params["size"] + if size is not None: + aspect_ratio = self._convert_size_to_aspect_ratio(size) + if aspect_ratio: + mapped_params["aspectRatio"] = aspect_ratio + + # Map seconds to durationSeconds, default to 4 seconds (matching OpenAI) + if "seconds" in video_create_optional_params: + seconds = video_create_optional_params["seconds"] + try: + duration = int(seconds) if isinstance(seconds, str) else seconds + if duration is not None: + mapped_params["durationSeconds"] = duration + except (ValueError, TypeError): + # If conversion fails, use default + pass + + # Pass through any other params that weren't mapped (Gemini-specific params) + for key, value in video_create_optional_params.items(): + if key not in openai_params_to_map and key not in mapped_params: + mapped_params[key] = value + + return mapped_params + + def _convert_size_to_aspect_ratio(self, size: str) -> Optional[str]: + """ + Convert OpenAI size format to Veo aspectRatio format. + + https://cloud.google.com/vertex-ai/generative-ai/docs/image/generate-videos + + Supported aspect ratios: 9:16 (portrait), 16:9 (landscape) + """ + if not size: + return None + + aspect_ratio_map = { + "1280x720": "16:9", + "1920x1080": "16:9", + "720x1280": "9:16", + "1080x1920": "9:16", + } + + return aspect_ratio_map.get(size, "16:9") + + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + """ + Validate environment and add Gemini API key to headers. + Gemini uses x-goog-api-key header for authentication. + """ + api_key = ( + api_key + or litellm.api_key + or get_secret_str("GOOGLE_API_KEY") + or get_secret_str("GEMINI_API_KEY") + ) + + if not api_key: + raise ValueError( + "GEMINI_API_KEY or GOOGLE_API_KEY is required for Veo video generation. " + "Set it via environment variable or pass it as api_key parameter." + ) + + headers.update({ + "x-goog-api-key": api_key, + "Content-Type": "application/json", + }) + return headers + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the complete URL for Veo video generation. + For video creation: returns full URL with :predictLongRunning + For status/delete: returns base URL only + """ + if api_base is None: + api_base = get_secret_str("GEMINI_API_BASE") or "https://generativelanguage.googleapis.com" + + if not model or model == "": + return api_base.rstrip('/') + + model_name = model.replace("gemini/", "") + url = f"{api_base.rstrip('/')}/v1beta/models/{model_name}:predictLongRunning" + + return url + + def transform_video_create_request( + self, + model: str, + prompt: str, + api_base: str, + video_create_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[Dict, RequestFiles, str]: + """ + Transform the video creation request for Veo API. + + Veo expects: + { + "instances": [ + { + "prompt": "A cat playing with a ball of yarn" + } + ], + "parameters": { + "aspectRatio": "16:9", + "durationSeconds": 8, + "resolution": "720p" + } + } + """ + instance = GeminiVideoGenerationInstance(prompt=prompt) + + params_copy = video_create_optional_request_params.copy() + + if "image" in params_copy and params_copy["image"] is not None: + image_data = _convert_image_to_gemini_format(params_copy["image"]) + params_copy["image"] = image_data + + parameters = GeminiVideoGenerationParameters(**params_copy) + + request_body_obj = GeminiVideoGenerationRequest( + instances=[instance], + parameters=parameters + ) + + request_data = request_body_obj.model_dump(exclude_none=True) + + return request_data, [], api_base + + def transform_video_create_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + request_data: Optional[Dict] = None, + ) -> VideoObject: + """ + Transform the Veo video creation response. + + Veo returns: + { + "name": "operations/generate_1234567890", + "metadata": {...}, + "done": false, + "error": {...} + } + + We return this as a VideoObject with: + - id: operation name (used for polling) + - status: "processing" + - usage: includes duration_seconds for cost calculation + """ + response_data = raw_response.json() + + # Parse response using Pydantic model for type safety + try: + operation_response = GeminiLongRunningOperationResponse(**response_data) + except Exception as e: + raise ValueError(f"Failed to parse operation response: {e}") + + operation_name = operation_response.name + if not operation_name: + raise ValueError(f"No operation name in Veo response: {response_data}") + + if custom_llm_provider: + video_id = encode_video_id_with_provider(operation_name, custom_llm_provider, model) + else: + video_id = operation_name + + video_obj = VideoObject( + id=video_id, + object="video", + status="processing", + model=model, + ) + + usage_data = {} + if request_data: + parameters = request_data.get("parameters", {}) + duration = parameters.get("durationSeconds") or DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS + if duration is not None: + try: + usage_data["duration_seconds"] = float(duration) + except (ValueError, TypeError): + pass + + video_obj.usage = usage_data + return video_obj + + def transform_video_status_retrieve_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform the video status retrieve request for Veo API. + + Veo polls operations at: + GET https://generativelanguage.googleapis.com/v1beta/{operation_name} + """ + operation_name = extract_original_video_id(video_id) + url = f"{api_base.rstrip('/')}/v1beta/{operation_name}" + params: Dict[str, Any] = {} + + return url, params + + def transform_video_status_retrieve_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + ) -> VideoObject: + """ + Transform the Veo operation status response. + + Veo returns: + { + "name": "operations/generate_1234567890", + "done": false # or true when complete + } + + When done=true: + { + "name": "operations/generate_1234567890", + "done": true, + "response": { + "generateVideoResponse": { + "generatedSamples": [ + { + "video": { + "uri": "files/abc123..." + } + } + ] + } + } + } + """ + response_data = raw_response.json() + # Parse response using Pydantic model for type safety + operation_response = GeminiLongRunningOperationResponse(**response_data) + + operation_name = operation_response.name + is_done = operation_response.done + + if custom_llm_provider: + video_id = encode_video_id_with_provider(operation_name, custom_llm_provider, None) + else: + video_id = operation_name + + video_obj = VideoObject( + id=video_id, + object="video", + status="processing" if not is_done else "completed" + ) + return video_obj + + def transform_video_content_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform the video content request for Veo API. + + For Veo, we need to: + 1. Get operation status to extract video URI + 2. Return download URL for the video + """ + operation_name = extract_original_video_id(video_id) + + status_url = f"{api_base.rstrip('/')}/v1beta/{operation_name}" + client = litellm.module_level_client + status_response = client.get(url=status_url, headers=headers) + status_response.raise_for_status() + response_data = status_response.json() + + operation_response = GeminiLongRunningOperationResponse(**response_data) + + if not operation_response.done: + raise ValueError( + "Video generation is not complete yet. " + "Please check status with video_status() before downloading." + ) + + if not operation_response.response: + raise ValueError("No response data in completed operation") + + generated_samples = operation_response.response.generateVideoResponse.generatedSamples + download_url = generated_samples[0].video.uri + + params: Dict[str, Any] = {} + + return download_url, params + + def transform_video_content_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> bytes: + """ + Transform the Veo video content download response. + Returns the video bytes directly. + """ + return raw_response.content + + def transform_video_remix_request( + self, + video_id: str, + prompt: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + extra_body: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + """ + Video remix is not supported by Veo API. + """ + raise NotImplementedError( + "Video remix is not supported by Google Veo. " + "Please use video_generation() to create new videos." + ) + + def transform_video_remix_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + ) -> VideoObject: + """Video remix is not supported.""" + raise NotImplementedError("Video remix is not supported by Google Veo.") + + def transform_video_list_request( + self, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + after: Optional[str] = None, + limit: Optional[int] = None, + order: Optional[str] = None, + extra_query: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + """ + Video list is not supported by Veo API. + """ + raise NotImplementedError( + "Video list is not supported by Google Veo. " + "Use the operations endpoint directly if you need to list operations." + ) + + def transform_video_list_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + ) -> Dict[str, str]: + """Video list is not supported.""" + raise NotImplementedError("Video list is not supported by Google Veo.") + + def transform_video_delete_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Video delete is not supported by Veo API. + """ + raise NotImplementedError( + "Video delete is not supported by Google Veo. " + "Videos are automatically cleaned up by Google." + ) + + def transform_video_delete_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> VideoObject: + """Video delete is not supported.""" + raise NotImplementedError("Video delete is not supported by Google Veo.") + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + from ..common_utils import GeminiError + + return GeminiError( + status_code=status_code, + message=error_message, + headers=headers, + ) + diff --git a/litellm/llms/google_pse/search/__init__.py b/litellm/llms/google_pse/search/__init__.py new file mode 100644 index 00000000000..cda3f360f9d --- /dev/null +++ b/litellm/llms/google_pse/search/__init__.py @@ -0,0 +1,8 @@ +""" +Google Programmable Search Engine (PSE) API module. +""" +from litellm.llms.google_pse.search.transformation import GooglePSESearchConfig + +__all__ = ["GooglePSESearchConfig"] + + diff --git a/litellm/llms/google_pse/search/transformation.py b/litellm/llms/google_pse/search/transformation.py new file mode 100644 index 00000000000..c1ba9cfe629 --- /dev/null +++ b/litellm/llms/google_pse/search/transformation.py @@ -0,0 +1,242 @@ +""" +Calls Google Programmable Search Engine (PSE) API to search the web. + +Google PSE API Reference: https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list +""" +from typing import Dict, List, Literal, Optional, TypedDict, Union + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) +from litellm.secret_managers.main import get_secret_str + + +class _GooglePSESearchRequestRequired(TypedDict): + """Required fields for Google PSE Search API request.""" + q: str # Required - search query + cx: str # Required - Programmable Search Engine ID + key: str # Required - API key + + +class GooglePSESearchRequest(_GooglePSESearchRequestRequired, total=False): + """ + Google Programmable Search Engine API request format. + Based on: https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list + """ + num: int # Optional - number of results (1-10), default 10 + start: int # Optional - index of first result (default 1) + cr: str # Optional - country restrict (e.g., 'countryUS', 'countryGB') + dateRestrict: str # Optional - restricts results by date (e.g., 'd[number]', 'w[number]', 'm[number]', 'y[number]') + exactTerms: str # Optional - phrase that all documents must contain + excludeTerms: str # Optional - word or phrase to exclude + fileType: str # Optional - file type to restrict results to + filter: str # Optional - controls duplicate content filtering ('0'=off, '1'=on) + gl: str # Optional - geolocation of end user (2-letter country code) + hq: str # Optional - append query terms to query + imgSize: str # Optional - returns images of specified size + imgType: str # Optional - returns images of specified type + linkSite: str # Optional - specifies all search results should contain a link to a URL + lr: str # Optional - language restrict (e.g., 'lang_en', 'lang_es') + orTerms: str # Optional - provides additional search terms + relatedSite: str # Optional - specifies all search results should be pages related to URL + rights: str # Optional - filters based on licensing + safe: str # Optional - search safety level ('active', 'off') + searchType: str # Optional - specifies search type ('image') + siteSearch: str # Optional - restricts results to URLs from specified site + siteSearchFilter: str # Optional - controls whether to include or exclude siteSearch ('e'=exclude, 'i'=include) + sort: str # Optional - sort expression + + +class GooglePSESearchConfig(BaseSearchConfig): + GOOGLE_PSE_API_BASE = "https://www.googleapis.com/customsearch/v1" + + @staticmethod + def ui_friendly_name() -> str: + return "Google PSE" + + def get_http_method(self) -> Literal["GET", "POST"]: + """ + Google PSE uses GET requests with query parameters. + """ + return "GET" + + def validate_environment( + self, + headers: Dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + **kwargs, + ) -> Dict: + """ + Validate environment and return headers. + + Google PSE uses API key as a query parameter, not in headers. + This method is called but headers are not used for authentication. + """ + api_key = api_key or get_secret_str("GOOGLE_PSE_API_KEY") + if not api_key: + raise ValueError("GOOGLE_PSE_API_KEY is not set. Set `GOOGLE_PSE_API_KEY` environment variable.") + + # Also check for search engine ID + search_engine_id = kwargs.get("search_engine_id") or get_secret_str("GOOGLE_PSE_ENGINE_ID") + if not search_engine_id: + raise ValueError("GOOGLE_PSE_ENGINE_ID is not set. Set `GOOGLE_PSE_ENGINE_ID` environment variable or pass `search_engine_id` parameter.") + + headers["Content-Type"] = "application/json" + return headers + + def get_complete_url( + self, + api_base: Optional[str], + optional_params: dict, + data: Optional[Union[Dict, List[Dict]]] = None, + **kwargs, + ) -> str: + """ + Get complete URL for Search endpoint with query parameters. + + Google PSE uses GET requests, so we build the full URL with query params here. + The transformed request body (data) contains the parameters needed for the URL. + """ + from urllib.parse import urlencode + + api_base = api_base or get_secret_str("GOOGLE_PSE_API_BASE") or self.GOOGLE_PSE_API_BASE + + # Build query parameters from the transformed request body + if data and isinstance(data, dict) and "_google_pse_params" in data: + params = data["_google_pse_params"] + query_string = urlencode(params) + return f"{api_base}?{query_string}" + + return api_base + + + def transform_search_request( + self, + query: Union[str, List[str]], + optional_params: dict, + api_key: Optional[str] = None, + search_engine_id: Optional[str] = None, + **kwargs, + ) -> Dict: + """ + Transform Search request to Google PSE API format. + + Transforms Perplexity unified spec parameters: + - query → q (same) + - max_results → num + - search_domain_filter → siteSearch + - country → gl + - max_tokens_per_page → (not applicable, ignored) + + All other Google PSE-specific parameters are passed through as-is. + + Args: + query: Search query (string or list of strings). Google PSE supports single string queries. + optional_params: Optional parameters for the request + api_key: Google API key + search_engine_id: Google Programmable Search Engine ID (cx parameter) + + Returns: + Dict with typed request data following GooglePSESearchRequest spec + """ + if isinstance(query, list): + # Google PSE only supports single string queries + query = " ".join(query) + + # Get API credentials + api_key = api_key or get_secret_str("GOOGLE_PSE_API_KEY") + search_engine_id = search_engine_id or get_secret_str("GOOGLE_PSE_ENGINE_ID") + + if not api_key: + raise ValueError("GOOGLE_PSE_API_KEY is required") + if not search_engine_id: + raise ValueError("GOOGLE_PSE_ENGINE_ID is required") + + request_data: GooglePSESearchRequest = { + "q": query, + "cx": search_engine_id, + "key": api_key, + } + + # Transform unified spec parameters to Google PSE format + if "max_results" in optional_params: + # Google PSE supports 1-10 results per request + num_results = min(optional_params["max_results"], 10) + request_data["num"] = num_results + + if "search_domain_filter" in optional_params: + # Convert list to single domain (take first if multiple) + domains = optional_params["search_domain_filter"] + if isinstance(domains, list) and len(domains) > 0: + request_data["siteSearch"] = domains[0] + request_data["siteSearchFilter"] = "i" # include + elif isinstance(domains, str): + request_data["siteSearch"] = domains + request_data["siteSearchFilter"] = "i" # include + + if "country" in optional_params: + # Google PSE uses 2-letter country codes for gl parameter + request_data["gl"] = optional_params["country"].upper() + + # Convert to dict before dynamic key assignments + result_data = dict(request_data) + + # Pass through all other parameters as-is + for param, value in optional_params.items(): + if param not in self.get_supported_perplexity_optional_params() and param not in result_data: + result_data[param] = value + + # Store params in special key for URL building (Google PSE uses GET not POST) + # Return a wrapper dict that stores params for get_complete_url to use + return { + "_google_pse_params": result_data, + } + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs, + ) -> SearchResponse: + """ + Transform Google PSE API response to LiteLLM unified SearchResponse format. + + Google PSE → LiteLLM mappings: + - items[].title → SearchResult.title + - items[].link → SearchResult.url + - items[].snippet → SearchResult.snippet + - No date/last_updated fields in Google PSE response (set to None) + + Args: + raw_response: Raw httpx response from Google PSE API + logging_obj: Logging object for tracking + + Returns: + SearchResponse with standardized format + """ + response_json = raw_response.json() + + # Transform results to SearchResult objects + results = [] + for item in response_json.get("items", []): + search_result = SearchResult( + title=item.get("title", ""), + url=item.get("link", ""), + snippet=item.get("snippet", ""), + date=None, # Google PSE doesn't provide date in standard response + last_updated=None, # Google PSE doesn't provide last_updated in response + ) + results.append(search_result) + + return SearchResponse( + results=results, + object="search", + ) + + diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 165301efb5c..20e0d412edc 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -1,9 +1,27 @@ """ Translate from OpenAI's `/v1/chat/completions` to Groq's `/v1/chat/completions` """ -from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload +from typing import ( + Any, + Coroutine, + List, + Literal, + Optional, + Tuple, + Union, + cast, + overload, + Iterator, + AsyncIterator, +) import httpx + +from litellm.llms.openai.chat.gpt_transformation import ( + OpenAIChatCompletionStreamingHandler, +) +from litellm.llms.openai.common_utils import OpenAIError + from pydantic import BaseModel import litellm @@ -16,7 +34,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolParam, ChatCompletionToolParamFunctionChunk, ) -from litellm.types.utils import ModelResponse +from litellm.types.utils import ModelResponse, ModelResponseStream from ...openai_like.chat.transformation import OpenAILikeChatConfig @@ -65,6 +83,18 @@ class GroqChatConfig(OpenAILikeChatConfig): def get_config(cls): return super().get_config() + def get_model_response_iterator( + self, + streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse], + sync_stream: bool, + json_mode: Optional[bool] = False, + ) -> Any: + return GroqChatCompletionStreamingHandler( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=json_mode, + ) + def get_supported_openai_params(self, model: str) -> list: base_params = super().get_supported_openai_params(model) try: @@ -209,7 +239,6 @@ class GroqChatConfig(OpenAILikeChatConfig): ) return optional_params - def transform_response( self, @@ -239,12 +268,17 @@ class GroqChatConfig(OpenAILikeChatConfig): json_mode=json_mode, ) - mapped_service_tier: Literal["auto", "default", "flex"] = self._map_groq_service_tier(original_service_tier=getattr(model_response, "service_tier")) + mapped_service_tier: Literal[ + "auto", "default", "flex" + ] = self._map_groq_service_tier( + original_service_tier=getattr(model_response, "service_tier") + ) setattr(model_response, "service_tier", mapped_service_tier) return model_response - - def _map_groq_service_tier(self, original_service_tier: Optional[str]) -> Literal["auto", "default", "flex"]: + def _map_groq_service_tier( + self, original_service_tier: Optional[str] + ) -> Literal["auto", "default", "flex"]: """ Ensure groq service tier is OpenAI compatible. """ @@ -252,5 +286,16 @@ class GroqChatConfig(OpenAILikeChatConfig): return "auto" if original_service_tier not in ["auto", "default", "flex"]: return "auto" - - return cast(Literal["auto", "default", "flex"], original_service_tier) \ No newline at end of file + + return cast(Literal["auto", "default", "flex"], original_service_tier) + + +class GroqChatCompletionStreamingHandler(OpenAIChatCompletionStreamingHandler): + def chunk_parser(self, chunk: dict) -> ModelResponseStream: + error = chunk.get("error") + if error: + raise OpenAIError( + status_code=error.get("code"), message=error.get("message"), body=error + ) + + return super().chunk_parser(chunk) diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index 2faef2c4c73..8316e923df3 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -37,7 +37,12 @@ class HostedVLLMRerankConfig(BaseRerankConfig): def __init__(self) -> None: pass - def get_complete_url(self, api_base: Optional[str], model: str) -> str: + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: Optional[dict] = None, + ) -> str: if api_base: # Remove trailing slashes and ensure clean base URL api_base = api_base.rstrip("/") @@ -91,6 +96,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): headers: dict, model: str, api_key: Optional[str] = None, + optional_params: Optional[dict] = None, ) -> dict: if api_key is None: api_key = get_secret_str("HOSTED_VLLM_API_KEY") or "fake-api-key" @@ -150,7 +156,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): f"Error parsing response: {raw_response.text}, status_code={raw_response.status_code}" ) - return RerankResponse(**raw_response_json) + return self._transform_response(raw_response_json) def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] diff --git a/litellm/llms/huggingface/rerank/transformation.py b/litellm/llms/huggingface/rerank/transformation.py index 1454328cc13..b386daf1c83 100644 --- a/litellm/llms/huggingface/rerank/transformation.py +++ b/litellm/llms/huggingface/rerank/transformation.py @@ -60,7 +60,12 @@ class HuggingFaceRerankConfig(BaseRerankConfig): else: return "https://api-inference.huggingface.co" - def get_complete_url(self, api_base: Optional[str], model: str) -> str: + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: Optional[dict] = None, + ) -> str: """ Get the complete URL for the API call, including the /rerank suffix if necessary. """ @@ -117,6 +122,7 @@ class HuggingFaceRerankConfig(BaseRerankConfig): headers: dict, model: str, api_key: Optional[str] = None, + optional_params: Optional[dict] = None, api_base: Optional[str] = None, ) -> dict: # Get API credentials diff --git a/litellm/llms/infinity/rerank/transformation.py b/litellm/llms/infinity/rerank/transformation.py index 55aac6033d5..1c15de714b6 100644 --- a/litellm/llms/infinity/rerank/transformation.py +++ b/litellm/llms/infinity/rerank/transformation.py @@ -26,7 +26,12 @@ from ..common_utils import InfinityError class InfinityRerankConfig(CohereRerankConfig): - def get_complete_url(self, api_base: Optional[str], model: str) -> str: + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: Optional[dict] = None, + ) -> str: if api_base is None: raise ValueError("api_base is required for Infinity rerank") # Remove trailing slashes and ensure clean base URL @@ -40,6 +45,7 @@ class InfinityRerankConfig(CohereRerankConfig): headers: dict, model: str, api_key: Optional[str] = None, + optional_params: Optional[dict] = None, ) -> dict: if api_key is None: api_key = ( diff --git a/litellm/llms/jina_ai/rerank/transformation.py b/litellm/llms/jina_ai/rerank/transformation.py index 3ba24680fd4..0fddd754a9c 100644 --- a/litellm/llms/jina_ai/rerank/transformation.py +++ b/litellm/llms/jina_ai/rerank/transformation.py @@ -55,7 +55,12 @@ class JinaAIRerankConfig(BaseRerankConfig): **optional_params, )) - def get_complete_url(self, api_base: Optional[str], model: str) -> str: + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: Optional[dict] = None, + ) -> str: base_path = "/v1/rerank" if api_base is None: @@ -122,7 +127,11 @@ class JinaAIRerankConfig(BaseRerankConfig): ) # Return response def validate_environment( - self, headers: Dict, model: str, api_key: Optional[str] = None + self, + headers: Dict, + model: str, + api_key: Optional[str] = None, + optional_params: Optional[dict] = None, ) -> Dict: if api_key is None: raise ValueError( diff --git a/litellm/llms/milvus/vector_stores/__init__.py b/litellm/llms/milvus/vector_stores/__init__.py new file mode 100644 index 00000000000..c20f5fa94b7 --- /dev/null +++ b/litellm/llms/milvus/vector_stores/__init__.py @@ -0,0 +1,3 @@ +from litellm.llms.milvus.vector_stores.transformation import MilvusVectorStoreConfig + +__all__ = ["MilvusVectorStoreConfig"] diff --git a/litellm/llms/milvus/vector_stores/transformation.py b/litellm/llms/milvus/vector_stores/transformation.py new file mode 100644 index 00000000000..fcf5d14db7c --- /dev/null +++ b/litellm/llms/milvus/vector_stores/transformation.py @@ -0,0 +1,281 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +import httpx + +import litellm +from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams +from litellm.types.vector_stores import ( + BaseVectorStoreAuthCredentials, + VectorStoreCreateOptionalRequestParams, + VectorStoreCreateResponse, + VectorStoreIndexEndpoints, + VectorStoreResultContent, + VectorStoreSearchOptionalRequestParams, + VectorStoreSearchResponse, + VectorStoreSearchResult, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + +MILVUS_OPTIONAL_PARAMS = { + "dbName", + "annsField", + "limit", + "filter", + "offset", + "groupingField", + "outputFields", + "searchParams", + "partitionNames", + "consistencyLevel", +} + + +class MilvusVectorStoreConfig(BaseVectorStoreConfig): + """ + Configuration for Milvus Vector Store + + This implementation uses the Azure AI Search API for vector store operations. + Supports vector search with embeddings generated via litellm.embeddings. + """ + + def __init__(self): + super().__init__() + + def validate_environment( + self, headers: dict, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + api_key: Optional[str] = None + if litellm_params is not None: + api_key = litellm_params.api_key or get_secret_str("MILVUS_API_KEY") + + if not api_key: + raise ValueError( + "MILVUS_API_KEY is not set. Either set it in the litellm_params or set the MILVUS_API_KEY environment variable." + ) + + headers.update({"Authorization": f"Bearer {api_key}"}) + + return headers + + def get_auth_credentials( + self, litellm_params: dict + ) -> BaseVectorStoreAuthCredentials: + api_key = litellm_params.get("api_key") + if not api_key: + raise ValueError( + "MILVUS_API_KEY is not set. Either set it in the litellm_params or set the MILVUS_API_KEY environment variable." + ) + return { + "headers": { + "Authorization": f"Bearer {api_key}", + }, + } + + def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints: + return { + "read": [ + ("POST", "/v2/vectordb/entities/search"), + ("POST", "/v2/vectordb/entities/get"), + ("POST", "/v2/vectordb/entities/query"), + ], + "write": [ + ("POST", "/v2/vectordb/entities/upsert"), + ("POST", "/v2/vectordb/entities/insert"), + ], + } + + def map_openai_params( + self, non_default_params: dict, optional_params: dict, drop_params: bool + ) -> dict: + for param, value in non_default_params.items(): + if param in MILVUS_OPTIONAL_PARAMS: + optional_params[param] = value + return optional_params + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the base endpoint for Milvus API + + Expected format: https://{milvus_api_base}.milvus.io + """ + api_base = api_base or get_secret_str("MILVUS_API_BASE") + + if not api_base: + raise ValueError( + "Milvus API base URL is required. Set MILVUS_API_BASE environment variable or pass api_base in litellm_params." + ) + + if api_base: + return api_base.rstrip("/") + + return api_base + + def transform_search_vector_store_request( + self, + vector_store_id: str, + query: Union[str, List[str]], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: LiteLLMLoggingObj, + litellm_params: dict, + ) -> Tuple[str, Dict[str, Any]]: + """ + Transform search request for Azure AI Search API + + Generates embeddings using litellm.embeddings and constructs Azure AI Search request + """ + # Convert query to string if it's a list + if isinstance(query, list): + query = " ".join(query) + + # Get embedding model from litellm_params (required) + embedding_model = litellm_params.get("litellm_embedding_model") + if not embedding_model: + raise ValueError( + "embedding_model is required in litellm_params for Milvus. You can call any litellm embedding model." + "Example: litellm_params['embedding_model'] = 'azure/text-embedding-3-large'" + ) + + embedding_config = litellm_params.get("litellm_embedding_config", {}) + if not embedding_config: + raise ValueError( + "embedding_config is required in litellm_params for Milvus. You can call any litellm embedding model." + "Example: litellm_params['embedding_config'] = {'api_base': 'https://krris-mh44uf7y-eastus2.cognitiveservices.azure.com/', 'api_key': 'os.environ/AZURE_API_KEY', 'api_version': '2025-09-01'}" + ) + + # Get top_k (number of results to return) + # Generate embedding for the query using litellm.embeddings + try: + embedding_response = litellm.embedding( + model=embedding_model, + input=[query], + **embedding_config, + ) + query_vector = embedding_response.data[0]["embedding"] + except Exception as e: + raise Exception(f"Failed to generate embedding for query: {str(e)}") + + # Azure AI Search endpoint for search + index_name = vector_store_id # vector_store_id is the index name + url = f"{api_base}/v2/vectordb/entities/search" + + # Build the request body for Azure AI Search with vector search + request_body = { + "collectionName": index_name, + "data": [query_vector], + "annsField": "book_intro_vector", + **vector_store_search_optional_params, + } + + ######################################################### + # Update logging object with details of the request + ######################################################### + litellm_logging_obj.model_call_details["input"] = query + litellm_logging_obj.model_call_details["embedding_model"] = embedding_model + + return url, request_body + + def transform_search_vector_store_response( + self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj + ) -> VectorStoreSearchResponse: + """ + Transform Azure AI Search API response to standard vector store search response + + Handles the format from Azure AI Search which returns: + { + "value": [ + { + "id": "...", + "content": "...", + "distance": 0.95, + } + ] + } + """ + try: + response_json = response.json() + + # Extract results from Azure AI Search API response + results = response_json.get("data", []) + + # Try to get text_field from optional_params first, then litellm_params + optional_params = litellm_logging_obj.model_call_details.get( + "optional_params", {} + ) + text_field = optional_params.get("milvus_text_field", "") + + # Fallback to litellm_params if not in optional_params + + if not text_field: + text_field = litellm_logging_obj.model_call_details.get( + "litellm_params", {} + ).get("milvus_text_field", "") + + # Transform results to standard format + search_results: List[VectorStoreSearchResult] = [] + for result in results: + # Extract text content + text_content = result.get(text_field, "") + + content = [ + VectorStoreResultContent( + text=text_content, + type="text", + ) + ] + + # Get the search score (distance from the query vector) + score = result.get("distance", 0.0) + + # Build attributes with all available metadata + # Exclude system fields and already-processed fields + attributes = {} + for key, value in result.items(): + if key not in ["id", "content", "distance", text_field]: + attributes[key] = value + + result_obj = VectorStoreSearchResult( + score=score, + content=content, + file_id=None, + filename=None, + attributes=attributes, + ) + search_results.append(result_obj) + + return VectorStoreSearchResponse( + object="vector_store.search_results.page", + search_query=litellm_logging_obj.model_call_details.get("input", ""), + data=search_results, + ) + + except Exception as e: + raise self.get_error_class( + error_message=str(e), + status_code=response.status_code, + headers=response.headers, + ) + + def transform_create_vector_store_request( + self, + vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams, + api_base: str, + ) -> Tuple[str, Dict]: + raise NotImplementedError + + def transform_create_vector_store_response( + self, response: httpx.Response + ) -> VectorStoreCreateResponse: + raise NotImplementedError diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index 51fa65244a0..26738623375 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -8,7 +8,9 @@ Docs - https://docs.mistral.ai/api/ from typing import ( Any, + AsyncIterator, Coroutine, + Iterator, List, Literal, Optional, @@ -26,11 +28,14 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, strip_none_values_from_message, ) -from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.llms.openai.chat.gpt_transformation import ( + OpenAIGPTConfig, + OpenAIChatCompletionStreamingHandler, +) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.mistral import MistralThinkingBlock, MistralToolCallMessage from litellm.types.llms.openai import AllMessageValues -from litellm.types.utils import ModelResponse +from litellm.types.utils import ModelResponse, ModelResponseStream from litellm.utils import convert_to_model_response_object @@ -602,3 +607,77 @@ class MistralConfig(OpenAIGPTConfig): ) return final_response_obj + + def get_model_response_iterator( + self, + streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse], + sync_stream: bool, + json_mode: Optional[bool] = False, + ): + return MistralChatResponseIterator( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=json_mode, + ) + + +class MistralChatResponseIterator(OpenAIChatCompletionStreamingHandler): + def chunk_parser(self, chunk: dict) -> ModelResponseStream: + try: + for choice in chunk.get("choices", []): + delta = choice.get("delta", {}) + content = delta.get("content") + if isinstance(content, list): + ( + normalized_text, + thinking_blocks, + reasoning_content, + ) = self._normalize_content_blocks(content) + delta["content"] = normalized_text + if thinking_blocks: + delta["thinking_blocks"] = thinking_blocks + delta["reasoning_content"] = reasoning_content + else: + delta.pop("thinking_blocks", None) + delta.pop("reasoning_content", None) + except Exception: + # Fall back to default parsing if custom handling fails + return super().chunk_parser(chunk) + + return super().chunk_parser(chunk) + + @staticmethod + def _normalize_content_blocks( + content_blocks: List[dict], + ) -> Tuple[Optional[str], List[dict], Optional[str]]: + """ + Convert Mistral magistral content blocks into OpenAI-compatible content + thinking_blocks. + """ + text_segments: List[str] = [] + thinking_blocks: List[dict] = [] + reasoning_segments: List[str] = [] + + for block in content_blocks: + block_type = block.get("type") + if block_type == "thinking": + mistral_thinking = block.get("thinking", []) + thinking_text_parts: List[str] = [] + for thinking_block in mistral_thinking: + if thinking_block.get("type") == "text": + thinking_text_parts.append(thinking_block.get("text", "")) + thinking_text = "".join(thinking_text_parts) + if thinking_text: + reasoning_segments.append(thinking_text) + thinking_blocks.append( + { + "type": "thinking", + "thinking": thinking_text, + "signature": "mistral", + } + ) + elif block_type == "text": + text_segments.append(block.get("text", "")) + + normalized_text = "".join(text_segments) if text_segments else None + reasoning_content = "\n".join(reasoning_segments) if reasoning_segments else None + return normalized_text, thinking_blocks, reasoning_content diff --git a/litellm/llms/mistral/ocr/__init__.py b/litellm/llms/mistral/ocr/__init__.py new file mode 100644 index 00000000000..40cc62696be --- /dev/null +++ b/litellm/llms/mistral/ocr/__init__.py @@ -0,0 +1,2 @@ +"""Mistral OCR transformation module.""" + diff --git a/litellm/llms/mistral/ocr/transformation.py b/litellm/llms/mistral/ocr/transformation.py new file mode 100644 index 00000000000..ed5e2359395 --- /dev/null +++ b/litellm/llms/mistral/ocr/transformation.py @@ -0,0 +1,225 @@ +""" +Mistral OCR transformation implementation. +""" +from typing import Any, Dict, Optional + +import httpx + +from litellm._logging import verbose_logger +from litellm.llms.base_llm.ocr.transformation import ( + BaseOCRConfig, + DocumentType, + OCRRequestData, + OCRResponse, +) +from litellm.secret_managers.main import get_secret_str + + +class MistralOCRConfig(BaseOCRConfig): + """ + Mistral OCR transformation configuration. + + Reference: https://docs.mistral.ai/api/#tag/ocr + """ + + def __init__(self) -> None: + super().__init__() + + def get_supported_ocr_params(self, model: str) -> list: + """ + Get supported OCR parameters for Mistral OCR. + + Mistral OCR supports: + - pages: List of page numbers to process + - include_image_base64: Whether to include base64 encoded images + - image_limit: Maximum number of images to return + - image_min_size: Minimum size of images to include + - bbox_annotation_format: Format for bounding box annotations + - document_annotation_format: Format for document annotations + """ + return [ + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + ] + + def map_ocr_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + ) -> dict: + """ + Map OCR parameters to Mistral-specific format. + + Mistral accepts these parameters directly, so no transformation needed. + Just filter out unsupported params. + """ + supported_params = self.get_supported_ocr_params(model=model) + + # Only include params that are in the supported list + mapped_params = {} + for param, value in non_default_params.items(): + if param in supported_params: + mapped_params[param] = value + + return mapped_params + + def validate_environment( + self, + headers: Dict, + model: str, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + litellm_params: Optional[dict] = None, + **kwargs, + ) -> Dict: + """ + Validate environment and return headers for Mistral OCR. + """ + # Get API key from environment if not provided + if api_key is None: + api_key = ( + get_secret_str("MISTRAL_API_KEY") + ) + + if api_key is None: + raise ValueError( + "Missing Mistral API Key - A call is being made to Mistral but no key is set either in the environment variables or via params" + ) + + headers = { + "Authorization": f"Bearer {api_key}", + **headers, + } + + # Don't set Content-Type for multipart/form-data - httpx will handle it + + return headers + + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: dict, + litellm_params: Optional[dict] = None, + **kwargs, + ) -> str: + """ + Get complete URL for Mistral OCR endpoint. + + Returns: https://api.mistral.ai/v1/ocr + """ + if api_base is None: + api_base = "https://api.mistral.ai/v1" + + # Ensure no trailing slash + api_base = api_base.rstrip("/") + + # Remove /v1 if it's already in the base to avoid duplication + if api_base.endswith("/v1"): + return f"{api_base}/ocr" + + return f"{api_base}/v1/ocr" + + + def transform_ocr_request( + self, + model: str, + document: DocumentType, + optional_params: dict, + headers: dict, + **kwargs, + ) -> OCRRequestData: + """ + Transform OCR request to Mistral-specific format. + + Mistral OCR API accepts: + { + "model": "mistral-ocr-latest", + "document": { + "type": "document_url", + "document_url": "" + }, + "pages": [0], # optional + "include_image_base64": false, # optional + ... + } + + Args: + model: Model name (e.g., "mistral-ocr-latest") + document: Document dict from user (Mistral format) - already validated in main.py + optional_params: Already mapped optional parameters + headers: Request headers + + Returns: + OCRRequestData with JSON data + """ + verbose_logger.debug(f"Mistral OCR transform_ocr_request - model: {model}") + + # Document parameter is the Mistral-format dict from the user + # Just pass it through as-is to the Mistral API + if not isinstance(document, dict): + raise ValueError(f"Expected document dict, got {type(document)}") + + # Build request data - use document dict directly + data = { + "model": model, + "document": document, # Pass through the Mistral-format document dict + } + + # Add all optional parameters from the already-mapped optional_params + data.update(optional_params) + + # No multipart files - using JSON + return OCRRequestData(data=data, files=None) + + def transform_ocr_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: Any, + **kwargs, + ) -> OCRResponse: + """ + Return Mistral OCR response in native format. + + Mistral OCR is the standard format for LiteLLM OCR responses. + No transformation needed - return native response. + + Mistral OCR returns: + { + "pages": [ + { + "index": 0, + "markdown": "extracted text content", + "images": [...], + "dimensions": {...} + }, + ... + ], + "model": "mistral-ocr-2505-completion", + "document_annotation": null, + "usage_info": {...} + } + """ + try: + response_json = raw_response.json() + + verbose_logger.debug(f"Mistral OCR response keys: {response_json.keys()}") + + # Return native Mistral format - no transformation + return OCRResponse( + pages=response_json.get("pages", []), + model=response_json.get("model", model), + document_annotation=response_json.get("document_annotation"), + usage_info=response_json.get("usage_info"), + object="ocr", + ) + except Exception as e: + verbose_logger.error(f"Error parsing Mistral OCR response: {e}") + raise e + diff --git a/litellm/llms/nvidia_nim/rerank/transformation.py b/litellm/llms/nvidia_nim/rerank/transformation.py index cb9fd4bebaa..5bbe16e5381 100644 --- a/litellm/llms/nvidia_nim/rerank/transformation.py +++ b/litellm/llms/nvidia_nim/rerank/transformation.py @@ -55,7 +55,12 @@ class NvidiaNimRerankConfig(BaseRerankConfig): def __init__(self) -> None: pass - def get_complete_url(self, api_base: Optional[str], model: str) -> str: + def get_complete_url( + self, + api_base: Optional[str], + model: str, + optional_params: Optional[dict] = None, + ) -> str: """ Construct the Nvidia NIM rerank URL. @@ -131,6 +136,7 @@ class NvidiaNimRerankConfig(BaseRerankConfig): headers: dict, model: str, api_key: Optional[str] = None, + optional_params: Optional[dict] = None, ) -> dict: """ Validate that the Nvidia NIM API key is present. diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 3ab827797c5..167ba26bacb 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -2,7 +2,8 @@ import base64 import datetime import hashlib import json -from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional, Tuple, Union +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional, Protocol, Tuple, Union from urllib.parse import urlparse import httpx @@ -62,6 +63,47 @@ else: LiteLLMLoggingObj = Any +class OCISignerProtocol(Protocol): + """ + Protocol for OCI request signers (e.g., oci.signer.Signer). + + This protocol defines the interface expected for OCI SDK signer objects. + Compatible with the OCI Python SDK's Signer class. + + See: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html + """ + + def do_request_sign(self, request: Any, *, enforce_content_headers: bool = False) -> None: + """ + Sign an HTTP request by adding authentication headers. + + Args: + request: Request object with method, url, headers, body, and path_url attributes + enforce_content_headers: Whether to enforce content-type and content-length headers + """ + ... + + +@dataclass +class OCIRequestWrapper: + """ + Wrapper for HTTP requests compatible with OCI signer interface. + + This class wraps request data in a format compatible with OCI SDK signers, + which expect objects with method, url, headers, body, and path_url attributes. + """ + method: str + url: str + headers: dict + body: bytes + + @property + def path_url(self) -> str: + """Returns the path + query string for OCI signing.""" + parsed_url = urlparse(self.url) + return parsed_url.path + ("?" + parsed_url.query if parsed_url.query else "") + + def sha256_base64(data: bytes) -> str: digest = hashlib.sha256(data).digest() return base64.b64encode(digest).decode() @@ -228,29 +270,89 @@ class OCIChatConfig(BaseConfig): return adapted_params - def sign_request( + def _sign_with_oci_signer( self, headers: dict, optional_params: dict, request_data: dict, api_base: str, - api_key: Optional[str] = None, - model: Optional[str] = None, - stream: Optional[bool] = None, - fake_stream: Optional[bool] = None, - ) -> Tuple[dict, Optional[bytes]]: + ) -> Tuple[dict, bytes]: """ - Some providers like Bedrock require signing the request. The sign request funtion needs access to `request_data` and `complete_url` - Args: - headers: dict - optional_params: dict - request_data: dict - the request body being sent in http request - api_base: str - the complete url being sent in http request - Returns: - dict - the signed headers - """ - import json + Sign request using OCI SDK Signer object. + Args: + headers: Request headers to be signed + optional_params: Optional parameters including oci_signer + request_data: The request body dict to be sent in HTTP request + api_base: The complete URL for the HTTP request + + Returns: + Tuple of (signed_headers, encoded_body) + + Raises: + OCIError: If signing fails + ValueError: If HTTP method is unsupported + """ + oci_signer = optional_params.get("oci_signer") + body = json.dumps(request_data).encode("utf-8") + method = str(optional_params.get("method", "POST")).upper() + + if method not in ["POST", "GET", "PUT", "DELETE", "PATCH"]: + raise ValueError(f"Unsupported HTTP method: {method}") + + prepared_headers = headers.copy() + prepared_headers.setdefault("content-type", "application/json") + prepared_headers.setdefault("content-length", str(len(body))) + + request_wrapper = OCIRequestWrapper( + method=method, + url=api_base, + headers=prepared_headers, + body=body + ) + + if oci_signer is None: + raise ValueError("oci_signer cannot be None when calling _sign_with_oci_signer") + + try: + oci_signer.do_request_sign(request_wrapper, enforce_content_headers=True) + except Exception as e: + raise OCIError( + status_code=500, + message=( + f"Failed to sign request with provided oci_signer: {str(e)}. " + "The signer must implement the OCI SDK Signer interface with a " + "do_request_sign(request, enforce_content_headers=True) method. " + "See: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html" + ) + ) from e + + headers.update(request_wrapper.headers) + return headers, body + + def _sign_with_manual_credentials( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + ) -> Tuple[dict, None]: + """ + Sign request using manual OCI credentials. + + Args: + headers: Request headers to be signed + optional_params: Optional parameters including OCI credentials + request_data: The request body dict to be sent in HTTP request + api_base: The complete URL for the HTTP request + + Returns: + Tuple of (signed_headers, None) + + Raises: + Exception: If required credentials are missing + ImportError: If cryptography package is not installed + """ oci_region = optional_params.get("oci_region", "us-ashburn-1") api_base = ( api_base @@ -355,6 +457,69 @@ class OCIChatConfig(BaseConfig): return headers, None + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ) -> Tuple[dict, Optional[bytes]]: + """ + Sign the OCI request by adding authentication headers. + + Supports two signing modes: + 1. OCI SDK Signer: Use an oci_signer object to sign the request + 2. Manual Signing: Use OCI credentials to manually sign the request + + Args: + headers: Request headers to be signed + optional_params: Optional parameters including auth credentials or oci_signer + request_data: The request body dict to be sent in HTTP request + api_base: The complete URL for the HTTP request + api_key: Optional API key (not used for OCI) + model: Optional model name + stream: Optional streaming flag + fake_stream: Optional fake streaming flag + + Returns: + Tuple of (signed_headers, encoded_body): + - If oci_signer is provided: Returns (headers, body) where body is the encoded JSON + - If manual credentials are provided: Returns (headers, None) as body is not returned + for the manual signing path + + Raises: + OCIError: If signing fails with oci_signer + Exception: If required credentials are missing + ImportError: If cryptography package is not installed (manual signing only) + + Example: + >>> from oci.signer import Signer + >>> signer = Signer( + ... tenancy="ocid1.tenancy.oc1..", + ... user="ocid1.user.oc1..", + ... fingerprint="xx:xx:xx", + ... private_key_file_location="~/.oci/key.pem" + ... ) + >>> headers, body = config.sign_request( + ... headers={}, + ... optional_params={"oci_signer": signer}, + ... request_data={"message": "Hello"}, + ... api_base="https://inference.generativeai.us-ashburn-1.oci.oraclecloud.com/..." + ... ) + """ + oci_signer = optional_params.get("oci_signer") + + # If a signer is provided, use it for request signing + if oci_signer is not None: + return self._sign_with_oci_signer(headers, optional_params, request_data, api_base) + + # Standard manual credential signing + return self._sign_with_manual_credentials(headers, optional_params, request_data, api_base) + def validate_environment( self, headers: dict, @@ -365,36 +530,67 @@ class OCIChatConfig(BaseConfig): api_key: Optional[str] = None, api_base: Optional[str] = None, ) -> dict: + """ + Validate the OCI environment and credentials. + + Supports two authentication modes: + 1. OCI SDK Signer: Pass an oci_signer object (e.g., oci.signer.Signer) + 2. Manual Credentials: Pass oci_user, oci_fingerprint, oci_tenancy, and oci_key/oci_key_file + + Args: + headers: Request headers to populate + model: Model name + messages: List of chat messages + optional_params: Optional parameters including authentication credentials + litellm_params: LiteLLM parameters + api_key: Optional API key (not used for OCI) + api_base: Optional API base URL + + Returns: + Updated headers dict + + Raises: + Exception: If required parameters are missing or invalid + """ + oci_signer = optional_params.get("oci_signer") oci_region = optional_params.get("oci_region", "us-ashburn-1") + + # Determine api_base api_base = ( api_base or litellm.api_base or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com" ) - oci_user = optional_params.get("oci_user") - oci_fingerprint = optional_params.get("oci_fingerprint") - oci_tenancy = optional_params.get("oci_tenancy") - oci_key = optional_params.get("oci_key") - oci_key_file = optional_params.get("oci_key_file") - oci_compartment_id = optional_params.get("oci_compartment_id") - - if ( - not oci_user - or not oci_fingerprint - or not oci_tenancy - or not (oci_key or oci_key_file) - or not oci_compartment_id - ): - raise Exception( - "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, oci_compartment_id " - "and at least one of oci_key or oci_key_file." - ) if not api_base: raise Exception( - "Either `api_base` must be provided or `litellm.api_base` must be set. Alternatively, you can set the `oci_region` optional parameter to use the default OCI region." + "Either `api_base` must be provided or `litellm.api_base` must be set. " + "Alternatively, you can set the `oci_region` optional parameter to use the default OCI region." ) + # Validate credentials only if signer is not provided + if oci_signer is None: + oci_user = optional_params.get("oci_user") + oci_fingerprint = optional_params.get("oci_fingerprint") + oci_tenancy = optional_params.get("oci_tenancy") + oci_key = optional_params.get("oci_key") + oci_key_file = optional_params.get("oci_key_file") + oci_compartment_id = optional_params.get("oci_compartment_id") + + if ( + not oci_user + or not oci_fingerprint + or not oci_tenancy + or not (oci_key or oci_key_file) + or not oci_compartment_id + ): + raise Exception( + "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, oci_compartment_id " + "and at least one of oci_key or oci_key_file. " + "Alternatively, provide an oci_signer object from the OCI SDK." + ) + + # Common header setup headers.update( { "content-type": "application/json", @@ -442,12 +638,12 @@ class OCIChatConfig(BaseConfig): for openai_key, oci_key in open_ai_to_oci_param_map.items(): if oci_key and openai_key in optional_params: selected_params[oci_key] = optional_params[openai_key] # type: ignore[index] - + # Also check for already-mapped OCI params (for backward compatibility) for oci_value in open_ai_to_oci_param_map.values(): if oci_value and oci_value in optional_params and oci_value not in selected_params: selected_params[oci_value] = optional_params[oci_value] # type: ignore[index] - + if "tools" in selected_params: if vendor == OCIVendors.COHERE: selected_params["tools"] = self.adapt_tool_definitions_to_cohere_standard( # type: ignore[assignment] @@ -465,7 +661,7 @@ class OCIChatConfig(BaseConfig): for msg in messages[:-1]: # All messages except the last one role = msg.get("role") content = msg.get("content") - + if isinstance(content, list): # Extract text from content array text_content = "" @@ -473,11 +669,11 @@ class OCIChatConfig(BaseConfig): if isinstance(content_item, dict) and content_item.get("type") == "text": text_content += content_item.get("text", "") content = text_content - + # Ensure content is a string if not isinstance(content, str): content = str(content) if content is not None else "" - + # Handle tool calls tool_calls: Optional[List[CohereToolCall]] = None if role == "assistant" and "tool_calls" in msg and msg.get("tool_calls"): # type: ignore[union-attr,typeddict-item] @@ -492,12 +688,12 @@ class OCIChatConfig(BaseConfig): arguments = {} else: arguments = raw_arguments - + tool_calls.append(CohereToolCall( name=str(tool_call.get("function", {}).get("name", "")), parameters=arguments )) - + if role == "user": chat_history.append(CohereMessage(role="USER", message=content)) elif role == "assistant": @@ -505,11 +701,11 @@ class OCIChatConfig(BaseConfig): elif role == "tool": # Tool messages need special handling chat_history.append(CohereMessage( - role="TOOL", + role="TOOL", message=content, toolCalls=None # Tool messages don't have tool calls )) - + return chat_history def adapt_tool_definitions_to_cohere_standard(self, tools: List[Dict[str, Any]]) -> List[CohereTool]: @@ -519,7 +715,7 @@ class OCIChatConfig(BaseConfig): function_def = tool.get("function", {}) parameters = function_def.get("parameters", {}).get("properties", {}) required = function_def.get("parameters", {}).get("required", []) - + parameter_definitions = {} for param_name, param_schema in parameters.items(): parameter_definitions[param_name] = CohereParameterDefinition( @@ -527,13 +723,13 @@ class OCIChatConfig(BaseConfig): type=param_schema.get("type", "string"), isRequired=param_name in required ) - + cohere_tools.append(CohereTool( name=function_def.get("name", ""), description=function_def.get("description", ""), parameterDefinitions=parameter_definitions )) - + return cohere_tools def _extract_text_content(self, content: Any) -> str: @@ -586,7 +782,7 @@ class OCIChatConfig(BaseConfig): user_messages = [msg for msg in messages if msg.get("role") == "user"] if not user_messages: raise Exception("No user message found for Cohere model") - + # Create Cohere-specific chat request chat_request = CohereChatRequest( @@ -595,7 +791,7 @@ class OCIChatConfig(BaseConfig): chatHistory=self.adapt_messages_to_cohere_standard(messages), **self._get_optional_params(OCIVendors.COHERE, optional_params) ) - + data = OCICompletionPayload( compartmentId=oci_compartment_id, servingMode=servingMode, @@ -616,24 +812,24 @@ class OCIChatConfig(BaseConfig): return data.model_dump(exclude_none=True) def _handle_cohere_response( - self, - json_response: dict, - model: str, + self, + json_response: dict, + model: str, model_response: ModelResponse ) -> ModelResponse: """Handle Cohere-specific response format.""" cohere_response = CohereChatResult(**json_response) # Cohere response format (uses camelCase) model_id = model - + # Set basic response info model_response.model = model_id model_response.created = int(datetime.datetime.now().timestamp()) - + # Extract the response text response_text = cohere_response.chatResponse.text oci_finish_reason = cohere_response.chatResponse.finishReason - + # Map finish reason if oci_finish_reason == "COMPLETE": finish_reason = "stop" @@ -641,7 +837,7 @@ class OCIChatConfig(BaseConfig): finish_reason = "length" else: finish_reason = "stop" - + # Handle tool calls tool_calls: Optional[List[Dict[str, Any]]] = None if cohere_response.chatResponse.toolCalls: @@ -655,7 +851,7 @@ class OCIChatConfig(BaseConfig): "arguments": json.dumps(tool_call.parameters) } }) - + # Create choice from litellm.types.utils import Choices choice = Choices( @@ -668,7 +864,7 @@ class OCIChatConfig(BaseConfig): finish_reason=finish_reason ) model_response.choices = [choice] - + # Extract usage info usage_info = cohere_response.chatResponse.usage from litellm.types.utils import Usage @@ -677,13 +873,13 @@ class OCIChatConfig(BaseConfig): completion_tokens=usage_info.completionTokens, # type: ignore[union-attr] total_tokens=usage_info.totalTokens # type: ignore[union-attr] ) - + return model_response def _handle_generic_response( - self, - json: dict, - model: str, + self, + json: dict, + model: str, model_response: ModelResponse, raw_response: httpx.Response ) -> ModelResponse: @@ -695,7 +891,7 @@ class OCIChatConfig(BaseConfig): message=f"Response cannot be casted to OCICompletionResponse: {str(e)}", status_code=raw_response.status_code, ) - + iso_str = completion_response.chatResponse.timeCreated dt = datetime.datetime.fromisoformat(iso_str.replace("Z", "+00:00")) model_response.created = int(dt.timestamp()) @@ -751,7 +947,7 @@ class OCIChatConfig(BaseConfig): ) vendor = get_vendor_from_model(model) - + # Handle response based on vendor type if vendor == OCIVendors.COHERE: model_response = self._handle_cohere_response(json, model, model_response) @@ -1080,7 +1276,7 @@ class OCIStreamWrapper(CustomStreamWrapper): if not chunk.startswith("data:"): raise ValueError(f"Chunk does not start with 'data:': {chunk}") dict_chunk = json.loads(chunk[5:]) # Remove 'data: ' prefix and parse JSON - + # Check if this is a Cohere stream chunk if "apiFormat" in dict_chunk and dict_chunk.get("apiFormat") == "COHERE": return self._handle_cohere_stream_chunk(dict_chunk) diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index b740eb122fd..9c8700daf83 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -188,7 +188,7 @@ class OllamaChatConfig(BaseConfig): if model.startswith("gpt-oss"): optional_params["think"] = value else: - optional_params["think"] = True + optional_params["think"] = value in {"low", "medium", "high"} ### FUNCTION CALLING LOGIC ### if param == "tools": ## CHECK IF MODEL SUPPORTS TOOL CALLING ## diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index b476e5c8a63..c4d08c83a2a 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, AsyncIterator, Iterator, List, Optional, from httpx._models import Headers, Response import litellm +from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, ) @@ -183,7 +184,7 @@ class OllamaConfig(BaseConfig): if model.startswith("gpt-oss"): optional_params["think"] = value else: - optional_params["think"] = True + optional_params["think"] = value in {"low", "medium", "high"} elif param == "response_format" and isinstance(value, dict): if value["type"] == "json_object": optional_params["format"] = "json" @@ -577,6 +578,18 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): ] ) else: - raise Exception(f"Unable to parse ollama chunk - {chunk}") + # In this case, 'thinking' is not present in the chunk, chunk["done"] is false, + # and chunk["response"] is falsy (None or empty string), + # but Ollama is just starting to stream, so it should be processed as a normal dict + return ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta(reasoning_content=""), + ) + ] + ) + # raise Exception(f"Unable to parse ollama chunk - {chunk}") except Exception as e: + verbose_proxy_logger.error(f"Unable to parse ollama chunk - {chunk}") raise e diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 3e18617905c..4e553a3da5c 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -158,6 +158,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): "parallel_tool_calls", "audio", "web_search_options", + "service_tier", "safety_identifier", ] # works across all models diff --git a/litellm/llms/openai/chat/guardrail_translation/README.md b/litellm/llms/openai/chat/guardrail_translation/README.md new file mode 100644 index 00000000000..05e3b55e54c --- /dev/null +++ b/litellm/llms/openai/chat/guardrail_translation/README.md @@ -0,0 +1,3 @@ +Translation of OpenAI `/chat/completions` input and output to a custom guardrail. + +This enables guardrails to be applied to OpenAI `/chat/completions` requests and responses. \ No newline at end of file diff --git a/litellm/llms/openai/chat/guardrail_translation/__init__.py b/litellm/llms/openai/chat/guardrail_translation/__init__.py new file mode 100644 index 00000000000..b0682aa4758 --- /dev/null +++ b/litellm/llms/openai/chat/guardrail_translation/__init__.py @@ -0,0 +1,12 @@ +"""OpenAI Chat Completions message handler for Unified Guardrails.""" + +from litellm.llms.openai.chat.guardrail_translation.handler import ( + OpenAIChatCompletionsHandler, +) +from litellm.types.utils import CallTypes + +guardrail_translation_mappings = { + CallTypes.completion: OpenAIChatCompletionsHandler, + CallTypes.acompletion: OpenAIChatCompletionsHandler, +} +__all__ = ["guardrail_translation_mappings"] diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py new file mode 100644 index 00000000000..b01f9f1b980 --- /dev/null +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -0,0 +1,283 @@ +""" +OpenAI Chat Completions Message Handler for Unified Guardrails + +This module provides a class-based handler for OpenAI-format chat completions. +The class methods can be overridden for custom behavior. + +Pattern Overview: +----------------- +1. Extract text content from messages/responses (both string and list formats) +2. Create async tasks to apply guardrails to each text segment +3. Track mappings to know where each response belongs +4. Apply guardrail responses back to the original structure + +This pattern can be replicated for other message formats (e.g., Anthropic). +""" + +import asyncio +from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Optional, Tuple, cast + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.types.utils import Choices + +if TYPE_CHECKING: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.utils import ModelResponse + + +class OpenAIChatCompletionsHandler(BaseTranslation): + """ + Handler for processing OpenAI chat completions messages with guardrails. + + This class provides methods to: + 1. Process input messages (pre-call hook) + 2. Process output responses (post-call hook) + + Methods can be overridden to customize behavior for different message formats. + """ + + async def process_input_messages( + self, + data: dict, + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process input messages by applying guardrails to text content. + """ + messages = data.get("messages") + if messages is None: + return data + + tasks: List[Coroutine[Any, Any, str]] = [] + task_mappings: List[Tuple[int, Optional[int]]] = [] + # Track (message_index, content_index) for each task + # content_index is None for string content, int for list content + + # Step 1: Extract all text content and create guardrail tasks + for msg_idx, message in enumerate(messages): + await self._extract_input_text_and_create_tasks( + message=message, + msg_idx=msg_idx, + tasks=tasks, + task_mappings=task_mappings, + guardrail_to_apply=guardrail_to_apply, + request_data=data, + ) + + # Step 2: Run all guardrail tasks in parallel + responses = await asyncio.gather(*tasks) + + # Step 3: Map guardrail responses back to original message structure + await self._apply_guardrail_responses_to_input( + messages=messages, + responses=responses, + task_mappings=task_mappings, + ) + + verbose_proxy_logger.debug( + "OpenAI Chat Completions: Processed input messages: %s", messages + ) + + return data + + async def _extract_input_text_and_create_tasks( + self, + message: Dict[str, Any], + msg_idx: int, + tasks: List, + task_mappings: List[Tuple[int, Optional[int]]], + guardrail_to_apply: "CustomGuardrail", + request_data: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Extract text content from a message and create guardrail tasks. + + Override this method to customize text extraction logic. + """ + content = message.get("content", None) + if content is None: + return + + if isinstance(content, str): + # Simple string content + tasks.append(guardrail_to_apply.apply_guardrail(text=content, request_data=request_data)) + task_mappings.append((msg_idx, None)) + + elif isinstance(content, list): + # List content (e.g., multimodal with text and images) + for content_idx, content_item in enumerate(content): + text_str = content_item.get("text", None) + if text_str is None: + continue + tasks.append(guardrail_to_apply.apply_guardrail(text=text_str, request_data=request_data)) + task_mappings.append((msg_idx, int(content_idx))) + + async def _apply_guardrail_responses_to_input( + self, + messages: List[Dict[str, Any]], + responses: List[str], + task_mappings: List[Tuple[int, Optional[int]]], + ) -> None: + """ + Apply guardrail responses back to input messages. + + Override this method to customize how responses are applied. + """ + for task_idx, guardrail_response in enumerate(responses): + mapping = task_mappings[task_idx] + msg_idx = cast(int, mapping[0]) + content_idx_optional = cast(Optional[int], mapping[1]) + + content = messages[msg_idx].get("content", None) + if content is None: + continue + + if isinstance(content, str) and content_idx_optional is None: + # Replace string content with guardrail response + messages[msg_idx]["content"] = guardrail_response + + elif isinstance(content, list) and content_idx_optional is not None: + # Replace specific text item in list content + messages[msg_idx]["content"][content_idx_optional][ + "text" + ] = guardrail_response + + async def process_output_response( + self, + response: "ModelResponse", + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process output response by applying guardrails to text content. + + Args: + response: LiteLLM ModelResponse object + guardrail_to_apply: The guardrail instance to apply + + Returns: + Modified response with guardrail applied to content + + Response Format Support: + - String content: choice.message.content = "text here" + - List content: choice.message.content = [{"type": "text", "text": "text here"}, ...] + """ + # Step 0: Check if response has any text content to process + if not self._has_text_content(response): + verbose_proxy_logger.warning( + "OpenAI Chat Completions: No text content in response, skipping guardrail" + ) + return response + + tasks: List[Coroutine[Any, Any, str]] = [] + task_mappings: List[Tuple[int, Optional[int]]] = [] + # Track (choice_index, content_index) for each task + + # Step 1: Extract all text content from response choices + for choice_idx, choice in enumerate(response.choices): + await self._extract_output_text_and_create_tasks( + choice=choice, + choice_idx=choice_idx, + tasks=tasks, + task_mappings=task_mappings, + guardrail_to_apply=guardrail_to_apply, + ) + + # Step 2: Run all guardrail tasks in parallel + responses = await asyncio.gather(*tasks) + + # Step 3: Map guardrail responses back to original response structure + await self._apply_guardrail_responses_to_output( + response=response, + responses=responses, + task_mappings=task_mappings, + ) + + verbose_proxy_logger.debug( + "OpenAI Chat Completions: Processed output response: %s", response + ) + + return response + + def _has_text_content(self, response: "ModelResponse") -> bool: + """ + Check if response has any text content to process. + + Override this method to customize text content detection. + """ + for choice in response.choices: + if isinstance(choice, litellm.Choices): + if choice.message.content and isinstance(choice.message.content, str): + return True + return False + + async def _extract_output_text_and_create_tasks( + self, + choice: Any, + choice_idx: int, + tasks: List, + task_mappings: List[Tuple[int, Optional[int]]], + guardrail_to_apply: "CustomGuardrail", + request_data: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Extract text content from a response choice and create guardrail tasks. + + Override this method to customize text extraction logic. + """ + if not isinstance(choice, litellm.Choices): + return + + verbose_proxy_logger.debug( + "OpenAI Chat Completions: Processing choice: %s", choice + ) + + if choice.message.content and isinstance(choice.message.content, str): + # Simple string content + tasks.append( + guardrail_to_apply.apply_guardrail(text=choice.message.content, request_data=request_data) + ) + task_mappings.append((choice_idx, None)) + + elif choice.message.content and isinstance(choice.message.content, list): + # List content (e.g., multimodal response) + for content_idx, content_item in enumerate(choice.message.content): + content_text = content_item.get("text") + if content_text: + tasks.append(guardrail_to_apply.apply_guardrail(text=content_text, request_data=request_data)) + task_mappings.append((choice_idx, int(content_idx))) + + async def _apply_guardrail_responses_to_output( + self, + response: "ModelResponse", + responses: List[str], + task_mappings: List[Tuple[int, Optional[int]]], + ) -> None: + """ + Apply guardrail responses back to output response. + + Override this method to customize how responses are applied. + """ + for task_idx, guardrail_response in enumerate(responses): + mapping = task_mappings[task_idx] + choice_idx = cast(int, mapping[0]) + content_idx_optional = cast(Optional[int], mapping[1]) + + content = cast(Choices, response.choices[choice_idx]).message.content + if content is None: + continue + + if isinstance(content, str) and content_idx_optional is None: + # Replace string content with guardrail response + cast(Choices, response.choices[choice_idx]).message.content = ( + guardrail_response + ) + + elif isinstance(content, list) and content_idx_optional is not None: + # Replace specific text item in list content + cast(Choices, response.choices[choice_idx]).message.content[ # type: ignore + content_idx_optional + ][ + "text" + ] = guardrail_response diff --git a/litellm/llms/openai/completion/guardrail_translation/README.md b/litellm/llms/openai/completion/guardrail_translation/README.md new file mode 100644 index 00000000000..93762206c47 --- /dev/null +++ b/litellm/llms/openai/completion/guardrail_translation/README.md @@ -0,0 +1,158 @@ +# OpenAI Text Completion Guardrail Translation Handler + +Handler for processing OpenAI's text completion endpoint (`/v1/completions`) with guardrails. + +## Overview + +This handler processes text completion requests by: +1. Extracting the text prompt(s) from the request +2. Applying guardrails to the prompt text(s) +3. Updating the request with the guardrailed prompt(s) +4. Applying guardrails to the completion output text + +## Data Format + +### Input Format + +**Single Prompt:** +```json +{ + "model": "gpt-3.5-turbo-instruct", + "prompt": "Say this is a test", + "max_tokens": 7, + "temperature": 0 +} +``` + +**Multiple Prompts (Batch):** +```json +{ + "model": "gpt-3.5-turbo-instruct", + "prompt": [ + "Tell me a joke", + "Write a poem" + ], + "max_tokens": 50 +} +``` + +### Output Format + +```json +{ + "id": "cmpl-uqkvlQyYK7bGYrRHQ0eXlWi7", + "object": "text_completion", + "created": 1589478378, + "model": "gpt-3.5-turbo-instruct", + "choices": [ + { + "text": "\n\nThis is indeed a test", + "index": 0, + "logprobs": null, + "finish_reason": "length" + } + ], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 7, + "total_tokens": 12 + } +} +``` + +## Usage + +The handler is automatically discovered and applied when guardrails are used with the text completion endpoint. + +### Example: Using Guardrails with Text Completion + +```bash +curl -X POST 'http://localhost:4000/v1/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer your-api-key' \ +-d '{ + "model": "gpt-3.5-turbo-instruct", + "prompt": "Say this is a test", + "guardrails": ["content_moderation"], + "max_tokens": 7 +}' +``` + +The guardrail will be applied to both: +- **Input**: The prompt text before sending to the LLM +- **Output**: The completion text in the response + +### Example: PII Masking in Prompts and Completions + +```bash +curl -X POST 'http://localhost:4000/v1/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer your-api-key' \ +-d '{ + "model": "gpt-3.5-turbo-instruct", + "prompt": "My name is John Doe and my email is john@example.com", + "guardrails": ["mask_pii"], + "metadata": { + "guardrails": ["mask_pii"] + } +}' +``` + +### Example: Batch Prompts with Guardrails + +```bash +curl -X POST 'http://localhost:4000/v1/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer your-api-key' \ +-d '{ + "model": "gpt-3.5-turbo-instruct", + "prompt": [ + "Tell me about AI", + "What is machine learning?" + ], + "guardrails": ["content_filter"], + "max_tokens": 100 +}' +``` + +## Implementation Details + +### Input Processing + +- **Field**: `prompt` (string or list of strings) +- **Processing**: + - String prompts: Apply guardrail directly + - List prompts: Apply guardrail to each string in the list +- **Result**: Updated prompt(s) in request + +### Output Processing + +- **Field**: `choices[*].text` (string) +- **Processing**: Applies guardrail to each completion text +- **Result**: Updated completion texts in response + +### Supported Prompt Types + +1. **String**: Single prompt as a string +2. **List of Strings**: Multiple prompts for batch completion +3. **List of Lists**: Token-based prompts (passed through unchanged) + +## Extension + +Override these methods to customize behavior: + +- `process_input_messages()`: Customize how prompts are processed +- `process_output_response()`: Customize how completion texts are processed + +## Supported Call Types + +- `CallTypes.text_completion` - Synchronous text completion +- `CallTypes.atext_completion` - Asynchronous text completion + +## Notes + +- The handler processes both input prompts and output completion texts +- List prompts are processed individually (each string in the list) +- Non-string prompt items (e.g., token lists) are passed through unchanged +- Both sync and async call types use the same handler + diff --git a/litellm/llms/openai/completion/guardrail_translation/__init__.py b/litellm/llms/openai/completion/guardrail_translation/__init__.py new file mode 100644 index 00000000000..51e43c45937 --- /dev/null +++ b/litellm/llms/openai/completion/guardrail_translation/__init__.py @@ -0,0 +1,13 @@ +"""OpenAI Text Completion handler for Unified Guardrails.""" + +from litellm.llms.openai.completion.guardrail_translation.handler import ( + OpenAITextCompletionHandler, +) +from litellm.types.utils import CallTypes + +guardrail_translation_mappings = { + CallTypes.text_completion: OpenAITextCompletionHandler, + CallTypes.atext_completion: OpenAITextCompletionHandler, +} + +__all__ = ["guardrail_translation_mappings", "OpenAITextCompletionHandler"] diff --git a/litellm/llms/openai/completion/guardrail_translation/handler.py b/litellm/llms/openai/completion/guardrail_translation/handler.py new file mode 100644 index 00000000000..b5db730620e --- /dev/null +++ b/litellm/llms/openai/completion/guardrail_translation/handler.py @@ -0,0 +1,137 @@ +""" +OpenAI Text Completion Handler for Unified Guardrails + +This module provides guardrail translation support for OpenAI's text completion endpoint. +The handler processes the 'prompt' parameter for guardrails. +""" + +from typing import TYPE_CHECKING, Any + +from litellm._logging import verbose_proxy_logger +from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation + +if TYPE_CHECKING: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.utils import TextCompletionResponse + + +class OpenAITextCompletionHandler(BaseTranslation): + """ + Handler for processing OpenAI text completion requests with guardrails. + + This class provides methods to: + 1. Process input prompt (pre-call hook) + 2. Process output response (post-call hook) + + The handler specifically processes the 'prompt' parameter which can be: + - A single string + - A list of strings (for batch completions) + """ + + async def process_input_messages( + self, + data: dict, + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process input prompt by applying guardrails to text content. + + Args: + data: Request data dictionary containing 'prompt' parameter + guardrail_to_apply: The guardrail instance to apply + + Returns: + Modified data with guardrails applied to prompt + """ + prompt = data.get("prompt") + if prompt is None: + verbose_proxy_logger.debug( + "OpenAI Text Completion: No prompt found in request data" + ) + return data + + if isinstance(prompt, str): + # Single string prompt + guardrailed_prompt = await guardrail_to_apply.apply_guardrail(text=prompt) + data["prompt"] = guardrailed_prompt + + verbose_proxy_logger.debug( + "OpenAI Text Completion: Applied guardrail to string prompt. " + "Original length: %d, New length: %d", + len(prompt), + len(guardrailed_prompt), + ) + + elif isinstance(prompt, list): + # List of string prompts (batch completion) + guardrailed_prompts = [] + for idx, p in enumerate(prompt): + if isinstance(p, str): + guardrailed_p = await guardrail_to_apply.apply_guardrail(text=p) + guardrailed_prompts.append(guardrailed_p) + verbose_proxy_logger.debug( + "OpenAI Text Completion: Applied guardrail to prompt[%d]. " + "Original length: %d, New length: %d", + idx, + len(p), + len(guardrailed_p), + ) + else: + # For non-string items (e.g., token lists), keep unchanged + guardrailed_prompts.append(p) + verbose_proxy_logger.debug( + "OpenAI Text Completion: Skipping guardrail for prompt[%d] " + "(not a string, type: %s)", + idx, + type(p), + ) + + data["prompt"] = guardrailed_prompts + + else: + verbose_proxy_logger.warning( + "OpenAI Text Completion: Unexpected prompt type: %s. Expected string or list.", + type(prompt), + ) + + return data + + async def process_output_response( + self, + response: "TextCompletionResponse", + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process output response by applying guardrails to completion text. + + Args: + response: Text completion response object + guardrail_to_apply: The guardrail instance to apply + + Returns: + Modified response with guardrails applied to completion text + """ + if not hasattr(response, "choices") or not response.choices: + verbose_proxy_logger.debug( + "OpenAI Text Completion: No choices in response to process" + ) + return response + + # Apply guardrails to each choice's text + for idx, choice in enumerate(response.choices): + if hasattr(choice, "text") and isinstance(choice.text, str): + original_text = choice.text + guardrailed_text = await guardrail_to_apply.apply_guardrail( + text=original_text + ) + choice.text = guardrailed_text + + verbose_proxy_logger.debug( + "OpenAI Text Completion: Applied guardrail to choice[%d] text. " + "Original length: %d, New length: %d", + idx, + len(original_text), + len(guardrailed_text), + ) + + return response diff --git a/litellm/llms/openai/containers/transformation.py b/litellm/llms/openai/containers/transformation.py new file mode 100644 index 00000000000..1a6343d7be4 --- /dev/null +++ b/litellm/llms/openai/containers/transformation.py @@ -0,0 +1,260 @@ +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union + +import httpx + +import litellm +from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( + StandardBuiltInToolCostTracking, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.containers.main import ( + ContainerCreateOptionalRequestParams, + ContainerListResponse, + ContainerObject, + DeleteContainerResult, +) +from litellm.types.router import GenericLiteLLMParams + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException + from ...base_llm.containers.transformation import BaseContainerConfig as _BaseContainerConfig + + LiteLLMLoggingObj = _LiteLLMLoggingObj + BaseContainerConfig = _BaseContainerConfig + BaseLLMException = _BaseLLMException +else: + LiteLLMLoggingObj = Any + BaseContainerConfig = Any + BaseLLMException = Any + + +class OpenAIContainerConfig(BaseContainerConfig): + """Configuration class for OpenAI container API. + """ + + def __init__(self): + super().__init__() + + def get_supported_openai_params(self) -> list: + """Get the list of supported OpenAI parameters for container API. + """ + return [ + "name", + "expires_after", + "file_ids", + "extra_headers", + ] + + def map_openai_params( + self, + container_create_optional_params: ContainerCreateOptionalRequestParams, + drop_params: bool, + ) -> Dict: + """No mapping applied since inputs are in OpenAI spec already""" + return dict(container_create_optional_params) + + def validate_environment( + self, + headers: dict, + api_key: Optional[str] = None, + ) -> dict: + api_key = ( + api_key + or litellm.api_key + or litellm.openai_key + or get_secret_str("OPENAI_API_KEY") + ) + headers.update( + { + "Authorization": f"Bearer {api_key}", + }, + ) + return headers + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """Get the complete URL for OpenAI container API. + """ + if api_base is None: + api_base = "https://api.openai.com/v1" + + return f"{api_base.rstrip('/')}/containers" + + def transform_container_create_request( + self, + name: str, + container_create_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + """Transform the container creation request for OpenAI API. + """ + # Remove extra_headers from optional params as they're handled separately + container_create_optional_request_params = { + k: v for k, v in container_create_optional_request_params.items() + if k not in ["extra_headers"] + } + + # Create the request data + request_dict = { + "name": name, + **container_create_optional_request_params, + } + + return request_dict + + def transform_container_create_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ContainerObject: + """Transform the OpenAI container creation response. + """ + response_data = raw_response.json() + + # Transform the response data + container_obj = ContainerObject(**response_data) # type: ignore[arg-type] + + # Add cost for container creation (OpenAI containers are code interpreter sessions) + # https://platform.openai.com/docs/pricing + # Each container creation is 1 code interpreter session + container_cost = StandardBuiltInToolCostTracking.get_cost_for_code_interpreter( + sessions=1, + provider="openai", + ) + + if not hasattr(container_obj, "_hidden_params") or container_obj._hidden_params is None: + container_obj._hidden_params = {} + if "additional_headers" not in container_obj._hidden_params: + container_obj._hidden_params["additional_headers"] = {} + container_obj._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = container_cost + + return container_obj + + def transform_container_list_request( + self, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + after: Optional[str] = None, + limit: Optional[int] = None, + order: Optional[str] = None, + extra_query: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + """Transform the container list request for OpenAI API. + + OpenAI API expects the following request: + - GET /v1/containers + """ + # Use the api_base directly for container list + url = api_base + + # Prepare query parameters + params = {} + if after is not None: + params["after"] = after + if limit is not None: + params["limit"] = str(limit) + if order is not None: + params["order"] = order + + # Add any extra query parameters + if extra_query: + params.update(extra_query) + + return url, params + + def transform_container_list_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ContainerListResponse: + """Transform the OpenAI container list response. + """ + response_data = raw_response.json() + + # Transform the response data + container_list = ContainerListResponse(**response_data) # type: ignore[arg-type] + + return container_list + + def transform_container_retrieve_request( + self, + container_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Transform the OpenAI container retrieve request. + """ + # For container retrieve, we just need to construct the URL + url = f"{api_base.rstrip('/')}/{container_id}" + + # No additional data needed for GET request + data: Dict[str, Any] = {} + + return url, data + + def transform_container_retrieve_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> ContainerObject: + """Transform the OpenAI container retrieve response. + """ + response_data = raw_response.json() + # Transform the response data + container_obj = ContainerObject(**response_data) # type: ignore[arg-type] + + return container_obj + + def transform_container_delete_request( + self, + container_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """Transform the container delete request for OpenAI API. + + OpenAI API expects the following request: + - DELETE /v1/containers/{container_id} + """ + # Construct the URL for container delete + url = f"{api_base.rstrip('/')}/{container_id}" + + # No data needed for DELETE request + data: Dict[str, Any] = {} + + return url, data + + def transform_container_delete_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> DeleteContainerResult: + """Transform the OpenAI container delete response. + """ + response_data = raw_response.json() + + # Transform the response data + delete_result = DeleteContainerResult(**response_data) # type: ignore[arg-type] + + return delete_result + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers], + ) -> BaseLLMException: + from ...base_llm.chat.transformation import BaseLLMException + + raise BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + ) + diff --git a/litellm/llms/openai/cost_calculation.py b/litellm/llms/openai/cost_calculation.py index 229f75f2657..e5349db3af7 100644 --- a/litellm/llms/openai/cost_calculation.py +++ b/litellm/llms/openai/cost_calculation.py @@ -18,7 +18,9 @@ def cost_router(call_type: CallTypes) -> Literal["cost_per_token", "cost_per_sec return "cost_per_token" -def cost_per_token(model: str, usage: Usage, service_tier: Optional[str] = None) -> Tuple[float, float]: +def cost_per_token( + model: str, usage: Usage, service_tier: Optional[str] = None +) -> Tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -31,7 +33,10 @@ def cost_per_token(model: str, usage: Usage, service_tier: Optional[str] = None) """ ## CALCULATE INPUT COST return generic_cost_per_token( - model=model, usage=usage, custom_llm_provider="openai", service_tier=service_tier + model=model, + usage=usage, + custom_llm_provider="openai", + service_tier=service_tier, ) # ### Non-cached text tokens # non_cached_text_tokens = usage.prompt_tokens @@ -92,6 +97,7 @@ def cost_per_second( Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ + ## GET MODEL INFO model_info = get_model_info( model=model, custom_llm_provider=custom_llm_provider or "openai" @@ -120,3 +126,45 @@ def cost_per_second( completion_cost = 0.0 return prompt_cost, completion_cost + + +def video_generation_cost( + model: str, duration_seconds: float, custom_llm_provider: Optional[str] = None +) -> float: + """ + Calculates the cost for video generation based on duration in seconds. + + Input: + - model: str, the model name without provider prefix + - duration_seconds: float, the duration of the generated video in seconds + - custom_llm_provider: str, the custom llm provider + + Returns: + float - total_cost_in_usd + """ + ## GET MODEL INFO + model_info = get_model_info( + model=model, custom_llm_provider=custom_llm_provider or "openai" + ) + + # Check for video-specific cost per second + video_cost_per_second = model_info.get("output_cost_per_video_per_second") + if video_cost_per_second is not None: + verbose_logger.debug( + f"For model={model} - output_cost_per_video_per_second: {video_cost_per_second}; duration: {duration_seconds}" + ) + return video_cost_per_second * duration_seconds + + # Fallback to general output cost per second + output_cost_per_second = model_info.get("output_cost_per_second") + if output_cost_per_second is not None: + verbose_logger.debug( + f"For model={model} - output_cost_per_second: {output_cost_per_second}; duration: {duration_seconds}" + ) + return output_cost_per_second * duration_seconds + + # If no cost information found, return 0 + verbose_logger.warning( + f"No cost information found for video model {model}. Please add pricing to model_prices_and_context_window.json" + ) + return 0.0 diff --git a/litellm/llms/openai/image_edit/__init__.py b/litellm/llms/openai/image_edit/__init__.py new file mode 100644 index 00000000000..c1898326b72 --- /dev/null +++ b/litellm/llms/openai/image_edit/__init__.py @@ -0,0 +1,26 @@ +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig + +from .dalle2_transformation import DallE2ImageEditConfig +from .transformation import OpenAIImageEditConfig + +__all__ = ["OpenAIImageEditConfig", "DallE2ImageEditConfig", "get_openai_image_edit_config"] + + +def get_openai_image_edit_config(model: str) -> BaseImageEditConfig: + """ + Get the appropriate OpenAI image edit config based on the model. + + Args: + model: The model name (e.g., "dall-e-2", "gpt-image-1") + + Returns: + The appropriate config instance for the model + """ + model_normalized = model.lower().replace("-", "").replace("_", "") + + if model_normalized == "dalle2": + return DallE2ImageEditConfig() + else: + # Default to standard OpenAI config for gpt-image-1 and other models + return OpenAIImageEditConfig() + diff --git a/litellm/llms/openai/image_edit/dalle2_transformation.py b/litellm/llms/openai/image_edit/dalle2_transformation.py new file mode 100644 index 00000000000..37e92be17a8 --- /dev/null +++ b/litellm/llms/openai/image_edit/dalle2_transformation.py @@ -0,0 +1,101 @@ +from io import BufferedReader +from typing import TYPE_CHECKING, Any, Dict, List, Tuple, cast + +from httpx._types import RequestFiles + +import litellm +from litellm.images.utils import ImageEditRequestUtils +from litellm.types.images.main import ImageEditRequestParams +from litellm.types.llms.openai import FileTypes +from litellm.types.router import GenericLiteLLMParams + +from .transformation import OpenAIImageEditConfig + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class DallE2ImageEditConfig(OpenAIImageEditConfig): + """ + DALL-E-2 specific configuration for image edit API. + + DALL-E-2 only supports editing a single image (not an array). + Uses "image" field name instead of "image[]". + """ + + def transform_image_edit_request( + self, + model: str, + prompt: str, + image: FileTypes, + image_edit_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[Dict, RequestFiles]: + """ + Transform image edit request for DALL-E-2. + + DALL-E-2 only accepts a single image with field name "image" (not "image[]"). + """ + request = ImageEditRequestParams( + model=model, + image=image, + prompt=prompt, + **image_edit_optional_request_params, + ) + request_dict = cast(Dict, request) + + ######################################################### + # Separate images and masks as `files` and send other parameters as `data` + ######################################################### + _image_list = request_dict.get("image") + _mask = request_dict.get("mask") + data_without_files = { + k: v for k, v in request_dict.items() if k not in ["image", "mask"] + } + files_list: List[Tuple[str, Any]] = [] + + # Handle image parameter - DALL-E-2 only supports single image + if _image_list is not None: + image_list = ( + [_image_list] if not isinstance(_image_list, list) else _image_list + ) + + # Validate only one image is provided + if len(image_list) > 1: + raise litellm.BadRequestError( + message="DALL-E-2 only supports editing a single image. Please provide one image.", + model=model, + llm_provider="openai", + ) + + # Use "image" field name (singular) for DALL-E-2 + for _image in image_list: + if _image is not None: + self._add_image_to_files( + files_list=files_list, + image=_image, + field_name="image", + ) + + # Handle mask parameter if provided + if _mask is not None: + # Handle case where mask can be a list (extract first mask) + if isinstance(_mask, list): + _mask = _mask[0] if _mask else None + + if _mask is not None: + mask_content_type: str = ImageEditRequestUtils.get_image_content_type( + _mask + ) + if isinstance(_mask, BufferedReader): + files_list.append(("mask", (_mask.name, _mask, mask_content_type))) + else: + files_list.append(("mask", ("mask.png", _mask, mask_content_type))) + + return data_without_files, files_list + diff --git a/litellm/llms/openai/image_edit/transformation.py b/litellm/llms/openai/image_edit/transformation.py index be960641154..1b90d96fa92 100644 --- a/litellm/llms/openai/image_edit/transformation.py +++ b/litellm/llms/openai/image_edit/transformation.py @@ -27,6 +27,11 @@ else: class OpenAIImageEditConfig(BaseImageEditConfig): + """ + Base configuration for OpenAI image edit API. + Used for models like gpt-image-1 that support multiple images. + """ + def get_supported_openai_params(self, model: str) -> list: """ All OpenAI Image Edits params are supported @@ -57,6 +62,20 @@ class OpenAIImageEditConfig(BaseImageEditConfig): """No mapping applied since inputs are in OpenAI spec already""" return dict(image_edit_optional_params) + def _add_image_to_files( + self, + files_list: List[Tuple[str, Any]], + image: Any, + field_name: str, + ) -> None: + """Add an image to the files list with appropriate content type""" + image_content_type = ImageEditRequestUtils.get_image_content_type(image) + + if isinstance(image, BufferedReader): + files_list.append((field_name, (image.name, image, image_content_type))) + else: + files_list.append((field_name, ("image.png", image, image_content_type))) + def transform_image_edit_request( self, model: str, @@ -67,9 +86,10 @@ class OpenAIImageEditConfig(BaseImageEditConfig): headers: dict, ) -> Tuple[Dict, RequestFiles]: """ - No transform applied since inputs are in OpenAI spec already + Transform image edit request to OpenAI API format. - This handles buffered readers as images to be sent as multipart/form-data for OpenAI + Handles multipart/form-data for images. Uses "image[]" field name + to support multiple images (e.g., for gpt-image-1). """ request = ImageEditRequestParams( model=model, @@ -94,19 +114,14 @@ class OpenAIImageEditConfig(BaseImageEditConfig): image_list = ( [_image_list] if not isinstance(_image_list, list) else _image_list ) + for _image in image_list: if _image is not None: - image_content_type: str = ( - ImageEditRequestUtils.get_image_content_type(_image) + self._add_image_to_files( + files_list=files_list, + image=_image, + field_name="image[]", ) - if isinstance(_image, BufferedReader): - files_list.append( - ("image[]", (_image.name, _image, image_content_type)) - ) - else: - files_list.append( - ("image[]", ("image.png", _image, image_content_type)) - ) # Handle mask parameter if provided if _mask is not None: # Handle case where mask can be a list (extract first mask) diff --git a/litellm/llms/openai/image_generation/__init__.py b/litellm/llms/openai/image_generation/__init__.py index eb2a0576b66..e20c80f20bb 100644 --- a/litellm/llms/openai/image_generation/__init__.py +++ b/litellm/llms/openai/image_generation/__init__.py @@ -5,11 +5,17 @@ from litellm.llms.base_llm.image_generation.transformation import ( from .dall_e_2_transformation import DallE2ImageGenerationConfig from .dall_e_3_transformation import DallE3ImageGenerationConfig from .gpt_transformation import GPTImageGenerationConfig +from .guardrail_translation import ( + OpenAIImageGenerationHandler, + guardrail_translation_mappings, +) __all__ = [ "DallE2ImageGenerationConfig", "DallE3ImageGenerationConfig", "GPTImageGenerationConfig", + "OpenAIImageGenerationHandler", + "guardrail_translation_mappings", ] diff --git a/litellm/llms/openai/image_generation/dall_e_2_transformation.py b/litellm/llms/openai/image_generation/dall_e_2_transformation.py index 8e306a83375..22c2349a837 100644 --- a/litellm/llms/openai/image_generation/dall_e_2_transformation.py +++ b/litellm/llms/openai/image_generation/dall_e_2_transformation.py @@ -1,9 +1,16 @@ -from typing import List +from typing import TYPE_CHECKING, Any, List, Optional + +import httpx from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams +from litellm.types.utils import ImageResponse +from litellm.utils import convert_to_model_response_object + +if TYPE_CHECKING: + from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj class DallE2ImageGenerationConfig(BaseImageGenerationConfig): @@ -36,3 +43,45 @@ class DallE2ImageGenerationConfig(BaseImageGenerationConfig): ) return optional_params + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: "LiteLLMLoggingObj", + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + response = raw_response.json() + + stringified_response = response + ## LOGGING + logging_obj.post_call( + input=request_data.get("prompt", ""), + api_key=api_key, + additional_args={"complete_input_dict": request_data}, + original_response=stringified_response, + ) + image_response: ImageResponse = convert_to_model_response_object( # type: ignore + response_object=stringified_response, + model_response_object=model_response, + response_type="image_generation", + ) + + # set optional params + image_response.size = optional_params.get( + "size", "1024x1024" + ) # default is always 1024x1024 + image_response.quality = optional_params.get( + "quality", "standard" + ) # always standard for dall-e-2 + image_response.output_format = optional_params.get( + "output_format", "png" + ) # always png for dall-e-2 + + return image_response diff --git a/litellm/llms/openai/image_generation/dall_e_3_transformation.py b/litellm/llms/openai/image_generation/dall_e_3_transformation.py index c4b0b66e112..9e2bdabc3a1 100644 --- a/litellm/llms/openai/image_generation/dall_e_3_transformation.py +++ b/litellm/llms/openai/image_generation/dall_e_3_transformation.py @@ -1,9 +1,16 @@ -from typing import List +from typing import TYPE_CHECKING, Any, List, Optional + +import httpx from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams +from litellm.types.utils import ImageResponse +from litellm.utils import convert_to_model_response_object + +if TYPE_CHECKING: + from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj class DallE3ImageGenerationConfig(BaseImageGenerationConfig): @@ -36,3 +43,45 @@ class DallE3ImageGenerationConfig(BaseImageGenerationConfig): ) return optional_params + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: "LiteLLMLoggingObj", + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + response = raw_response.json() + + stringified_response = response + ## LOGGING + logging_obj.post_call( + input=request_data.get("prompt", ""), + api_key=api_key, + additional_args={"complete_input_dict": request_data}, + original_response=stringified_response, + ) + image_response: ImageResponse = convert_to_model_response_object( # type: ignore + response_object=stringified_response, + model_response_object=model_response, + response_type="image_generation", + ) + + # set optional params + image_response.size = optional_params.get( + "size", "1024x1024" + ) # default is always 1024x1024 + image_response.quality = optional_params.get( + "quality", "hd" + ) # always hd for dall-e-3 + image_response.output_format = optional_params.get( + "output_format", "png" + ) # always png for dall-e-3 + + return image_response diff --git a/litellm/llms/openai/image_generation/gpt_transformation.py b/litellm/llms/openai/image_generation/gpt_transformation.py index 1cee13784e7..c106d7f17b6 100644 --- a/litellm/llms/openai/image_generation/gpt_transformation.py +++ b/litellm/llms/openai/image_generation/gpt_transformation.py @@ -1,9 +1,16 @@ -from typing import List +from typing import TYPE_CHECKING, Any, List, Optional + +import httpx from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams +from litellm.types.utils import ImageResponse +from litellm.utils import convert_to_model_response_object + +if TYPE_CHECKING: + from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj class GPTImageGenerationConfig(BaseImageGenerationConfig): @@ -45,3 +52,45 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig): ) return optional_params + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: "LiteLLMLoggingObj", + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + response = raw_response.json() + + stringified_response = response + ## LOGGING + logging_obj.post_call( + input=request_data.get("prompt", ""), + api_key=api_key, + additional_args={"complete_input_dict": request_data}, + original_response=stringified_response, + ) + image_response: ImageResponse = convert_to_model_response_object( # type: ignore + response_object=stringified_response, + model_response_object=model_response, + response_type="image_generation", + ) + + # set optional params + image_response.size = optional_params.get( + "size", "1024x1024" + ) # default is always 1024x1024 + image_response.quality = optional_params.get( + "quality", "high" + ) # always hd for dall-e-3 + image_response.output_format = optional_params.get( + "response_format", "png" + ) # always png for dall-e-3 + + return image_response diff --git a/litellm/llms/openai/image_generation/guardrail_translation/README.md b/litellm/llms/openai/image_generation/guardrail_translation/README.md new file mode 100644 index 00000000000..fcbd2d154de --- /dev/null +++ b/litellm/llms/openai/image_generation/guardrail_translation/README.md @@ -0,0 +1,106 @@ +# OpenAI Image Generation Guardrail Translation Handler + +Handler for processing OpenAI's image generation endpoint with guardrails. + +## Overview + +This handler processes image generation requests by: +1. Extracting the text prompt from the request +2. Applying guardrails to the prompt text +3. Updating the request with the guardrailed prompt + +## Data Format + +### Input Format + +```json +{ + "model": "dall-e-3", + "prompt": "A cute baby sea otter", + "n": 1, + "size": "1024x1024", + "quality": "standard" +} +``` + +### Output Format + +```json +{ + "created": 1589478378, + "data": [ + { + "url": "https://...", + "revised_prompt": "A cute baby sea otter..." + } + ] +} +``` + +## Usage + +The handler is automatically discovered and applied when guardrails are used with the image generation endpoint. + +### Example: Using Guardrails with Image Generation + +```bash +curl -X POST 'http://localhost:4000/v1/images/generations' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer your-api-key' \ +-d '{ + "model": "dall-e-3", + "prompt": "A cute baby sea otter wearing a hat", + "guardrails": ["content_moderation"], + "size": "1024x1024" +}' +``` + +The guardrail will be applied to the prompt text before the image generation request is sent to the provider. + +### Example: PII Masking in Prompts + +```bash +curl -X POST 'http://localhost:4000/v1/images/generations' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer your-api-key' \ +-d '{ + "model": "dall-e-3", + "prompt": "Generate an image of John Doe at john@example.com", + "guardrails": ["mask_pii"], + "metadata": { + "guardrails": ["mask_pii"] + } +}' +``` + +## Implementation Details + +### Input Processing + +- **Field**: `prompt` (string) +- **Processing**: Applies guardrail to prompt text +- **Result**: Updated prompt in request + +### Output Processing + +- **Processing**: Not applicable (images don't contain text to guardrail) +- **Result**: Response returned unchanged + +## Extension + +Override these methods to customize behavior: + +- `process_input_messages()`: Customize how the prompt is processed +- `process_output_response()`: Add custom processing for image metadata if needed + +## Supported Call Types + +- `CallTypes.image_generation` - Synchronous image generation +- `CallTypes.aimage_generation` - Asynchronous image generation + +## Notes + +- The handler only processes the `prompt` parameter +- Output processing is a no-op since images don't contain text +- Both sync and async call types use the same handler + diff --git a/litellm/llms/openai/image_generation/guardrail_translation/__init__.py b/litellm/llms/openai/image_generation/guardrail_translation/__init__.py new file mode 100644 index 00000000000..1fba2a36927 --- /dev/null +++ b/litellm/llms/openai/image_generation/guardrail_translation/__init__.py @@ -0,0 +1,13 @@ +"""OpenAI Image Generation handler for Unified Guardrails.""" + +from litellm.llms.openai.image_generation.guardrail_translation.handler import ( + OpenAIImageGenerationHandler, +) +from litellm.types.utils import CallTypes + +guardrail_translation_mappings = { + CallTypes.image_generation: OpenAIImageGenerationHandler, + CallTypes.aimage_generation: OpenAIImageGenerationHandler, +} + +__all__ = ["guardrail_translation_mappings", "OpenAIImageGenerationHandler"] diff --git a/litellm/llms/openai/image_generation/guardrail_translation/handler.py b/litellm/llms/openai/image_generation/guardrail_translation/handler.py new file mode 100644 index 00000000000..5fcb5278f01 --- /dev/null +++ b/litellm/llms/openai/image_generation/guardrail_translation/handler.py @@ -0,0 +1,93 @@ +""" +OpenAI Image Generation Handler for Unified Guardrails + +This module provides guardrail translation support for OpenAI's image generation endpoint. +The handler processes the 'prompt' parameter for guardrails. +""" + +from typing import TYPE_CHECKING, Any + +from litellm._logging import verbose_proxy_logger +from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation + +if TYPE_CHECKING: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.utils import ImageResponse + + +class OpenAIImageGenerationHandler(BaseTranslation): + """ + Handler for processing OpenAI image generation requests with guardrails. + + This class provides methods to: + 1. Process input prompt (pre-call hook) + 2. Process output response (post-call hook) - typically not needed for images + + The handler specifically processes the 'prompt' parameter which contains + the text description for image generation. + """ + + async def process_input_messages( + self, + data: dict, + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process input prompt by applying guardrails to text content. + + Args: + data: Request data dictionary containing 'prompt' parameter + guardrail_to_apply: The guardrail instance to apply + + Returns: + Modified data with guardrails applied to prompt + """ + prompt = data.get("prompt") + if prompt is None: + verbose_proxy_logger.debug( + "OpenAI Image Generation: No prompt found in request data" + ) + return data + + # Apply guardrail to the prompt + if isinstance(prompt, str): + guardrailed_prompt = await guardrail_to_apply.apply_guardrail(text=prompt) + data["prompt"] = guardrailed_prompt + + verbose_proxy_logger.debug( + "OpenAI Image Generation: Applied guardrail to prompt. " + "Original length: %d, New length: %d", + len(prompt), + len(guardrailed_prompt), + ) + else: + verbose_proxy_logger.debug( + "OpenAI Image Generation: Unexpected prompt type: %s. Expected string.", + type(prompt), + ) + + return data + + async def process_output_response( + self, + response: "ImageResponse", + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process output response - typically not needed for image generation. + + Image responses don't contain text to apply guardrails to, so this + method returns the response unchanged. This is provided for completeness + and can be overridden if needed for custom image metadata processing. + + Args: + response: Image generation response object + guardrail_to_apply: The guardrail instance to apply + + Returns: + Unmodified response (images don't need text guardrails) + """ + verbose_proxy_logger.debug( + "OpenAI Image Generation: Output processing not needed for image responses" + ) + return response diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 324205237dc..2949e35e5e7 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1203,7 +1203,6 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): ) -> EmbeddingResponse: super().embedding() try: - model = model data = {"model": model, "input": input, **optional_params} max_retries = max_retries or litellm.DEFAULT_MAX_RETRIES if not isinstance(max_retries, int): @@ -1286,6 +1285,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): api_base: Optional[str] = None, client=None, max_retries=None, + organization: Optional[str] = None, ): response = None try: @@ -1295,6 +1295,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): api_base=api_base, timeout=timeout, max_retries=max_retries, + organization=organization, client=client, ) @@ -1329,17 +1330,17 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): model_response: Optional[ImageResponse] = None, client=None, aimg_generation=None, + organization: Optional[str] = None, ) -> ImageResponse: data = {} try: - model = model data = {"model": model, "prompt": prompt, **optional_params} max_retries = data.pop("max_retries", 2) if not isinstance(max_retries, int): raise OpenAIError(status_code=422, message="max retries must be an int") if aimg_generation is True: - return self.aimage_generation(data=data, prompt=prompt, logging_obj=logging_obj, model_response=model_response, api_base=api_base, api_key=api_key, timeout=timeout, client=client, max_retries=max_retries) # type: ignore + return self.aimage_generation(data=data, prompt=prompt, logging_obj=logging_obj, model_response=model_response, api_base=api_base, api_key=api_key, timeout=timeout, client=client, max_retries=max_retries, organization=organization) # type: ignore openai_client: OpenAI = self._get_openai_client( # type: ignore is_async=False, @@ -1347,6 +1348,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): api_base=api_base, timeout=timeout, max_retries=max_retries, + organization=organization, client=client, ) diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index e0c85d18178..e1fb3f12602 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -6,10 +6,12 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. from typing import Any, Optional, cast +from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.types.realtime import RealtimeQueryParams + from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from ....litellm_core_utils.realtime_streaming import RealTimeStreaming from ..openai import OpenAIChatCompletion -from litellm.types.realtime import RealtimeQueryParams class OpenAIRealtime(OpenAIChatCompletion): @@ -59,6 +61,7 @@ class OpenAIRealtime(OpenAIChatCompletion): "Authorization": f"Bearer {api_key}", # type: ignore "OpenAI-Beta": "realtime=v1", }, + max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ) as backend_ws: realtime_streaming = RealTimeStreaming( websocket, cast(ClientConnection, backend_ws), logging_obj diff --git a/litellm/llms/openai/responses/guardrail_translation/README.md b/litellm/llms/openai/responses/guardrail_translation/README.md new file mode 100644 index 00000000000..bc1bd6f4f2c --- /dev/null +++ b/litellm/llms/openai/responses/guardrail_translation/README.md @@ -0,0 +1,119 @@ +# OpenAI Responses API Guardrail Translation Handler + +This module provides guardrail translation support for the OpenAI Responses API format. + +## Overview + +The `OpenAIResponsesHandler` class handles the translation of guardrail operations for both input and output of the Responses API. It follows the same pattern as the Chat Completions handler but is adapted for the Responses API's specific data structures. + +## Responses API Format + +### Input Format +The Responses API accepts input in two formats: + +1. **String input**: Simple text string + ```python + {"input": "Hello world", "model": "gpt-4"} + ``` + +2. **List input**: Array of message objects (ResponseInputParam) + ```python + { + "input": [ + { + "role": "user", + "content": "Hello", # Can be string or list of content items + "type": "message" + } + ], + "model": "gpt-4" + } + ``` + +### Output Format +The Responses API returns a `ResponsesAPIResponse` object with: + +```python +{ + "id": "resp_123", + "output": [ + { + "type": "message", + "id": "msg_123", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Assistant response", + "annotations": [] + } + ] + } + ] +} +``` + +## Usage + +The handler is automatically discovered and registered for `CallTypes.responses` and `CallTypes.aresponses`. + +### Example + +```python +from litellm.llms import get_guardrail_translation_mapping +from litellm.types.utils import CallTypes + +# Get the handler +handler_class = get_guardrail_translation_mapping(CallTypes.responses) +handler = handler_class() + +# Process input +data = {"input": "User message", "model": "gpt-4"} +processed_data = await handler.process_input_messages(data, guardrail_instance) + +# Process output +response = await litellm.aresponses(**processed_data) +processed_response = await handler.process_output_response(response, guardrail_instance) +``` + +## Key Methods + +### `process_input_messages(data, guardrail_to_apply)` +Processes input data by: +1. Handling both string and list input formats +2. Extracting text content from messages +3. Applying guardrails to text content in parallel +4. Mapping guardrail responses back to the original structure + +### `process_output_response(response, guardrail_to_apply)` +Processes output response by: +1. Extracting text from output items' content +2. Applying guardrails to all text content in parallel +3. Replacing original text with guardrailed versions + +## Extending the Handler + +The handler can be customized by overriding these methods: + +- `_extract_input_text_and_create_tasks()`: Customize input text extraction logic +- `_apply_guardrail_responses_to_input()`: Customize how guardrail responses are applied to input +- `_extract_output_text_and_create_tasks()`: Customize output text extraction logic +- `_apply_guardrail_responses_to_output()`: Customize how guardrail responses are applied to output +- `_has_text_content()`: Customize text content detection + +## Testing + +Comprehensive tests are available in `tests/llm_translation/test_openai_responses_guardrail_handler.py`: + +```bash +pytest tests/llm_translation/test_openai_responses_guardrail_handler.py -v +``` + +## Implementation Details + +- **Parallel Processing**: All text content is processed in parallel using `asyncio.gather()` +- **Mapping Tracking**: Uses tuples to track the location of each text segment for accurate replacement +- **Type Safety**: Handles both Pydantic objects and dict representations +- **Multimodal Support**: Properly handles mixed content with text and other media types + diff --git a/litellm/llms/openai/responses/guardrail_translation/__init__.py b/litellm/llms/openai/responses/guardrail_translation/__init__.py new file mode 100644 index 00000000000..d2d9e5375c1 --- /dev/null +++ b/litellm/llms/openai/responses/guardrail_translation/__init__.py @@ -0,0 +1,12 @@ +"""OpenAI Responses API handler for Unified Guardrails.""" + +from litellm.llms.openai.responses.guardrail_translation.handler import ( + OpenAIResponsesHandler, +) +from litellm.types.utils import CallTypes + +guardrail_translation_mappings = { + CallTypes.responses: OpenAIResponsesHandler, + CallTypes.aresponses: OpenAIResponsesHandler, +} +__all__ = ["guardrail_translation_mappings"] diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py new file mode 100644 index 00000000000..fdac13176b1 --- /dev/null +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -0,0 +1,332 @@ +""" +OpenAI Responses API Handler for Unified Guardrails + +This module provides a class-based handler for OpenAI Responses API format. +The class methods can be overridden for custom behavior. + +Pattern Overview: +----------------- +1. Extract text content from input/output (both string and list formats) +2. Create async tasks to apply guardrails to each text segment +3. Track mappings to know where each response belongs +4. Apply guardrail responses back to the original structure + +Responses API Format: +--------------------- +Input: Union[str, List[Dict]] where each dict has: + - role: str + - content: Union[str, List[Dict]] (can have text items) + - type: str (e.g., "message") + +Output: response.output is List[GenericResponseOutputItem] where each has: + - type: str (e.g., "message") + - id: str + - status: str + - role: str + - content: List[OutputText] where OutputText has: + - type: str (e.g., "output_text") + - text: str +""" + +import asyncio +from typing import TYPE_CHECKING, Any, Coroutine, List, Optional, Tuple, Union, cast + +from litellm._logging import verbose_proxy_logger +from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.types.responses.main import GenericResponseOutputItem, OutputText + +if TYPE_CHECKING: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.llms.openai import ResponseInputParam + from litellm.types.utils import ResponsesAPIResponse + + +class OpenAIResponsesHandler(BaseTranslation): + """ + Handler for processing OpenAI Responses API with guardrails. + + This class provides methods to: + 1. Process input (pre-call hook) + 2. Process output response (post-call hook) + + Methods can be overridden to customize behavior for different message formats. + """ + + async def process_input_messages( + self, + data: dict, + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process input by applying guardrails to text content. + + Handles both string input and list of message objects. + """ + input_data: Optional[Union[str, "ResponseInputParam"]] = data.get("input") + if input_data is None: + return data + + # Handle simple string input + if isinstance(input_data, str): + guardrail_response = await guardrail_to_apply.apply_guardrail( + text=input_data + ) + data["input"] = guardrail_response + verbose_proxy_logger.debug("OpenAI Responses API: Processed string input") + return data + + # Handle list input (ResponseInputParam) + if not isinstance(input_data, list): + return data + + tasks: List[Coroutine[Any, Any, str]] = [] + task_mappings: List[Tuple[int, Optional[int]]] = [] + # Track (message_index, content_index) for each task + # content_index is None for string content, int for list content + + # Step 1: Extract all text content and create guardrail tasks + for msg_idx, message in enumerate(input_data): + await self._extract_input_text_and_create_tasks( + message=message, + msg_idx=msg_idx, + tasks=tasks, + task_mappings=task_mappings, + guardrail_to_apply=guardrail_to_apply, + ) + + # Step 2: Run all guardrail tasks in parallel + if tasks: + responses = await asyncio.gather(*tasks) + + # Step 3: Map guardrail responses back to original input structure + await self._apply_guardrail_responses_to_input( + messages=input_data, + responses=responses, + task_mappings=task_mappings, + ) + + verbose_proxy_logger.debug( + "OpenAI Responses API: Processed input messages: %s", input_data + ) + + return data + + async def _extract_input_text_and_create_tasks( + self, + message: Any, # Can be Dict[str, Any] or ResponseInputParam + msg_idx: int, + tasks: List[Coroutine[Any, Any, str]], + task_mappings: List[Tuple[int, Optional[int]]], + guardrail_to_apply: "CustomGuardrail", + ) -> None: + """ + Extract text content from an input message and create guardrail tasks. + + Override this method to customize text extraction logic. + """ + content = message.get("content", None) + if content is None: + return + + if isinstance(content, str): + # Simple string content + tasks.append(guardrail_to_apply.apply_guardrail(text=content)) + task_mappings.append((msg_idx, None)) + + elif isinstance(content, list): + # List content (e.g., multimodal with text and images) + for content_idx, content_item in enumerate(content): + if isinstance(content_item, dict): + text_str = content_item.get("text", None) + if text_str is not None: + tasks.append(guardrail_to_apply.apply_guardrail(text=text_str)) + task_mappings.append((msg_idx, int(content_idx))) + + async def _apply_guardrail_responses_to_input( + self, + messages: Any, # Can be List[Dict[str, Any]] or ResponseInputParam + responses: List[str], + task_mappings: List[Tuple[int, Optional[int]]], + ) -> None: + """ + Apply guardrail responses back to input messages. + + Override this method to customize how responses are applied. + """ + for task_idx, guardrail_response in enumerate(responses): + mapping = task_mappings[task_idx] + msg_idx = cast(int, mapping[0]) + content_idx_optional = cast(Optional[int], mapping[1]) + + content = messages[msg_idx].get("content", None) + if content is None: + continue + + if isinstance(content, str) and content_idx_optional is None: + # Replace string content with guardrail response + messages[msg_idx]["content"] = guardrail_response + + elif isinstance(content, list) and content_idx_optional is not None: + # Replace specific text item in list content + if isinstance(messages[msg_idx]["content"][content_idx_optional], dict): + messages[msg_idx]["content"][content_idx_optional][ + "text" + ] = guardrail_response + + async def process_output_response( + self, + response: "ResponsesAPIResponse", + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process output response by applying guardrails to text content. + + Args: + response: LiteLLM ResponsesAPIResponse object + guardrail_to_apply: The guardrail instance to apply + + Returns: + Modified response with guardrail applied to content + + Response Format Support: + - response.output is a list of output items + - Each output item has a content list with OutputText objects + - Each OutputText object has a text field + """ + # Step 0: Check if response has any text content to process + if not self._has_text_content(response): + verbose_proxy_logger.warning( + "OpenAI Responses API: No text content in response, skipping guardrail" + ) + return response + + tasks: List[Coroutine[Any, Any, str]] = [] + task_mappings: List[Tuple[int, int]] = [] + # Track (output_item_index, content_index) for each task + + # Step 1: Extract all text content from response output + for output_idx, output_item in enumerate(response.output): + await self._extract_output_text_and_create_tasks( + output_item=output_item, + output_idx=output_idx, + tasks=tasks, + task_mappings=task_mappings, + guardrail_to_apply=guardrail_to_apply, + ) + + # Step 2: Run all guardrail tasks in parallel + if tasks: + responses = await asyncio.gather(*tasks) + + # Step 3: Map guardrail responses back to original response structure + await self._apply_guardrail_responses_to_output( + response=response, + responses=responses, + task_mappings=task_mappings, + ) + + verbose_proxy_logger.debug( + "OpenAI Responses API: Processed output response: %s", response + ) + + return response + + def _has_text_content(self, response: "ResponsesAPIResponse") -> bool: + """ + Check if response has any text content to process. + + Override this method to customize text content detection. + """ + if not hasattr(response, "output") or response.output is None: + return False + + for output_item in response.output: + if isinstance(output_item, (GenericResponseOutputItem, dict)): + content = ( + output_item.content + if isinstance(output_item, GenericResponseOutputItem) + else output_item.get("content", []) + ) + if content: + for content_item in content: + # Check if it's an OutputText with text + if isinstance(content_item, OutputText): + if content_item.text: + return True + elif isinstance(content_item, dict): + if content_item.get("text"): + return True + return False + + async def _extract_output_text_and_create_tasks( + self, + output_item: Any, + output_idx: int, + tasks: List, + task_mappings: List[Tuple[int, int]], + guardrail_to_apply: "CustomGuardrail", + ) -> None: + """ + Extract text content from a response output item and create guardrail tasks. + + Override this method to customize text extraction logic. + """ + # Handle both GenericResponseOutputItem and dict + if isinstance(output_item, GenericResponseOutputItem): + content = output_item.content + elif isinstance(output_item, dict): + content = output_item.get("content", []) + else: + return + + if not content: + return + + verbose_proxy_logger.debug( + "OpenAI Responses API: Processing output item: %s", output_item + ) + + # Iterate through content items (list of OutputText objects) + for content_idx, content_item in enumerate(content): + # Handle both OutputText objects and dicts + if isinstance(content_item, OutputText): + text_content = content_item.text + elif isinstance(content_item, dict): + text_content = content_item.get("text") + else: + continue + + if text_content: + tasks.append(guardrail_to_apply.apply_guardrail(text=text_content)) + task_mappings.append((output_idx, int(content_idx))) + + async def _apply_guardrail_responses_to_output( + self, + response: "ResponsesAPIResponse", + responses: List[str], + task_mappings: List[Tuple[int, int]], + ) -> None: + """ + Apply guardrail responses back to output response. + + Override this method to customize how responses are applied. + """ + for task_idx, guardrail_response in enumerate(responses): + mapping = task_mappings[task_idx] + output_idx = cast(int, mapping[0]) + content_idx = cast(int, mapping[1]) + + output_item = response.output[output_idx] + + # Handle both GenericResponseOutputItem and dict + if isinstance(output_item, GenericResponseOutputItem): + content_item = output_item.content[content_idx] + if isinstance(content_item, OutputText): + content_item.text = guardrail_response + elif isinstance(content_item, dict): + content_item["text"] = guardrail_response + elif isinstance(output_item, dict): + content = output_item.get("content", []) + if content and content_idx < len(content): + if isinstance(content[content_idx], dict): + content[content_idx]["text"] = guardrail_response diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 1e949e434d3..f75213b0688 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -15,7 +15,7 @@ from litellm.types.llms.openai import * from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders - +from litellm.litellm_core_utils.core_helpers import process_response_headers from ..common_utils import OpenAIError if TYPE_CHECKING: @@ -123,8 +123,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): try: # Ensure required fields are present for ResponseReasoningItem item_data = dict(item) - if "id" not in item_data: - item_data["id"] = f"rs_{hash(str(item_data))}" if "summary" not in item_data: item_data["summary"] = ( item_data.get("reasoning_content", "")[:100] + "..." @@ -173,13 +171,19 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): raise OpenAIError( message=raw_response.text, status_code=raw_response.status_code ) + raw_response_headers = dict(raw_response.headers) + processed_headers = process_response_headers(raw_response_headers) try: - return ResponsesAPIResponse(**raw_response_json) + response = ResponsesAPIResponse(**raw_response_json) except Exception: verbose_logger.debug( f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct" ) - return ResponsesAPIResponse.model_construct(**raw_response_json) + response = ResponsesAPIResponse.model_construct(**raw_response_json) + + response._hidden_params["additional_headers"] = processed_headers + response._hidden_params["headers"] = raw_response_headers + return response def validate_environment( self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams] @@ -378,14 +382,21 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) -> ResponsesAPIResponse: """ Transform the get response API response into a ResponsesAPIResponse - """ + """ try: raw_response_json = raw_response.json() except Exception: raise OpenAIError( message=raw_response.text, status_code=raw_response.status_code ) - return ResponsesAPIResponse(**raw_response_json) + raw_response_headers = dict(raw_response.headers) + processed_headers = process_response_headers(raw_response_headers) + + response = ResponsesAPIResponse(**raw_response_json) + response._hidden_params["additional_headers"] = processed_headers + response._hidden_params["headers"] = raw_response_headers + + return response ######################################################### ########## LIST INPUT ITEMS TRANSFORMATION ############# @@ -462,4 +473,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): raise OpenAIError( message=raw_response.text, status_code=raw_response.status_code ) - return ResponsesAPIResponse(**raw_response_json) + raw_response_headers = dict(raw_response.headers) + processed_headers = process_response_headers(raw_response_headers) + + response = ResponsesAPIResponse(**raw_response_json) + response._hidden_params["additional_headers"] = processed_headers + response._hidden_params["headers"] = raw_response_headers + + return response diff --git a/litellm/llms/openai/speech/guardrail_translation/README.md b/litellm/llms/openai/speech/guardrail_translation/README.md new file mode 100644 index 00000000000..52e89ffa929 --- /dev/null +++ b/litellm/llms/openai/speech/guardrail_translation/README.md @@ -0,0 +1,178 @@ +# OpenAI Text-to-Speech Guardrail Translation Handler + +Handler for processing OpenAI's text-to-speech endpoint (`/v1/audio/speech`) with guardrails. + +## Overview + +This handler processes text-to-speech requests by: +1. Extracting the input text from the request +2. Applying guardrails to the input text +3. Updating the request with the guardrailed text +4. Returning the output unchanged (audio is binary, not text) + +## Data Format + +### Input Format + +```json +{ + "model": "tts-1", + "input": "The quick brown fox jumped over the lazy dog.", + "voice": "alloy", + "response_format": "mp3", + "speed": 1.0 +} +``` + +### Output Format + +The output is binary audio data (MP3, WAV, etc.), not text, so it cannot be guardrailed. + +## Usage + +The handler is automatically discovered and applied when guardrails are used with the text-to-speech endpoint. + +### Example: Using Guardrails with Text-to-Speech + +```bash +curl -X POST 'http://localhost:4000/v1/audio/speech' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer your-api-key' \ +-d '{ + "model": "tts-1", + "input": "The quick brown fox jumped over the lazy dog.", + "voice": "alloy", + "guardrails": ["content_moderation"] +}' \ +--output speech.mp3 +``` + +The guardrail will be applied to the input text before the text-to-speech conversion. + +### Example: PII Masking in TTS Input + +```bash +curl -X POST 'http://localhost:4000/v1/audio/speech' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer your-api-key' \ +-d '{ + "model": "tts-1", + "input": "Please call John Doe at john@example.com", + "voice": "nova", + "guardrails": ["mask_pii"] +}' \ +--output speech.mp3 +``` + +The audio will say: "Please call [NAME_REDACTED] at [EMAIL_REDACTED]" + +### Example: Content Filtering Before TTS + +```bash +curl -X POST 'http://localhost:4000/v1/audio/speech' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer your-api-key' \ +-d '{ + "model": "tts-1-hd", + "input": "This is the text that will be spoken", + "voice": "shimmer", + "guardrails": ["content_filter"] +}' \ +--output speech.mp3 +``` + +## Implementation Details + +### Input Processing + +- **Field**: `input` (string) +- **Processing**: Applies guardrail to input text +- **Result**: Updated input text in request + +### Output Processing + +- **Processing**: Not applicable (audio is binary data) +- **Result**: Response returned unchanged + +## Use Cases + +1. **PII Protection**: Remove personally identifiable information before converting to speech +2. **Content Filtering**: Remove inappropriate content before TTS conversion +3. **Compliance**: Ensure text meets requirements before voice synthesis +4. **Text Sanitization**: Clean up text before audio generation + +## Extension + +Override these methods to customize behavior: + +- `process_input_messages()`: Customize how input text is processed +- `process_output_response()`: Currently a no-op, but can be overridden if needed + +## Supported Call Types + +- `CallTypes.speech` - Synchronous text-to-speech +- `CallTypes.aspeech` - Asynchronous text-to-speech + +## Notes + +- Only the input text is processed by guardrails +- Output processing is a no-op since audio cannot be text-guardrailed +- Both sync and async call types use the same handler +- Works with all TTS models (tts-1, tts-1-hd, etc.) +- Works with all voice options + +## Common Patterns + +### Remove PII Before TTS + +```python +import litellm +from pathlib import Path + +speech_file_path = Path(__file__).parent / "speech.mp3" +response = litellm.speech( + model="tts-1", + voice="alloy", + input="Hi, this is John Doe calling from john@company.com", + guardrails=["mask_pii"], +) +response.stream_to_file(speech_file_path) +# Audio will have PII masked +``` + +### Content Moderation Before TTS + +```python +import litellm +from pathlib import Path + +speech_file_path = Path(__file__).parent / "speech.mp3" +response = litellm.speech( + model="tts-1-hd", + voice="nova", + input="Your text here", + guardrails=["content_moderation"], +) +response.stream_to_file(speech_file_path) +``` + +### Async TTS with Guardrails + +```python +import litellm +import asyncio +from pathlib import Path + +async def generate_speech(): + speech_file_path = Path(__file__).parent / "speech.mp3" + response = await litellm.aspeech( + model="tts-1", + voice="echo", + input="Text to convert to speech", + guardrails=["pii_mask"], + ) + response.stream_to_file(speech_file_path) + +asyncio.run(generate_speech()) +``` + diff --git a/litellm/llms/openai/speech/guardrail_translation/__init__.py b/litellm/llms/openai/speech/guardrail_translation/__init__.py new file mode 100644 index 00000000000..ef7d50f861a --- /dev/null +++ b/litellm/llms/openai/speech/guardrail_translation/__init__.py @@ -0,0 +1,13 @@ +"""OpenAI Text-to-Speech handler for Unified Guardrails.""" + +from litellm.llms.openai.speech.guardrail_translation.handler import ( + OpenAITextToSpeechHandler, +) +from litellm.types.utils import CallTypes + +guardrail_translation_mappings = { + CallTypes.speech: OpenAITextToSpeechHandler, + CallTypes.aspeech: OpenAITextToSpeechHandler, +} + +__all__ = ["guardrail_translation_mappings", "OpenAITextToSpeechHandler"] diff --git a/litellm/llms/openai/speech/guardrail_translation/handler.py b/litellm/llms/openai/speech/guardrail_translation/handler.py new file mode 100644 index 00000000000..aa049801d16 --- /dev/null +++ b/litellm/llms/openai/speech/guardrail_translation/handler.py @@ -0,0 +1,93 @@ +""" +OpenAI Text-to-Speech Handler for Unified Guardrails + +This module provides guardrail translation support for OpenAI's text-to-speech endpoint. +The handler processes the 'input' text parameter (output is audio, so no text to guardrail). +""" + +from typing import TYPE_CHECKING, Any + +from litellm._logging import verbose_proxy_logger +from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation + +if TYPE_CHECKING: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.llms.openai import HttpxBinaryResponseContent + + +class OpenAITextToSpeechHandler(BaseTranslation): + """ + Handler for processing OpenAI text-to-speech requests with guardrails. + + This class provides methods to: + 1. Process input text (pre-call hook) + + Note: Output processing is not applicable since the output is audio (binary), + not text. Only the input text is processed. + """ + + async def process_input_messages( + self, + data: dict, + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process input text by applying guardrails. + + Args: + data: Request data dictionary containing 'input' parameter + guardrail_to_apply: The guardrail instance to apply + + Returns: + Modified data with guardrails applied to input text + """ + input_text = data.get("input") + if input_text is None: + verbose_proxy_logger.debug( + "OpenAI Text-to-Speech: No input text found in request data" + ) + return data + + if isinstance(input_text, str): + guardrailed_input = await guardrail_to_apply.apply_guardrail( + text=input_text + ) + data["input"] = guardrailed_input + + verbose_proxy_logger.debug( + "OpenAI Text-to-Speech: Applied guardrail to input text. " + "Original length: %d, New length: %d", + len(input_text), + len(guardrailed_input), + ) + else: + verbose_proxy_logger.debug( + "OpenAI Text-to-Speech: Unexpected input type: %s. Expected string.", + type(input_text), + ) + + return data + + async def process_output_response( + self, + response: "HttpxBinaryResponseContent", + guardrail_to_apply: "CustomGuardrail", + ) -> Any: + """ + Process output - not applicable for text-to-speech. + + The output is audio (binary data), not text, so there's nothing to apply + guardrails to. This method returns the response unchanged. + + Args: + response: Binary audio response + guardrail_to_apply: The guardrail instance (unused) + + Returns: + Unmodified response (audio data doesn't need text guardrails) + """ + verbose_proxy_logger.debug( + "OpenAI Text-to-Speech: Output processing not applicable " + "(output is audio data, not text)" + ) + return response diff --git a/litellm/llms/openai/transcriptions/guardrail_translation/README.md b/litellm/llms/openai/transcriptions/guardrail_translation/README.md new file mode 100644 index 00000000000..08e5b6f85c5 --- /dev/null +++ b/litellm/llms/openai/transcriptions/guardrail_translation/README.md @@ -0,0 +1,159 @@ +# OpenAI Audio Transcription Guardrail Translation Handler + +Handler for processing OpenAI's audio transcription endpoint (`/v1/audio/transcriptions`) with guardrails. + +## Overview + +This handler processes audio transcription responses by: +1. Applying guardrails to the transcribed text output +2. Returning the input unchanged (since input is an audio file, not text) + +## Data Format + +### Input Format + +The input is an audio file, which cannot be guardrailed (it's binary data, not text). + +```json +{ + "model": "whisper-1", + "file": "