Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_scan_only_tool_results

.github/ci-coverage-allowlist.yml

@@ -0,0 +1,159 @@
+description: >-
+  Paths deliberately outside CI coverage, each with the reason it is exempt.
+  assert_ci_coverage.py fails when a test file or Dockerfile is neither invoked
+  by a job nor listed here, so every entry below is a decision on the record.
+
+test_paths:
+  - reason: >-
+      The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
+      from a pull request; it needs a live gateway and provider credentials no PR job holds
+    paths:
+      - tests/e2e
+  - reason: >-
+      The documentation and code-quality workflows execute four files in this directory by name as
+      scripts and pytest never collects the directory, so these six run nowhere; listed individually
+      so a seventh cannot inherit the exemption
+    paths:
+      - tests/documentation_tests/test_exception_types.py
+      - tests/documentation_tests/test_general_setting_keys.py
+      - tests/documentation_tests/test_optional_params.py
+      - tests/documentation_tests/test_readme_providers.py
+      - tests/documentation_tests/test_requests_lib_usage.py
+      - tests/documentation_tests/test_standard_logging_payload.py
+  - reason: >-
+      Sibling files here are executed by name from the code-quality workflow; this one is referenced
+      by no job
+    paths:
+      - tests/code_coverage_tests/test_aio_http_image_conversion.py
+  - reason: >-
+      A second mirror of the package tree living beside tests/test_litellm, which is the mirror the
+      repo convention names; only test_no_hardcoded_secrets.py is invoked, from the linting
+      workflow, and whether this directory should exist at all is unresolved
+    paths:
+      - tests/litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_headers.py
+      - tests/litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_transformation.py
+      - tests/litellm/integrations/helicone/test_helicone_gemini.py
+      - tests/litellm/litellm_core_utils/test_json_schema_validation.py
+      - tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py
+      - tests/litellm/llms/anthropic/test_anthropic_schema_filter.py
+      - tests/litellm/llms/azure/test_azure_embedding.py
+      - tests/litellm/llms/bedrock/embed/test_embedding.py
+      - tests/litellm/llms/bedrock/test_nova_imported_models.py
+      - tests/litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py
+      - tests/litellm/llms/gradient_ai/chat/test_gradient_ai_chat_transformation.py
+      - tests/litellm/llms/oci/chat/test_oci_chat_transformation.py
+      - tests/litellm/llms/openai_like/test_abliteration_provider.py
+      - tests/litellm/llms/openai_like/test_assemblyai_provider.py
+      - tests/litellm/llms/openai_like/test_empiriolabs_provider.py
+      - tests/litellm/llms/vertex_ai/agent_engine/test_transformation.py
+      - tests/litellm/llms/vertex_ai/gemini/test_transformation.py
+      - tests/litellm/llms/vertex_ai/text_to_speech/test_transformation.py
+      - tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
+      - tests/litellm/proxy/agent_endpoints/test_agent_rbac.py
+      - tests/litellm/proxy/common_utils/test_rbac_utils.py
+      - tests/litellm/proxy/management_endpoints/test_common_utils.py
+      - tests/litellm/proxy/management_endpoints/test_cost_estimate_endpoint.py
+      - tests/litellm/proxy/test_claude_code_marketplace.py
+      - tests/litellm/proxy/test_init_litellm_callbacks.py
+      - tests/litellm/proxy/test_prisma_engine_watchdog.py
+      - tests/litellm/proxy/vector_store_endpoints/test_vector_store_rbac.py
+      - tests/litellm/test_bedrock_extended_beta_models.py
+      - tests/litellm/test_bedrock_nemotron_super.py
+      - tests/litellm/test_proxy_auth.py
+      - tests/litellm/test_router_retry_backoff_headers.py
+      - tests/litellm/test_sambanova_model_metadata.py
+      - tests/litellm/test_stream_chunk_builder_images.py
+  - reason: >-
+      Legacy proxy suite superseded by the proxy shards; no job invokes it and whether it still
+      describes supported behaviour is unresolved
+    paths:
+      - tests/old_proxy_tests/tests/test_anthropic_context_caching.py
+      - tests/old_proxy_tests/tests/test_anthropic_sdk.py
+      - tests/old_proxy_tests/tests/test_async.py
+      - tests/old_proxy_tests/tests/test_gemini_context_caching.py
+      - tests/old_proxy_tests/tests/test_langchain_embedding.py
+      - tests/old_proxy_tests/tests/test_langchain_request.py
+      - tests/old_proxy_tests/tests/test_llamaindex.py
+      - tests/old_proxy_tests/tests/test_mistral_sdk.py
+      - tests/old_proxy_tests/tests/test_openai_embedding.py
+      - tests/old_proxy_tests/tests/test_openai_exception_request.py
+      - tests/old_proxy_tests/tests/test_openai_request.py
+      - tests/old_proxy_tests/tests/test_openai_request_with_traceparent.py
+      - tests/old_proxy_tests/tests/test_openai_simple_embedding.py
+      - tests/old_proxy_tests/tests/test_openai_tts_request.py
+      - tests/old_proxy_tests/tests/test_pass_through_langfuse.py
+      - tests/old_proxy_tests/tests/test_q.py
+      - tests/old_proxy_tests/tests/test_simple_traceparent_openai.py
+      - tests/old_proxy_tests/tests/test_vertex_sdk_forward_headers.py
+      - tests/old_proxy_tests/tests/test_vtx_embedding.py
+      - tests/old_proxy_tests/tests/test_vtx_sdk_embedding.py
+  - reason: >-
+      No job invokes this suite and its files mix pure transformation tests with ones driving live
+      vendor vector stores, so assigning them needs a per-file decision
+    paths:
+      - tests/vector_store_tests/rag/test_rag_bedrock.py
+      - tests/vector_store_tests/rag/test_rag_openai.py
+      - tests/vector_store_tests/rag/test_rag_s3_vectors.py
+      - tests/vector_store_tests/rag/test_rag_vertex_ai.py
+      - tests/vector_store_tests/test_azure_ai_vector_store.py
+      - tests/vector_store_tests/test_azure_vector_store.py
+      - tests/vector_store_tests/test_bedrock_vector_store.py
+      - tests/vector_store_tests/test_gemini_vector_store.py
+      - tests/vector_store_tests/test_milvus_vector_store.py
+      - tests/vector_store_tests/test_openai_vector_store.py
+      - tests/vector_store_tests/test_ragflow_vector_store.py
+      - tests/vector_store_tests/test_s3_vectors_vector_store.py
+      - tests/vector_store_tests/test_vertex_ai_search_api_vector_store.py
+      - tests/vector_store_tests/test_vertex_ai_vector_store.py
+  - reason: >-
+      Throughput and memory-growth measurements whose runtime and variance make them unsuitable for
+      a per-pull-request job
+    paths:
+      - tests/load_tests/test_datadog_load_test.py
+      - tests/load_tests/test_langsmith_load_test.py
+      - tests/load_tests/test_linear_memory_growth.py
+      - tests/load_tests/test_memory_usage.py
+      - tests/load_tests/test_otel_load_test.py
+      - tests/load_tests/test_vertex_embeddings_load_test.py
+      - tests/load_tests/test_vertex_load_tests.py
+  - reason: >-
+      Third-party integration tests that skip themselves without OCI configuration or sandbox
+      credentials, neither of which a pull request job holds
+    paths:
+      - tests/integration/sandbox/test_e2b_sandbox.py
+      - tests/integration/test_oci_integration.py
+      - tests/integration/test_oci_proxy_integration.py
+  - reason: >-
+      Two prompt-factory tests sitting at the top level of tests/ instead of under the
+      tests/test_litellm mirror the shards enumerate; they need moving rather than a shard entry
+    paths:
+      - tests/litellm_core_utils/test_anthropic_dedup_factory.py
+      - tests/litellm_core_utils/test_bedrock_converse_dedup_factory.py
+  - reason: >-
+      A unit test for the proxy-extras package that no job invokes, while the package's other tests
+      live under tests/proxy_migration_tests
+    paths:
+      - tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
+
+dockerfiles:
+  - reason: >-
+      The componentized images the microservices chart deploys are built by no job; wiring both into
+      the scan workflow costs a full image build each and is deferred to a change that prices the
+      whole set
+    paths:
+      - backend/Dockerfile
+      - gateway/Dockerfile
+  - reason: >-
+      The dashboard container is a static Next.js export served by nginx, and the dashboard build
+      and lint workflows already exercise that output, so building the image adds no signal about it
+    paths:
+      - ui/Dockerfile
+  - reason: >-
+      The Rust gateway ships as its own chart and package with a separate release pipeline, so its
+      image is not part of this repo's Python image set
+    paths:
+      - litellm-rust/crates/ai-gateway/Dockerfile
+  - reason: >-
+      An example image under cookbook/ that is documentation rather than a shipped artifact
+    paths:
+      - cookbook/litellm-ollama-docker-image/Dockerfile

.github/scripts/assert_ci_coverage.py

@@ -0,0 +1,262 @@
+from __future__ import annotations
+
+import pathlib
+import re
+import sys
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
+
+import yaml
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parents[2]
+WORKFLOW_DIR = REPO_ROOT / ".github" / "workflows"
+CIRCLECI_CONFIG = REPO_ROOT / ".circleci" / "config.yml"
+ALLOWLIST_FILE = REPO_ROOT / ".github" / "ci-coverage-allowlist.yml"
+TESTS_ROOT = REPO_ROOT / "tests"
+
+ALLOWLIST_KEYS = frozenset({"description", "test_paths", "dockerfiles"})
+PATH_FILTER_KEYS = frozenset({"paths", "paths-ignore"})
+TEST_PATH_KEYS = frozenset({"test-path", "test-paths"})
+DOCKERFILE_INPUT_KEYS = frozenset({"file", "dockerfile"})
+TEST_RUNNER_RE = re.compile(r"\bpytest\b|\bcircleci tests\b|\bhelm unittest\b|\bplaywright test\b|\bpython[0-9.]*\s")
+IMAGE_BUILD_RE = re.compile(r"\bdocker\s+(?:buildx\s+)?build\b")
+TEST_TOKEN_RE = re.compile(r"tests/[A-Za-z0-9_./*?-]+")
+DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
+COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
+GLOB_CHARS = frozenset("*?")
+
+
+@dataclass(frozen=True, slots=True)
+class AllowEntry:
+    paths: tuple[str, ...]
+    reason: str
+
+
+@dataclass(frozen=True, slots=True)
+class Allowlist:
+    test_paths: tuple[AllowEntry, ...]
+    dockerfiles: tuple[AllowEntry, ...]
+
+    def covers_test(self, relative_path: str) -> bool:
+        return any(_token_covers(path, relative_path) for entry in self.test_paths for path in entry.paths)
+
+    def covers_dockerfile(self, relative_path: str) -> bool:
+        return any(relative_path == path for entry in self.dockerfiles for path in entry.paths)
+
+
+@dataclass(frozen=True, slots=True)
+class Scalar:
+    key: str
+    value: str
+
+
+@dataclass(frozen=True, slots=True)
+class Finding:
+    subject: str
+    detail: str
+
+
+def _scalars(node: object, key: str) -> tuple[Scalar, ...]:
+    if isinstance(node, str):
+        return (Scalar(key=key, value=node),)
+    if isinstance(node, Mapping):
+        return tuple(
+            scalar
+            for child_key, value in node.items()
+            if child_key not in PATH_FILTER_KEYS
+            for scalar in _scalars(value, str(child_key))
+        )
+    if isinstance(node, Sequence):
+        return tuple(scalar for item in node for scalar in _scalars(item, key))
+    return ()
+
+
+def _config_files() -> tuple[pathlib.Path, ...]:
+    workflows = tuple(sorted(path for path in WORKFLOW_DIR.iterdir() if path.suffix in (".yml", ".yaml")))
+    circleci = (CIRCLECI_CONFIG,) if CIRCLECI_CONFIG.is_file() else ()
+    return workflows + circleci
+
+
+def _all_scalars() -> tuple[Scalar, ...]:
+    return tuple(
+        scalar
+        for path in _config_files()
+        for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name)
+    )
+
+
+def _uncommented(value: str) -> str:
+    return COMMENT_RE.sub("", value)
+
+
+def _invoked_test_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
+    return frozenset(
+        match.group(0).rstrip("/")
+        for scalar in scalars
+        if scalar.key in TEST_PATH_KEYS or TEST_RUNNER_RE.search(scalar.value)
+        for match in TEST_TOKEN_RE.finditer(_uncommented(scalar.value))
+    )
+
+
+def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
+    return frozenset(
+        match.group(0)
+        for scalar in scalars
+        if scalar.key in DOCKERFILE_INPUT_KEYS or IMAGE_BUILD_RE.search(scalar.value)
+        for match in DOCKERFILE_TOKEN_RE.finditer(_uncommented(scalar.value))
+    )
+
+
+def _glob_to_regex(token: str) -> re.Pattern[str]:
+    parts = re.split(r"(\*\*/|\*\*|\*|\?)", token)
+    translated = "".join(
+        {"**/": r"(?:.*/)?", "**": r".*", "*": r"[^/]*", "?": r"[^/]"}.get(part, re.escape(part)) for part in parts
+    )
+    return re.compile(rf"{translated}(?:/.*)?$")
+
+
+def _token_covers(token: str, relative_path: str) -> bool:
+    if GLOB_CHARS & set(token):
+        return _glob_to_regex(token).match(relative_path) is not None
+    return relative_path == token or relative_path.startswith(f"{token}/")
+
+
+def _test_files() -> tuple[str, ...]:
+    return tuple(
+        sorted(
+            path.relative_to(REPO_ROOT).as_posix()
+            for path in TESTS_ROOT.rglob("test_*.py")
+            if path.is_file() and "node_modules" not in path.parts
+        )
+    )
+
+
+def _dockerfiles() -> tuple[str, ...]:
+    return tuple(
+        sorted(
+            path.relative_to(REPO_ROOT).as_posix()
+            for path in REPO_ROOT.rglob("Dockerfile*")
+            if path.is_file()
+            and ".git" not in path.parts
+            and "node_modules" not in path.parts
+            and not path.name.endswith(".dockerignore")
+        )
+    )
+
+
+def _uncovered_tests(allowlist: Allowlist, tokens: frozenset[str]) -> tuple[Finding, ...]:
+    uncovered = tuple(
+        relative_path
+        for relative_path in _test_files()
+        if not any(_token_covers(token, relative_path) for token in tokens) and not allowlist.covers_test(relative_path)
+    )
+    directories = tuple(dict.fromkeys(path.rsplit("/", 1)[0] for path in uncovered))
+    return tuple(
+        Finding(
+            subject=directory,
+            detail=_describe(tuple(p for p in uncovered if p.rsplit("/", 1)[0] == directory)),
+        )
+        for directory in directories
+    )
+
+
+def _describe(paths: tuple[str, ...]) -> str:
+    names = ", ".join(path.rsplit("/", 1)[1] for path in paths[:3])
+    suffix = f", +{len(paths) - 3} more" if len(paths) > 3 else ""
+    return f"{len(paths)} test file(s) invoked by no job: {names}{suffix}"
+
+
+def _uncovered_dockerfiles(allowlist: Allowlist, tokens: frozenset[str]) -> tuple[Finding, ...]:
+    return tuple(
+        Finding(subject=relative_path, detail="built by no job")
+        for relative_path in _dockerfiles()
+        if relative_path not in tokens and not allowlist.covers_dockerfile(relative_path)
+    )
+
+
+def _parse_entry(item: object, section: str) -> AllowEntry:
+    if not isinstance(item, dict):
+        raise SystemExit(f"{ALLOWLIST_FILE.name}: '{section}' entries must be mappings")
+    paths = item.get("paths")
+    reason = item.get("reason")
+    if (
+        not isinstance(paths, list)
+        or not paths
+        or not all(isinstance(path, str) for path in paths)
+        or not isinstance(reason, str)
+        or not reason.strip()
+    ):
+        raise SystemExit(
+            f"{ALLOWLIST_FILE.name}: every '{section}' entry needs a non-empty 'paths' "
+            "list of strings and a non-empty 'reason'"
+        )
+    return AllowEntry(paths=tuple(paths), reason=reason)
+
+
+def _parse_entries(raw: object, section: str) -> tuple[AllowEntry, ...]:
+    if not isinstance(raw, list):
+        raise SystemExit(f"{ALLOWLIST_FILE.name}: '{section}' must be a list")
+    return tuple(_parse_entry(item, section) for item in raw)
+
+
+def _load_allowlist() -> Allowlist:
+    if not ALLOWLIST_FILE.is_file():
+        return Allowlist(test_paths=(), dockerfiles=())
+    raw = yaml.safe_load(ALLOWLIST_FILE.read_text(encoding="utf-8")) or {}
+    if not isinstance(raw, dict):
+        raise SystemExit(f"{ALLOWLIST_FILE.name}: top level must be a mapping")
+    unknown = sorted(str(key) for key in raw if key not in ALLOWLIST_KEYS)
+    if unknown:
+        raise SystemExit(
+            f"{ALLOWLIST_FILE.name}: unknown top-level key(s) {unknown}; expected only {sorted(ALLOWLIST_KEYS)}"
+        )
+    return Allowlist(
+        test_paths=_parse_entries(raw.get("test_paths", []), "test_paths"),
+        dockerfiles=_parse_entries(raw.get("dockerfiles", []), "dockerfiles"),
+    )
+
+
+def _write(message: str) -> None:
+    sys.stdout.write(f"{message}\n")
+
+
+def _report(title: str, findings: tuple[Finding, ...], remedy: str) -> None:
+    _write(f"ERROR: {title}")
+    for finding in findings:
+        _write(f"  - {finding.subject}: {finding.detail}")
+    _write("")
+    _write(remedy)
+    _write("")
+
+
+def main() -> int:
+    allowlist = _load_allowlist()
+    scalars = _all_scalars()
+
+    test_findings = _uncovered_tests(allowlist, _invoked_test_tokens(scalars))
+    dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
+
+    if test_findings:
+        _report(
+            "test files that no CI job invokes",
+            test_findings,
+            "Add each to a job's test path, or list it in .github/ci-coverage-allowlist.yml with a reason.",
+        )
+    if dockerfile_findings:
+        _report(
+            "Dockerfiles that no CI job builds",
+            dockerfile_findings,
+            "Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.",
+        )
+    if test_findings or dockerfile_findings:
+        return 1
+
+    _write(
+        f"OK: {len(_test_files())} test files and {len(_dockerfiles())} Dockerfiles are each "
+        "invoked by at least one job or carry an explicit allowlist entry."
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/ci-coverage.yml

@@ -0,0 +1,42 @@
+name: "CI Coverage"
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  assert-ci-coverage:
+    name: assert-ci-coverage
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Assert every test file and Dockerfile is invoked by a job
+        run: |
+          python -m pip install "pyyaml==6.0.3"
+          python .github/scripts/assert_ci_coverage.py

.github/workflows/create_daily_oss_agent_shin_branch.yml

@@ -13,35 +13,16 @@ jobs:
       contents: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
       - name: Create daily oss-agent-shin branch
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          # Configure Git user
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          # Generate branch name with MM_DD_YYYY format
           BRANCH_NAME="litellm_oss_agent_shin_$(date +'%m_%d_%Y')"
           echo "Creating branch: $BRANCH_NAME"
-
-          # Fetch all branches
-          git fetch --all
-
-          # Check if the branch already exists
-          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+          if gh api "repos/${{ github.repository }}/git/ref/heads/$BRANCH_NAME" --silent 2>/dev/null; then
             echo "Branch $BRANCH_NAME already exists. Skipping creation."
-          else
-            echo "Creating new branch: $BRANCH_NAME"
-            # Create the new branch from main
-            git checkout -b $BRANCH_NAME origin/main
-            # Push the new branch
-            git push origin $BRANCH_NAME
-            echo "Successfully created and pushed branch: $BRANCH_NAME"
+            exit 0
           fi
+          MAIN_SHA=$(gh api "repos/${{ github.repository }}/git/ref/heads/main" --jq '.object.sha')
+          gh api "repos/${{ github.repository }}/git/refs" -f ref="refs/heads/$BRANCH_NAME" -f sha="$MAIN_SHA" --silent
+          echo "Successfully created branch: $BRANCH_NAME at $MAIN_SHA"

.github/workflows/create_daily_staging_branch.yml

@@ -13,38 +13,19 @@ jobs:
       contents: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
       - name: Create daily staging branch
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          # Configure Git user
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          # Generate branch name with MM_DD_YYYY format
           BRANCH_NAME="litellm_oss_staging_$(date +'%m_%d_%Y')"
           echo "Creating branch: $BRANCH_NAME"
-
-          # Fetch all branches
-          git fetch --all
-
-          # Check if the branch already exists
-          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+          if gh api "repos/${{ github.repository }}/git/ref/heads/$BRANCH_NAME" --silent 2>/dev/null; then
             echo "Branch $BRANCH_NAME already exists. Skipping creation."
-          else
-            echo "Creating new branch: $BRANCH_NAME"
-            # Create the new branch from main
-            git checkout -b $BRANCH_NAME origin/main
-            # Push the new branch
-            git push origin $BRANCH_NAME
-            echo "Successfully created and pushed branch: $BRANCH_NAME"
+            exit 0
           fi
+          MAIN_SHA=$(gh api "repos/${{ github.repository }}/git/ref/heads/main" --jq '.object.sha')
+          gh api "repos/${{ github.repository }}/git/refs" -f ref="refs/heads/$BRANCH_NAME" -f sha="$MAIN_SHA" --silent
+          echo "Successfully created branch: $BRANCH_NAME at $MAIN_SHA"
 
   create-internal-dev-branch:
     if: github.repository == 'BerriAI/litellm'
@@ -53,35 +34,16 @@ jobs:
       contents: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
       - name: Create internal dev branch
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          # Configure Git user
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-
-          # Generate branch name with MM_DD_YYYY format
           BRANCH_NAME="litellm_internal_dev_$(date +'%m_%d_%Y')"
           echo "Creating branch: $BRANCH_NAME"
-
-          # Fetch all branches
-          git fetch --all
-
-          # Check if the branch already exists
-          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+          if gh api "repos/${{ github.repository }}/git/ref/heads/$BRANCH_NAME" --silent 2>/dev/null; then
             echo "Branch $BRANCH_NAME already exists. Skipping creation."
-          else
-            echo "Creating new branch: $BRANCH_NAME"
-            # Create the new branch from main
-            git checkout -b $BRANCH_NAME origin/main
-            # Push the new branch
-            git push origin $BRANCH_NAME
-            echo "Successfully created and pushed branch: $BRANCH_NAME"
+            exit 0
           fi
+          MAIN_SHA=$(gh api "repos/${{ github.repository }}/git/ref/heads/main" --jq '.object.sha')
+          gh api "repos/${{ github.repository }}/git/refs" -f ref="refs/heads/$BRANCH_NAME" -f sha="$MAIN_SHA" --silent
+          echo "Successfully created branch: $BRANCH_NAME at $MAIN_SHA"

.github/workflows/helm_unit_test.yml

@@ -23,21 +23,28 @@ jobs:
         with:
           version: "3.11.1"
 
+      - name: Download and verify Helm Unit Test Plugin
+        run: |
+          curl -fsSLo "$RUNNER_TEMP/helm-unittest.tgz" https://github.com/helm-unittest/helm-unittest/releases/download/v0.8.2/helm-unittest-linux-amd64-0.8.2.tgz
+          echo "56ab3091e6fa52a7c92ee951def9bed957f295d9ce98483aed404e748d7b3a94  $RUNNER_TEMP/helm-unittest.tgz" | sha256sum -c -
+
       - name: Install Helm Unit Test Plugin
         run: |
-          helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4
-      - name: Verify Helm Unit Test Plugin integrity
-        run: |
-          EXPECTED_SHA="e251ba198448629678ff2168e1a469249d998155"
           PLUGIN_DIR="$(helm env HELM_PLUGINS)/helm-unittest"
-          ACTUAL_SHA="$(git -C "$PLUGIN_DIR" rev-parse HEAD)"
-          if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then
-            echo "::error::Helm unittest plugin checksum mismatch! Expected $EXPECTED_SHA but got $ACTUAL_SHA"
-            exit 1
-          fi
-          echo "Helm unittest plugin integrity verified: $ACTUAL_SHA"
+          mkdir -p "$PLUGIN_DIR"
+          tar -xzf "$RUNNER_TEMP/helm-unittest.tgz" -C "$PLUGIN_DIR"
+          helm plugin list
 
       - name: Run unit tests
         run: |
-          helm unittest -f 'tests/*.yaml' helm/litellm-helm
-          helm unittest -f 'tests/*.yaml' helm/litellm
+          for chart in helm/litellm-helm helm/litellm; do
+            declared="$(grep -h '^suite:' "$chart"/tests/*.yaml | wc -l | tr -d '[:space:]')"
+            output="$(mktemp)"
+            helm unittest -f 'tests/*.yaml' "$chart" | tee "$output"
+            executed="$(sed -n 's/^Test Suites:.*[[:space:]]\([0-9][0-9]*\) total$/\1/p' "$output")"
+            if [ "$declared" != "$executed" ]; then
+              echo "::error::$chart declares $declared test suites but helm-unittest ran $executed. Suites are being skipped silently, so their assertions never execute."
+              exit 1
+            fi
+            echo "$chart: all $declared declared test suites ran"
+          done

.github/workflows/image-scan.yml

@@ -8,10 +8,12 @@ on:
       - litellm_oss_branch
       - "litellm_**"
     paths:
+      - Dockerfile
       - docker/Dockerfile.non_root
       - migrations/Dockerfile
       - migrations/run.py
-      - tests/proxy_migration_tests/test_offline_image_migration.py
+      - litellm-proxy-extras/**
+      - tests/proxy_migration_tests/**
       - uv.lock
       - ui/litellm-dashboard/package-lock.json
       - .github/workflows/image-scan.yml
@@ -86,6 +88,35 @@ jobs:
             --fail-on high \
             --output table
 
+  runtime-image:
+    name: runtime-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build runtime image
+        run: docker build -f Dockerfile -t litellm-runtime-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify offline migration as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+
   migrations-image:
     name: migrations-image
     runs-on: ubuntu-latest

.github/workflows/publish-basedpyright-base-counts.yml

@@ -0,0 +1,70 @@
+name: Publish basedpyright base counts
+
+# Every commit on litellm_internal_staging is some branch's future merge-base.
+# Publishing its per-rule basedpyright counts as an artifact lets
+# scripts/type_check_gate.py download them in seconds instead of paying a
+# 60-110s second basedpyright pass on every fresh worktree or moved merge-base.
+# No concurrency group on purpose: runs must never cancel each other, because
+# every sha's artifact matters (any of them can become a merge-base).
+
+on:
+  push:
+    branches:
+      - litellm_internal_staging
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: "Ref to compute and publish base counts for"
+        required: false
+        default: litellm_internal_staging
+
+permissions:
+  contents: read
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          ref: ${{ inputs.ref || github.sha }}
+          clean: true
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Install dependencies
+        run: |
+          uv sync --frozen --group proxy-dev --group e2e-dev
+
+      # Mirrors test-linting.yml's lint job: basedpyright resolves Prisma's
+      # generated client only after `prisma generate`, and the published counts
+      # must match what that job would measure for the same tree.
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Emit basedpyright counts for HEAD
+        run: |
+          uv run --no-sync python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts"
+          counts_file=$(ls "$RUNNER_TEMP"/basedpyright-counts/basedpyright-counts-*.json)
+          echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV"
+
+      - name: Upload counts artifact
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: ${{ env.COUNTS_ARTIFACT_NAME }}
+          path: ${{ runner.temp }}/basedpyright-counts/
+          if-no-files-found: error

.github/workflows/test-linting.yml

@@ -15,6 +15,12 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    # actions: read lets scripts/type_check_gate.py download the base-counts
+    # artifact published by publish-basedpyright-base-counts.yml instead of
+    # re-running basedpyright over the merge-base tree.
+    permissions:
+      contents: read
+      actions: read
 
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
@@ -23,10 +29,21 @@ jobs:
         # Any-discipline) would otherwise blame on this branch.
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
+          fetch-depth: 1
           clean: true
           persist-credentials: false
 
+      - name: Fetch gate base (merge-base with target branch)
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          MERGE_BASE=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          test -n "$MERGE_BASE"
+          git fetch --no-tags --depth=1 origin "$MERGE_BASE"
+          echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
+
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
@@ -60,10 +77,8 @@ jobs:
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Check ruff format
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          git diff --name-only --diff-filter=ACMR "$BASE_SHA"...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
+          git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
           if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then
             echo "No changed litellm Python files to check with ruff format."
             exit 0
@@ -86,16 +101,12 @@ jobs:
           cd ..
 
       - name: Check strict-rule budget (delta vs base)
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          uv run --no-sync python scripts/ruff_strict_gate.py --base "$BASE_SHA"
+          uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA"
 
       - name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base)
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          uv run --no-sync python scripts/type_discipline_gate.py --base "$BASE_SHA"
+          uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
 
       - name: Print OpenAI version
         run: |
@@ -103,15 +114,13 @@ jobs:
 
       - name: Check basedpyright budget (delta vs base)
         env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          GH_TOKEN: ${{ github.token }}
         run: |
-          uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA"
+          uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA"
 
       - name: Check tests/e2e basedpyright (zero errors)
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          if git diff --name-only --diff-filter=ACMRD "$BASE_SHA"...HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
+          if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
             uv run --no-sync basedpyright tests/e2e
           else
             echo "No changed tests/e2e Python files; skipping."
@@ -140,9 +149,15 @@ jobs:
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
+      - name: Fetch ratchet base
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          git fetch --no-tags --depth=1 origin "$BASE_SHA"
+
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
@@ -163,7 +178,7 @@ jobs:
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
       - name: Set up Python
@@ -178,13 +193,14 @@ jobs:
 
       - name: Run secret scan test
         run: |
-          uv run --frozen --with 'pytest==9.0.2' pytest tests/litellm/test_no_hardcoded_secrets.py -v
+          uv run --no-project --with 'pytest==9.0.2' pytest tests/litellm/test_no_hardcoded_secrets.py -v
 
       - name: Run ggshield secret scan
         env:
           GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
         run: |
           if [ -n "$GITGUARDIAN_API_KEY" ]; then
+            git fetch --no-tags --unshallow origin
             uv tool run --from 'ggshield==1.48.0' ggshield secret scan repo .
           else
             echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"

.github/workflows/test-litellm-ui-lint.yml

@@ -22,12 +22,13 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
       - name: Collect changed files
         id: changed
         env:
+          GH_TOKEN: ${{ github.token }}
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
@@ -37,7 +38,9 @@ jobs:
           # landed since, so a PR that touches no UI file still gets linted
           # against hundreds of other people's files. Diff the PR head against its
           # own merge base instead, which is exactly what this PR changed.
-          merge_base=$(git merge-base "$BASE_SHA" "$HEAD_SHA")
+          merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          test -n "$merge_base"
+          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
           : > "$RUNNER_TEMP/prettier_files.txt"
           : > "$RUNNER_TEMP/eslint_files.txt"
           while IFS= read -r f; do

.github/workflows/test-litellm-ui-unit.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
       - name: Setup Node.js
@@ -45,11 +45,24 @@ jobs:
       - name: Run UI unit tests (Vitest)
         env:
           CI: "true"
+          GH_TOKEN: ${{ github.token }}
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
           if [ -n "$BASE_SHA" ]; then
-            echo "Pull request: running only tests related to changes since $BASE_SHA"
-            npm run test -- --run --changed "$BASE_SHA" --passWithNoTests \
+            merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+            test -n "$merge_base"
+            git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
+            changed_files=()
+            while IFS= read -r f; do
+              changed_files+=("$f")
+            done < <(git diff --name-only --relative "$merge_base" "$HEAD_SHA" -- .)
+            if [ ${#changed_files[@]} -eq 0 ]; then
+              echo "No UI files changed in this PR; skipping unit tests."
+              exit 0
+            fi
+            echo "Pull request: running tests related to ${#changed_files[@]} changed UI files"
+            npm run test -- related "${changed_files[@]}" --run --passWithNoTests \
               --pool forks --poolOptions.forks.maxForks=14
           else
             echo "Push to $GITHUB_REF_NAME: running the full suite"

.github/workflows/test-unit-misc.yml

@@ -40,7 +40,11 @@ jobs:
         tests/test_litellm/interactions
         tests/test_litellm/ocr
         tests/test_litellm/passthrough
+        tests/test_litellm/rag
+        tests/test_litellm/realtime_api
+        tests/test_litellm/rerank_api
         tests/test_litellm/sandbox
+        tests/test_litellm/test_router
         tests/test_litellm/vector_stores
         tests/test_litellm/videos
         tests/test_litellm/test_*.py

.github/workflows/test-unit-proxy-endpoints.yml

@@ -29,7 +29,9 @@ jobs:
     uses: ./.github/workflows/_test-unit-base.yml
     with:
       test-path: >-
+        tests/test_litellm/proxy/analytics_endpoints
         tests/test_litellm/proxy/management_endpoints
+        tests/test_litellm/proxy/memory
         tests/test_litellm/proxy/guardrails
         tests/test_litellm/proxy/management_helpers
         tests/test_litellm/proxy/anthropic_endpoints

.github/workflows/test-unit-proxy-infra.yml

@@ -33,6 +33,8 @@ jobs:
         tests/test_litellm/proxy/_experimental
         tests/test_litellm/proxy/experimental
         tests/test_litellm/proxy/common_utils
+        tests/test_litellm/proxy/enterprise_billing
+        tests/test_litellm/proxy/types_utils
         tests/test_litellm/proxy/logging_endpoints
         tests/test_litellm/proxy/test_*.py
       workers: 2

CLAUDE.md

@@ -41,6 +41,8 @@ Python max line length is 120, not 88
 
 When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered limits so the ceilings ratchet down instead of leaving stale headroom. It measures the working tree, so it must contain exactly the fixes you're committing
 
+`make pre-commit` always saves its complete output to a per-worktree log file and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice, and re-run only after the working tree actually changed
+
 If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
 
 If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason

Dockerfile

@@ -134,7 +134,8 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete && \
     chmod -R a+rX /opt/prisma && \
     test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
-    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js
+    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 EXPOSE 4000/tcp
 

Makefile

@@ -99,7 +99,10 @@ install-test-deps: install-proxy-dev
 	$(UV_RUN) prisma generate --schema litellm/proxy/schema.prisma
 
 install-helm-unittest:
-	helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
+	@helm plugin list | grep -qE '^unittest[[:space:]]+0\.8\.2([[:space:]]|$$)' || { \
+		helm plugin uninstall unittest >/dev/null 2>&1 || true; \
+		helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.8.2; \
+	}
 
 # Install git hooks that enforce Conventional Commits and Conventional Branches.
 # Opt-in: not chained into install-dev.

backend/Dockerfile

@@ -59,9 +59,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra semantic-router \
         --python python3
 
-RUN mkdir -p /home/nonroot && \
-    HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
-    chown -R nonroot:nonroot /home/nonroot/.cache
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
 
 RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh
 
@@ -83,13 +83,16 @@ ENV HOME=/home/nonroot \
     PATH="/app/.venv/bin:${PATH}" \
     PYTHONPATH="/app" \
     PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1
+    PYTHONUNBUFFERED=1 \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
 COPY --from=builder --chown=nonroot:nonroot /app /app
-COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
+COPY --from=builder /opt/prisma /opt/prisma
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
-    find /app/.venv -type d -path "*/tornado/test" -delete
+    find /app/.venv -type d -path "*/tornado/test" -delete && \
+    chmod -R a+rX /opt/prisma && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 USER nonroot
 

backend/routes/allowlist.py

@@ -82,6 +82,9 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/user_agent",
     "/usage/",
     "/daily/",
+    # Deployment-wide gateway request counts. Scoped to the analytics read rather
+    # than all of /gateway/, which stays free for data-plane routes.
+    "/gateway/daily/",
     # CloudZero cost-export admin (init / settings / export / dry-run / delete)
     "/cloudzero/",
     # Caching admin

basedpyright-code-budget.json

@@ -1,9 +1,9 @@
 {
   "reportAny": {
-    "limit": 29809
+    "limit": 29204
   },
   "reportArgumentType": {
-    "limit": 2645
+    "limit": 2635
   },
   "reportAssignmentType": {
     "limit": 329
@@ -21,10 +21,10 @@
     "limit": 215
   },
   "reportDuplicateImport": {
-    "limit": 24
+    "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 9473
+    "limit": 9227
   },
   "reportFunctionMemberAccess": {
     "limit": 7
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5855
+    "limit": 5850
   },
   "reportMissingTypeArgument": {
-    "limit": 15849
+    "limit": 15833
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -72,7 +72,7 @@
     "limit": 0
   },
   "reportOptionalMemberAccess": {
-    "limit": 1079
+    "limit": 1078
   },
   "reportOptionalOperand": {
     "limit": 0
@@ -90,7 +90,7 @@
     "limit": 8
   },
   "reportReturnType": {
-    "limit": 219
+    "limit": 218
   },
   "reportTypedDictNotRequiredAccess": {
     "limit": 27
@@ -99,22 +99,22 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 45262
+    "limit": 45242
   },
   "reportUnknownLambdaType": {
     "limit": 113
   },
   "reportUnknownMemberType": {
-    "limit": 40452
+    "limit": 40340
   },
   "reportUnknownParameterType": {
-    "limit": 20309
+    "limit": 20293
   },
   "reportUnknownVariableType": {
-    "limit": 31978
+    "limit": 31796
   },
   "reportUnnecessaryCast": {
-    "limit": 124
+    "limit": 122
   },
   "reportUnnecessaryComparison": {
     "limit": 703
@@ -123,10 +123,10 @@
     "limit": 5
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 866
+    "limit": 865
   },
   "reportUntypedBaseClass": {
-    "limit": 165
+    "limit": 72
   },
   "reportUntypedFunctionDecorator": {
     "limit": 33
@@ -138,9 +138,9 @@
     "limit": 139
   },
   "reportUnusedImport": {
-    "limit": 588
+    "limit": 555
   },
   "reportUnusedVariable": {
-    "limit": 147
+    "limit": 146
   }
 }

docker/Dockerfile.database

@@ -133,7 +133,8 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete && \
     chmod -R a+rX /opt/prisma && \
     test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
-    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js
+    test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 EXPOSE 4000/tcp
 

docker/Dockerfile.non_root

@@ -185,7 +185,8 @@ RUN mkdir -p /nonexistent /app/.cache /var/lib/litellm/assets /var/lib/litellm/u
     chmod -R a+rX /opt/prisma && \
     test -x /opt/prisma/binaries/node_modules/.bin/prisma && \
     test -f /opt/prisma/binaries/node_modules/prisma/build/index.js && \
-    ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null 2>&1
+    ls /opt/prisma/binaries/node_modules/@prisma/engines/query-engine-* >/dev/null 2>&1 && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 USER 65534
 

docker/build_from_pip/Dockerfile.build_from_pip

@@ -47,7 +47,13 @@ RUN uv venv --python python && \
       "prisma==0.11.0" \
       "openai==2.24.0"
 
-RUN prisma generate --schema=./schema.prisma
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma && \
+    chmod -R a+rX /opt/prisma && \
+    python -c "import sys; from prisma.client import BINARY_PATHS; bad = sorted(p for group in BINARY_PATHS.model_dump().values() for p in group.values() if not p.startswith('/opt/prisma/')); sys.exit('prisma engines baked outside /opt/prisma: %r' % bad) if bad else None"
+
+ENV PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
 EXPOSE 4000/tcp
 

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -831,7 +831,7 @@ async def project_info(
             )
 
         # Check if user has access to this project (admin or team member)
-        is_admin = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
+        is_admin = user_api_key_has_admin_view(user_api_key_dict)
         is_team_member = False
 
         if project.team_id and user_api_key_dict.user_id:
@@ -886,7 +886,7 @@ async def list_projects(
             )
 
         # If proxy admin, get all projects
-        if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
+        if user_api_key_has_admin_view(user_api_key_dict):
             projects: Sequence[
                 prisma_models.LiteLLM_ProjectTable
             ] = await prisma_client.db.litellm_projecttable.find_many(

gateway/Dockerfile

@@ -61,9 +61,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra bedrock-realtime \
         --python python3
 
-RUN mkdir -p /home/nonroot && \
-    HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
-    chown -R nonroot:nonroot /home/nonroot/.cache
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma
 
 RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component_entrypoint.sh
 
@@ -85,13 +85,16 @@ ENV HOME=/home/nonroot \
     PATH="/app/.venv/bin:${PATH}" \
     PYTHONPATH="/app" \
     PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1
+    PYTHONUNBUFFERED=1 \
+    PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
 COPY --from=builder --chown=nonroot:nonroot /app /app
-COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
+COPY --from=builder /opt/prisma /opt/prisma
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
-    find /app/.venv -type d -path "*/tornado/test" -delete
+    find /app/.venv -type d -path "*/tornado/test" -delete && \
+    chmod -R a+rX /opt/prisma && \
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
 
 USER nonroot
 

litellm-proxy-extras/litellm_proxy_extras/migrations/20260803000000_add_daily_gateway_requests/migration.sql

@@ -0,0 +1,15 @@
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_DailyGatewayRequests" (
+    "date" TEXT NOT NULL,
+    "category" TEXT NOT NULL,
+    "route" TEXT NOT NULL,
+    "successful_requests" BIGINT NOT NULL DEFAULT 0,
+    "failed_requests" BIGINT NOT NULL DEFAULT 0,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LiteLLM_DailyGatewayRequests_pkey" PRIMARY KEY ("date","category","route")
+);
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_DailyGatewayRequests_date_idx" ON "LiteLLM_DailyGatewayRequests"("date");

litellm-proxy-extras/litellm_proxy_extras/migrations/20260805000000_add_autorouter_session_rollup/migration.sql

@@ -0,0 +1,31 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterSession" (
+    "api_key" TEXT NOT NULL,
+    "session_id" TEXT NOT NULL,
+    "router_name" TEXT NOT NULL,
+    "router_type" TEXT NOT NULL,
+    "first_turn_at" TIMESTAMP(3) NOT NULL,
+    "last_turn_at" TIMESTAMP(3) NOT NULL,
+    "last_model" TEXT NOT NULL,
+    "models" JSONB NOT NULL DEFAULT '{}',
+    "turns" INTEGER NOT NULL DEFAULT 0,
+    "unordered_turns" INTEGER NOT NULL DEFAULT 0,
+    "covered_turns" INTEGER NOT NULL DEFAULT 0,
+    "cache_hits" INTEGER NOT NULL DEFAULT 0,
+    "same_model_turns" INTEGER NOT NULL DEFAULT 0,
+    "same_model_hits" INTEGER NOT NULL DEFAULT 0,
+    "first_visit_turns" INTEGER NOT NULL DEFAULT 0,
+    "first_visit_hits" INTEGER NOT NULL DEFAULT 0,
+    "return_turns" INTEGER NOT NULL DEFAULT 0,
+    "return_hits" INTEGER NOT NULL DEFAULT 0,
+    "return_expired_misses" INTEGER NOT NULL DEFAULT 0,
+    "return_within_ttl_misses" INTEGER NOT NULL DEFAULT 0,
+    "ttl_5m_turns" INTEGER NOT NULL DEFAULT 0,
+    "ttl_1h_turns" INTEGER NOT NULL DEFAULT 0,
+    "total_tokens" BIGINT NOT NULL DEFAULT 0,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+
+    CONSTRAINT "LiteLLM_AutoRouterSession_pkey" PRIMARY KEY ("api_key", "session_id", "router_name")
+);
+
+CREATE INDEX IF NOT EXISTS "idx_autorouter_session_last_turn" ON "LiteLLM_AutoRouterSession"("last_turn_at");

litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py

@@ -121,9 +121,13 @@ def heal_incomplete_nodeenv_cache() -> bool:
     Prisma invocation reinstalls it instead of failing on a missing binary.
     """
     cache_dir = nodeenv_cache_dir()
-    if cache_dir is None or not cache_dir.is_dir():
+    if cache_dir is None:
         return False
-    if node_binary_path(cache_dir).exists():
+    try:
+        if not cache_dir.is_dir() or node_binary_path(cache_dir).exists():
+            return False
+    except OSError as e:
+        logger.warning("Could not inspect the Node toolchain at %s: %s", cache_dir, e)
         return False
     logger.warning(
         "Node toolchain at %s has no %s, so a previous install was interrupted. "

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1118,6 +1118,26 @@ model LiteLLM_DailyToolSpend {
   @@id([date, tool_name])
 }
 
+// Gateway request counts recorded at the ASGI edge by
+// BillableRequestMetricsMiddleware. This is the source of truth for SGR
+// (successful gateway requests): it counts what the proxy actually answered,
+// independent of whether the request reached litellm's logging callbacks.
+// The key carries no deployment or caller dimension. Every part of it is
+// chosen by the proxy and drawn from a closed set, so the table is bounded by
+// (days x categories x routes) rather than by anything a caller can vary.
+model LiteLLM_DailyGatewayRequests {
+  date                String
+  category            String
+  route               String
+  successful_requests BigInt   @default(0)
+  failed_requests     BigInt   @default(0)
+  created_at          DateTime @default(now())
+  updated_at          DateTime @updatedAt
+
+  @@id([date, category, route])
+  @@index([date])
+}
+
 // Prompt table for storing prompt configurations
 model LiteLLM_PromptTable {
   id String @id @default(uuid())
@@ -1393,6 +1413,37 @@ model LiteLLM_AdaptiveRouterSession {
   @@index([last_activity_at], map: "idx_adaptive_router_session_activity")
 }
 
+model LiteLLM_AutoRouterSession {
+  api_key               String
+  session_id            String
+  router_name           String
+  router_type           String
+  first_turn_at         DateTime
+  last_turn_at          DateTime
+  last_model            String
+  models                Json     @default("{}")
+  turns                 Int      @default(0)
+  unordered_turns       Int      @default(0)
+  covered_turns         Int      @default(0)
+  cache_hits            Int      @default(0)
+  same_model_turns      Int      @default(0)
+  same_model_hits       Int      @default(0)
+  first_visit_turns     Int      @default(0)
+  first_visit_hits      Int      @default(0)
+  return_turns          Int      @default(0)
+  return_hits           Int      @default(0)
+  return_expired_misses Int      @default(0)
+  return_within_ttl_misses  Int      @default(0)
+  ttl_5m_turns          Int      @default(0)
+  ttl_1h_turns          Int      @default(0)
+  total_tokens          BigInt   @default(0)
+  spend                 Float    @default(0)
+  saved_spend           Float    @default(0)
+
+  @@id([api_key, session_id, router_name])
+  @@index([last_turn_at], map: "idx_autorouter_session_last_turn")
+}
+
 // ---------------------------------------------------------------------------
 // Workflow Run Tracking
 //

litellm/__init__.py

@@ -704,9 +704,8 @@ def is_openai_finetune_model(key: str) -> bool:
     return key.startswith("ft:") and not key.count(":") > 1
 
 
-def add_known_models(model_cost_map: Optional[Dict] = None):
-    _map: Final = model_cost_map if model_cost_map is not None else model_cost
-    for key, value in _map.items():
+def _populate_provider_model_sets(model_cost_map: Dict) -> None:
+    for key, value in model_cost_map.items():
         if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
             open_ai_chat_completion_models.add(key)
         elif value.get("litellm_provider") == "text-completion-openai":
@@ -949,7 +948,16 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
             bedrock_mantle_models.add(key)
 
 
-add_known_models()
+def add_known_models(model_cost_map: Optional[Dict] = None):
+    """Fold `model_cost_map` (defaults to `litellm.model_cost`) into the per-provider model sets,
+    then refresh `models_by_provider` from those sets so the additions reach wildcard expansion.
+    The refresh updates the dict in place, so references captured before a reload stay live.
+    """
+    _populate_provider_model_sets(model_cost_map if model_cost_map is not None else model_cost)
+    models_by_provider.update(_build_models_by_provider())
+
+
+_populate_provider_model_sets(model_cost)
 # known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary
 
 # this is maintained for Exception Mapping
@@ -1071,112 +1079,116 @@ model_list_set = set(model_list)
 # provider_list is lazy-loaded via __getattr__ to avoid importing LlmProviders at import time
 
 
-models_by_provider: dict = {
-    "openai": open_ai_chat_completion_models | open_ai_text_completion_models,
-    "text-completion-openai": open_ai_text_completion_models,
-    "cohere": cohere_models | cohere_chat_models,
-    "cohere_chat": cohere_chat_models,
-    "anthropic": anthropic_models,
-    "replicate": replicate_models,
-    "huggingface": huggingface_models,
-    "together_ai": together_ai_models,
-    "baseten": baseten_models,
-    "openrouter": openrouter_models,
-    "vercel_ai_gateway": vercel_ai_gateway_models,
-    "datarobot": datarobot_models,
-    "vertex_ai": vertex_chat_models
-    | vertex_text_models
-    | vertex_anthropic_models
-    | vertex_vision_models
-    | vertex_language_models
-    | vertex_deepseek_models
-    | vertex_minimax_models
-    | vertex_moonshot_models
-    | vertex_zai_models,
-    "ai21": ai21_models,
-    "bedrock": bedrock_models | bedrock_converse_models,
-    "petals": petals_models,
-    "ollama": ollama_models,
-    "ollama_chat": ollama_models,
-    "deepinfra": deepinfra_models,
-    "perplexity": perplexity_models,
-    "maritalk": maritalk_models,
-    "watsonx": watsonx_models,
-    "gemini": gemini_models,
-    "fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models,
-    "aleph_alpha": aleph_alpha_models,
-    "text-completion-codestral": text_completion_codestral_models,
-    "text-completion-inception": text_completion_inception_models,
-    "xai": xai_models,
-    "zai": zai_models,
-    "fal_ai": fal_ai_models,
-    "deepseek": deepseek_models,
-    "tencent": tencent_models,
-    "runwayml": runwayml_models,
-    "mistral": mistral_chat_models,
-    "azure_ai": azure_ai_models,
-    "voyage": voyage_models,
-    "infinity": infinity_models,
-    "databricks": databricks_models,
-    "cloudflare": cloudflare_models,
-    "codestral": codestral_models,
-    "nlp_cloud": nlp_cloud_models,
-    "friendliai": friendliai_models,
-    "palm": palm_models,
-    "groq": groq_models,
-    "azure": azure_models | azure_text_models,
-    "azure_anthropic": azure_anthropic_models,
-    "azure_text": azure_text_models,
-    "anyscale": anyscale_models,
-    "cerebras": cerebras_models,
-    "galadriel": galadriel_models,
-    "nvidia_nim": nvidia_nim_models,
-    "nvidia_riva": nvidia_riva_models,
-    "soniox": soniox_models,
-    "sambanova": sambanova_models | sambanova_embedding_models,
-    "novita": novita_models,
-    "nebius": nebius_models | nebius_embedding_models,
-    "aiml": aiml_models,
-    "assemblyai": assemblyai_models,
-    "jina_ai": jina_ai_models,
-    "snowflake": snowflake_models,
-    "gradient_ai": gradient_ai_models,
-    "meta_llama": llama_models,
-    "nscale": nscale_models,
-    "featherless_ai": featherless_ai_models,
-    "deepgram": deepgram_models,
-    "elevenlabs": elevenlabs_models,
-    "heroku": heroku_models,
-    "dashscope": dashscope_models,
-    "modelscope": modelscope_models,
-    "moonshot": moonshot_models,
-    "publicai": publicai_models,
-    "darkbloom": darkbloom_models,
-    "v0": v0_models,
-    "morph": morph_models,
-    "lambda_ai": lambda_ai_models,
-    "inception": inception_models,
-    "hyperbolic": hyperbolic_models,
-    "black_forest_labs": black_forest_labs_models,
-    "recraft": recraft_models,
-    "cometapi": cometapi_models,
-    "oci": oci_models,
-    "volcengine": volcengine_models,
-    "wandb": wandb_models,
-    "ovhcloud": ovhcloud_models | ovhcloud_embedding_models,
-    "lemonade": lemonade_models,
-    "clarifai": clarifai_models,
-    "amazon_nova": amazon_nova_models,
-    "stability": stability_models,
-    "github_copilot": github_copilot_models,
-    "chatgpt": chatgpt_models,
-    "minimax": minimax_models,
-    "aws_polly": aws_polly_models,
-    "gigachat": gigachat_models,
-    "llamagate": llamagate_models,
-    "reducto": reducto_models,
-    "bedrock_mantle": bedrock_mantle_models,
-}
+def _build_models_by_provider() -> dict:
+    return {
+        "openai": open_ai_chat_completion_models | open_ai_text_completion_models,
+        "text-completion-openai": open_ai_text_completion_models,
+        "cohere": cohere_models | cohere_chat_models,
+        "cohere_chat": cohere_chat_models,
+        "anthropic": anthropic_models,
+        "replicate": replicate_models,
+        "huggingface": huggingface_models,
+        "together_ai": together_ai_models,
+        "baseten": baseten_models,
+        "openrouter": openrouter_models,
+        "vercel_ai_gateway": vercel_ai_gateway_models,
+        "datarobot": datarobot_models,
+        "vertex_ai": vertex_chat_models
+        | vertex_text_models
+        | vertex_anthropic_models
+        | vertex_vision_models
+        | vertex_language_models
+        | vertex_deepseek_models
+        | vertex_minimax_models
+        | vertex_moonshot_models
+        | vertex_zai_models,
+        "ai21": ai21_models,
+        "bedrock": bedrock_models | bedrock_converse_models,
+        "petals": petals_models,
+        "ollama": ollama_models,
+        "ollama_chat": ollama_models,
+        "deepinfra": deepinfra_models,
+        "perplexity": perplexity_models,
+        "maritalk": maritalk_models,
+        "watsonx": watsonx_models,
+        "gemini": gemini_models,
+        "fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models,
+        "aleph_alpha": aleph_alpha_models,
+        "text-completion-codestral": text_completion_codestral_models,
+        "text-completion-inception": text_completion_inception_models,
+        "xai": xai_models,
+        "zai": zai_models,
+        "fal_ai": fal_ai_models,
+        "deepseek": deepseek_models,
+        "tencent": tencent_models,
+        "runwayml": runwayml_models,
+        "mistral": mistral_chat_models,
+        "azure_ai": azure_ai_models,
+        "voyage": voyage_models,
+        "infinity": infinity_models,
+        "databricks": databricks_models,
+        "cloudflare": cloudflare_models,
+        "codestral": codestral_models,
+        "nlp_cloud": nlp_cloud_models,
+        "friendliai": friendliai_models,
+        "palm": palm_models,
+        "groq": groq_models,
+        "azure": azure_models | azure_text_models,
+        "azure_anthropic": azure_anthropic_models,
+        "azure_text": azure_text_models,
+        "anyscale": anyscale_models,
+        "cerebras": cerebras_models,
+        "galadriel": galadriel_models,
+        "nvidia_nim": nvidia_nim_models,
+        "nvidia_riva": nvidia_riva_models,
+        "soniox": soniox_models,
+        "sambanova": sambanova_models | sambanova_embedding_models,
+        "novita": novita_models,
+        "nebius": nebius_models | nebius_embedding_models,
+        "aiml": aiml_models,
+        "assemblyai": assemblyai_models,
+        "jina_ai": jina_ai_models,
+        "snowflake": snowflake_models,
+        "gradient_ai": gradient_ai_models,
+        "meta_llama": llama_models,
+        "nscale": nscale_models,
+        "featherless_ai": featherless_ai_models,
+        "deepgram": deepgram_models,
+        "elevenlabs": elevenlabs_models,
+        "heroku": heroku_models,
+        "dashscope": dashscope_models,
+        "modelscope": modelscope_models,
+        "moonshot": moonshot_models,
+        "publicai": publicai_models,
+        "darkbloom": darkbloom_models,
+        "v0": v0_models,
+        "morph": morph_models,
+        "lambda_ai": lambda_ai_models,
+        "inception": inception_models,
+        "hyperbolic": hyperbolic_models,
+        "black_forest_labs": black_forest_labs_models,
+        "recraft": recraft_models,
+        "cometapi": cometapi_models,
+        "oci": oci_models,
+        "volcengine": volcengine_models,
+        "wandb": wandb_models,
+        "ovhcloud": ovhcloud_models | ovhcloud_embedding_models,
+        "lemonade": lemonade_models,
+        "clarifai": clarifai_models,
+        "amazon_nova": amazon_nova_models,
+        "stability": stability_models,
+        "github_copilot": github_copilot_models,
+        "chatgpt": chatgpt_models,
+        "minimax": minimax_models,
+        "aws_polly": aws_polly_models,
+        "gigachat": gigachat_models,
+        "llamagate": llamagate_models,
+        "reducto": reducto_models,
+        "bedrock_mantle": bedrock_mantle_models,
+    }
+
+
+models_by_provider: dict = _build_models_by_provider()
 
 # mapping for those models which have larger equivalents
 longer_context_model_fallback_dict: dict = {
@@ -2150,9 +2162,9 @@ def __getattr__(name: str) -> Any:
 
     # Lazy load encoding from main.py to avoid heavy tiktoken import
     if name == "encoding":
-        from ._lazy_imports import _get_litellm_globals
+        from ._lazy_imports import get_litellm_globals
 
-        _globals = _get_litellm_globals()
+        _globals = get_litellm_globals()
         # Check if already cached
         if "encoding" not in _globals:
             from .main import encoding as _encoding
@@ -2162,9 +2174,9 @@ def __getattr__(name: str) -> Any:
 
     # Lazy load bedrock_tool_name_mappings instance
     if name == "bedrock_tool_name_mappings":
-        from ._lazy_imports import _get_litellm_globals
+        from ._lazy_imports import get_litellm_globals
 
-        _globals = _get_litellm_globals()
+        _globals = get_litellm_globals()
         # Check if already cached
         if "bedrock_tool_name_mappings" not in _globals:
             from .llms.bedrock.chat.invoke_handler import (
@@ -2176,9 +2188,9 @@ def __getattr__(name: str) -> Any:
 
     # Lazy load AzureOpenAIError exception class
     if name == "AzureOpenAIError":
-        from ._lazy_imports import _get_litellm_globals
+        from ._lazy_imports import get_litellm_globals
 
-        _globals = _get_litellm_globals()
+        _globals = get_litellm_globals()
         # Check if already cached
         if "AzureOpenAIError" not in _globals:
             from .llms.azure.common_utils import AzureOpenAIError as _AzureOpenAIError
@@ -2188,9 +2200,9 @@ def __getattr__(name: str) -> Any:
 
     # Lazy load openaiOSeriesConfig instance
     if name == "openaiOSeriesConfig":
-        from ._lazy_imports import _get_litellm_globals
+        from ._lazy_imports import get_litellm_globals
 
-        _globals = _get_litellm_globals()
+        _globals = get_litellm_globals()
         if "openaiOSeriesConfig" not in _globals:
             # Import the config class and instantiate it
             config_class = __getattr__("OpenAIOSeriesConfig")
@@ -2206,9 +2218,9 @@ def __getattr__(name: str) -> Any:
         "nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig",
     }
     if name in _config_instances:
-        from ._lazy_imports import _get_litellm_globals
+        from ._lazy_imports import get_litellm_globals
 
-        _globals = _get_litellm_globals()
+        _globals = get_litellm_globals()
         if name not in _globals:
             # Import the config class and instantiate it
             config_class = __getattr__(_config_instances[name])
@@ -2221,9 +2233,9 @@ def __getattr__(name: str) -> Any:
 
     # Lazy load provider_list
     if name == "provider_list":
-        from ._lazy_imports import _get_litellm_globals
+        from ._lazy_imports import get_litellm_globals
 
-        _globals = _get_litellm_globals()
+        _globals = get_litellm_globals()
         # Check if already cached
         if "provider_list" not in _globals:
             # LlmProviders is eagerly imported above, so we can import it directly
@@ -2234,9 +2246,9 @@ def __getattr__(name: str) -> Any:
 
     # Lazy load priority_reservation_settings instance
     if name == "priority_reservation_settings":
-        from ._lazy_imports import _get_litellm_globals
+        from ._lazy_imports import get_litellm_globals
 
-        _globals = _get_litellm_globals()
+        _globals = get_litellm_globals()
         # Check if already cached
         if "priority_reservation_settings" not in _globals:
             # Import the class and instantiate it
@@ -2246,9 +2258,9 @@ def __getattr__(name: str) -> Any:
 
     # Lazy load logging_callback_manager instance
     if name == "logging_callback_manager":
-        from ._lazy_imports import _get_litellm_globals
+        from ._lazy_imports import get_litellm_globals
 
-        _globals = _get_litellm_globals()
+        _globals = get_litellm_globals()
         # Check if already cached
         if "logging_callback_manager" not in _globals:
             # Import the class and instantiate it
@@ -2258,9 +2270,9 @@ def __getattr__(name: str) -> Any:
 
     # Lazy load _service_logger module
     if name == "_service_logger":
-        from ._lazy_imports import _get_litellm_globals
+        from ._lazy_imports import get_litellm_globals
 
-        _globals = _get_litellm_globals()
+        _globals = get_litellm_globals()
         # Check if already cached
         if "_service_logger" not in _globals:
             # Import the module lazily

litellm/_lazy_imports.py

@@ -54,7 +54,7 @@ from ._lazy_imports_registry import (
 )
 
 
-def _get_litellm_globals() -> dict:
+def get_litellm_globals() -> dict:
     """
     Get the globals dictionary of the litellm module.
 
@@ -233,7 +233,7 @@ def _generic_lazy_import(name: str, import_map: dict[str, tuple[str, str]], cate
         raise AttributeError(f"{category} lazy import: unknown attribute {name!r}")
 
     # Step 2: Get the cache (where we store imported things)
-    _globals: Final = _get_litellm_globals()
+    _globals: Final = get_litellm_globals()
 
     # Step 3: If we've already imported it, just return the cached version
     if name in _globals:
@@ -332,7 +332,7 @@ def _lazy_import_utils_module(name: str) -> Any:
     Handler for utils module lazy imports.
 
     This uses a custom implementation because utils module needs to use
-    _get_utils_globals() instead of _get_litellm_globals() for caching.
+    _get_utils_globals() instead of get_litellm_globals() for caching.
     """
     # Check if this attribute exists in our map
     if name not in _UTILS_MODULE_IMPORT_MAP:
@@ -379,7 +379,7 @@ def _lazy_import_llm_client_cache(name: str) -> Any:
     - "in_memory_llm_clients_cache" is a singleton instance of that class
     So we need custom logic to handle both cases.
     """
-    _globals: Final = _get_litellm_globals()
+    _globals: Final = get_litellm_globals()
 
     # If already cached, return it
     if name in _globals:
@@ -412,7 +412,7 @@ def _lazy_import_http_handlers(name: str) -> Any:
     - They need configuration (timeout, etc.) from the module globals
     - They use factory functions instead of direct instantiation
     """
-    _globals: Final = _get_litellm_globals()
+    _globals: Final = get_litellm_globals()
 
     if name == "module_level_aclient":
         # Create an async HTTP client using the factory function

litellm/_lazy_imports_registry.py

@@ -1461,32 +1461,30 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
 
 # Export all name tuples and import maps for use in _lazy_imports.py
 __all__ = [
-    # Name tuples
-    "COST_CALCULATOR_NAMES",
-    "LITELLM_LOGGING_NAMES",
-    "UTILS_NAMES",
-    "TOKEN_COUNTER_NAMES",
-    "LLM_CLIENT_CACHE_NAMES",
     "BEDROCK_TYPES_NAMES",
-    "TYPES_UTILS_NAMES",
     "CACHING_NAMES",
-    "HTTP_HANDLER_NAMES",
+    "COST_CALCULATOR_NAMES",
     "DOTPROMPT_NAMES",
+    "HTTP_HANDLER_NAMES",
+    "LITELLM_LOGGING_NAMES",
+    "LLM_CLIENT_CACHE_NAMES",
     "LLM_CONFIG_NAMES",
-    "TYPES_NAMES",
     "LLM_PROVIDER_LOGIC_NAMES",
+    "TOKEN_COUNTER_NAMES",
+    "TYPES_NAMES",
+    "TYPES_UTILS_NAMES",
     "UTILS_MODULE_NAMES",
-    # Import maps
-    "_UTILS_IMPORT_MAP",
-    "_COST_CALCULATOR_IMPORT_MAP",
-    "_TYPES_UTILS_IMPORT_MAP",
-    "_TOKEN_COUNTER_IMPORT_MAP",
+    "UTILS_NAMES",
     "_BEDROCK_TYPES_IMPORT_MAP",
     "_CACHING_IMPORT_MAP",
-    "_LITELLM_LOGGING_IMPORT_MAP",
+    "_COST_CALCULATOR_IMPORT_MAP",
     "_DOTPROMPT_IMPORT_MAP",
-    "_TYPES_IMPORT_MAP",
+    "_LITELLM_LOGGING_IMPORT_MAP",
     "_LLM_CONFIGS_IMPORT_MAP",
     "_LLM_PROVIDER_LOGIC_IMPORT_MAP",
+    "_TOKEN_COUNTER_IMPORT_MAP",
+    "_TYPES_IMPORT_MAP",
+    "_TYPES_UTILS_IMPORT_MAP",
+    "_UTILS_IMPORT_MAP",
     "_UTILS_MODULE_IMPORT_MAP",
 ]

litellm/_logging.py

@@ -273,8 +273,42 @@ def _suppress_loggers():
     apscheduler_scheduler_logger.setLevel(logging.WARNING)
 
 
+_REDACTED_THIRD_PARTY_LOGGERS: Final[tuple[str, ...]] = (
+    "apscheduler.executors.default",
+    "apscheduler.scheduler",
+    "asyncio",
+    "backoff",
+    "httpx",
+    "uvicorn.error",
+)
+
+
+def _redact_third_party_loggers() -> None:
+    """Extend secret redaction to records litellm does not emit directly.
+
+    litellm's own loggers are covered by the filter on their shared handler, but a
+    litellm value can also reach a log record through a dependency that logs on its
+    own logger. Those records never pass through a litellm handler.
+
+    The filter is attached to each emitting logger rather than to the root logger or
+    to root's handlers. `Logger.handle` applies the emitting logger's filters before
+    any handler runs, so redaction happens once, at the earliest point in the
+    record's life, and covers every downstream handler regardless of who owns it.
+    The alternatives do not hold: `callHandlers` consults ancestors for handlers but
+    never for filters, so a filter on the root logger never sees these records at
+    all, and a filter on a root handler only covers that one handler, leaving
+    handlers registered earlier or on the emitting logger itself untouched.
+
+    Each name is the exact logger a dependency emits on; a parent name would not
+    cover its children, for the same reason the root logger does not.
+    """
+    for name in _REDACTED_THIRD_PARTY_LOGGERS:
+        logging.getLogger(name).addFilter(_secret_filter)
+
+
 # Call the suppression function
 _suppress_loggers()
+_redact_third_party_loggers()
 
 ALL_LOGGERS: Final = [
     logging.getLogger(),

litellm/_redis.py

@@ -395,9 +395,7 @@ def _get_redis_client_logic(**env_overrides):
     if _sentinel_password is not None:
         redis_kwargs["sentinel_password"] = _sentinel_password
 
-    _service_name: Final[str | None] = redis_kwargs.get("service_name", None) or get_secret(
-        "REDIS_SERVICE_NAME"
-    )
+    _service_name: Final[str | None] = redis_kwargs.get("service_name", None) or get_secret("REDIS_SERVICE_NAME")
 
     if _service_name is not None:
         redis_kwargs["service_name"] = _service_name

litellm/_service_logger.py

@@ -1,6 +1,6 @@
 import asyncio
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 import litellm
 from litellm._logging import verbose_logger
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
 
     from litellm.proxy._types import UserAPIKeyAuth
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
     OTELClass = OpenTelemetry
 else:
     Span = Any

litellm/a2a_protocol/__init__.py

@@ -55,19 +55,15 @@ from litellm.a2a_protocol.main import (
 from litellm.types.agents import LiteLLMSendMessageResponse
 
 __all__ = [
-    # Client
-    "A2AClient",
-    # Functions
-    "asend_message",
-    "send_message",
-    "asend_message_streaming",
-    "aget_agent_card",
-    "create_a2a_client",
-    # Response types
-    "LiteLLMSendMessageResponse",
-    # Exceptions
-    "A2AError",
-    "A2AConnectionError",
     "A2AAgentCardError",
+    "A2AClient",
+    "A2AConnectionError",
+    "A2AError",
     "A2ALocalhostURLError",
+    "LiteLLMSendMessageResponse",
+    "aget_agent_card",
+    "asend_message",
+    "asend_message_streaming",
+    "create_a2a_client",
+    "send_message",
 ]

litellm/a2a_protocol/litellm_completion_bridge/handler.py

@@ -10,7 +10,7 @@ A2A Streaming Events (in order):
 4. Status update (kind: "status-update") - Final status "completed" with final=true
 """
 
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Mapping
 from typing import Any, Final
 
 import litellm
@@ -21,6 +21,8 @@ from litellm.a2a_protocol.litellm_completion_bridge.transformation import (
 )
 from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager
 from litellm.interactions.agents.utils import merge_agent_headers
+from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
+from litellm.types.utils import ModelResponse
 
 # litellm_params key carrying the authenticated principal (hashed virtual key) so
 # A2A provider configs can scope provider-side state (e.g. LangFlow session memory)
@@ -45,47 +47,14 @@ class A2ACompletionBridgeHandler:
     """
 
     @staticmethod
-    async def handle_non_streaming(
-        request_id: str,
+    def _build_completion_params(
         params: dict[str, Any],
-        litellm_params: dict[str, Any],
-        api_base: str | None = None,
-        agent_extra_headers: dict[str, str] | None = None,
+        litellm_params: Mapping[str, Any],
+        api_base: str | None,
+        agent_extra_headers: Mapping[str, str] | None,
         *,
-        _skip_a2a_provider_routing: bool = False,
-    ) -> dict[str, Any]:
-        """
-        Handle non-streaming A2A request via litellm.acompletion.
-
-        Args:
-            request_id: A2A JSON-RPC request ID
-            params: A2A MessageSendParams containing the message
-            litellm_params: Agent's litellm_params (custom_llm_provider, model, etc.)
-            api_base: API base URL from agent_card_params
-            agent_extra_headers: Per-request headers (from x-a2a-{agent}-* rewrite and
-                admin extra_headers) to forward on the upstream HTTP call.
-
-        Returns:
-            A2A SendMessageResponse dict
-        """
-        custom_llm_provider = litellm_params.get("custom_llm_provider")
-        if not _skip_a2a_provider_routing:
-            a2a_provider_config: Final = A2AProviderConfigManager.get_provider_config(
-                custom_llm_provider=custom_llm_provider,
-                model=litellm_params.get("model"),
-            )
-
-            if a2a_provider_config is not None:
-                verbose_logger.info("A2A: Using provider config for %s", custom_llm_provider)
-
-                return await a2a_provider_config.handle_non_streaming(
-                    request_id=request_id,
-                    params=params,
-                    api_base=api_base,
-                    litellm_params=litellm_params,
-                    agent_extra_headers=agent_extra_headers,
-                )
-
+        stream: bool,
+    ) -> Mapping[str, Any]:
         # Extract message from params
         message: Final = params.get("message", {})
 
@@ -93,7 +62,7 @@ class A2ACompletionBridgeHandler:
         openai_messages: Final = A2ACompletionBridgeTransformation.a2a_message_to_openai_messages(message)
 
         # Get completion params
-        custom_llm_provider = litellm_params.get("custom_llm_provider")
+        custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
         model: Final = litellm_params.get("model", "agent")
 
         # Build full model string if provider specified
@@ -103,14 +72,17 @@ class A2ACompletionBridgeHandler:
         else:
             full_model = model
 
-        verbose_logger.info("A2A completion bridge: model=%s, api_base=%s", full_model, api_base)
+        if stream:
+            verbose_logger.info("A2A completion bridge streaming: model=%s, api_base=%s", full_model, api_base)
+        else:
+            verbose_logger.info("A2A completion bridge: model=%s, api_base=%s", full_model, api_base)
 
         # Build completion params dict
         completion_params: Final[dict[str, Any]] = {
             "model": full_model,
             "messages": openai_messages,
             "api_base": api_base,
-            "stream": False,
+            "stream": stream,
         }
         # Add litellm_params (contains api_key, client_id, client_secret, tenant_id, etc.)
         litellm_params_to_add: Final = {
@@ -134,8 +106,64 @@ class A2ACompletionBridgeHandler:
                 static_headers=completion_params.get("extra_headers"),
             )
 
+        return completion_params
+
+    @staticmethod
+    async def _acompletion(completion_params: Mapping[str, Any]) -> ModelResponse | CustomStreamWrapper:
+        return await litellm.acompletion(**completion_params)
+
+    @staticmethod
+    async def handle_non_streaming(
+        request_id: str,
+        params: dict[str, Any],
+        litellm_params: dict[str, Any],
+        api_base: str | None = None,
+        agent_extra_headers: dict[str, str] | None = None,
+        *,
+        _skip_a2a_provider_routing: bool = False,
+    ) -> dict[str, object]:
+        """
+        Handle non-streaming A2A request via litellm.acompletion.
+
+        Args:
+            request_id: A2A JSON-RPC request ID
+            params: A2A MessageSendParams containing the message
+            litellm_params: Agent's litellm_params (custom_llm_provider, model, etc.)
+            api_base: API base URL from agent_card_params
+            agent_extra_headers: Per-request headers (from x-a2a-{agent}-* rewrite and
+                admin extra_headers) to forward on the upstream HTTP call.
+
+        Returns:
+            A2A SendMessageResponse dict
+        """
+        custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
+        if not _skip_a2a_provider_routing:
+            a2a_provider_config: Final = A2AProviderConfigManager.get_provider_config(
+                custom_llm_provider=custom_llm_provider,
+                model=litellm_params.get("model"),
+            )
+
+            if a2a_provider_config is not None:
+                verbose_logger.info("A2A: Using provider config for %s", custom_llm_provider)
+
+                return await a2a_provider_config.handle_non_streaming(
+                    request_id=request_id,
+                    params=params,
+                    api_base=api_base,
+                    litellm_params=litellm_params,
+                    agent_extra_headers=agent_extra_headers,
+                )
+
+        completion_params: Final = A2ACompletionBridgeHandler._build_completion_params(
+            params=params,
+            litellm_params=litellm_params,
+            api_base=api_base,
+            agent_extra_headers=agent_extra_headers,
+            stream=False,
+        )
+
         # Call litellm.acompletion
-        response: Final = await litellm.acompletion(**completion_params)
+        response: Final = await A2ACompletionBridgeHandler._acompletion(completion_params)
 
         # Transform response to A2A format
         a2a_response: Final = A2ACompletionBridgeTransformation.openai_response_to_a2a_response(
@@ -156,7 +184,7 @@ class A2ACompletionBridgeHandler:
         agent_extra_headers: dict[str, str] | None = None,
         *,
         _skip_a2a_provider_routing: bool = False,
-    ) -> AsyncIterator[dict[str, Any]]:
+    ) -> AsyncIterator[dict[str, object]]:
         """
         Handle streaming A2A request via litellm.acompletion with stream=True.
 
@@ -177,7 +205,7 @@ class A2ACompletionBridgeHandler:
         Yields:
             A2A streaming response events
         """
-        custom_llm_provider = litellm_params.get("custom_llm_provider")
+        custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
         if not _skip_a2a_provider_routing:
             a2a_provider_config: Final = A2AProviderConfigManager.get_provider_config(
                 custom_llm_provider=custom_llm_provider,
@@ -198,60 +226,20 @@ class A2ACompletionBridgeHandler:
 
                 return
 
-        # Extract message from params
-        message: Final = params.get("message", {})
-
         # Create streaming context
         ctx: Final = A2AStreamingContext(
             request_id=request_id,
-            input_message=message,
+            input_message=params.get("message", {}),
         )
 
-        # Transform A2A message to OpenAI format
-        openai_messages: Final = A2ACompletionBridgeTransformation.a2a_message_to_openai_messages(message)
-
-        # Get completion params
-        custom_llm_provider = litellm_params.get("custom_llm_provider")
-        model: Final = litellm_params.get("model", "agent")
-
-        # Build full model string if provider specified
-        # Skip prepending if model already starts with the provider prefix
-        if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"):
-            full_model = f"{custom_llm_provider}/{model}"
-        else:
-            full_model = model
-
-        verbose_logger.info("A2A completion bridge streaming: model=%s, api_base=%s", full_model, api_base)
-
-        # Build completion params dict
-        completion_params: Final[dict[str, Any]] = {
-            "model": full_model,
-            "messages": openai_messages,
-            "api_base": api_base,
-            "stream": True,
-        }
-        # Add litellm_params (contains api_key, client_id, client_secret, tenant_id, etc.)
-        litellm_params_to_add: Final = {
-            k: v
-            for k, v in litellm_params.items()
-            if k not in ("model", "custom_llm_provider") and k not in _AGENT_ONLY_PARAMS
-        }
-        completion_params.update(litellm_params_to_add)
-        # Apply forward metadata AFTER the litellm_params merge so the helper
-        # sees any agent-owner-configured ``extra_body.metadata`` and can keep
-        # those keys authoritative over the client-supplied A2A metadata.
-        A2ACompletionBridgeTransformation.apply_forward_metadata_to_completion_params(
-            completion_params=completion_params,
-            a2a_message=message,
+        completion_params: Final = A2ACompletionBridgeHandler._build_completion_params(
             params=params,
+            litellm_params=litellm_params,
+            api_base=api_base,
+            agent_extra_headers=agent_extra_headers,
+            stream=True,
         )
 
-        if agent_extra_headers:
-            completion_params["extra_headers"] = merge_agent_headers(
-                dynamic_headers=agent_extra_headers,
-                static_headers=completion_params.get("extra_headers"),
-            )
-
         # 1. Emit initial task event (kind: "task", status: "submitted")
         task_event: Final = A2ACompletionBridgeTransformation.create_task_event(ctx)
         yield task_event
@@ -266,7 +254,7 @@ class A2ACompletionBridgeHandler:
         yield working_event
 
         # Call litellm.acompletion with streaming
-        response: Final = await litellm.acompletion(**completion_params)
+        response: Final = await A2ACompletionBridgeHandler._acompletion(completion_params)
 
         # 3. Accumulate content and emit artifact update
         accumulated_text = ""
@@ -312,7 +300,7 @@ async def handle_a2a_completion(
     litellm_params: dict[str, Any],
     api_base: str | None = None,
     agent_extra_headers: dict[str, str] | None = None,
-) -> dict[str, Any]:
+) -> dict[str, object]:
     """Convenience function for non-streaming A2A completion."""
     return await A2ACompletionBridgeHandler.handle_non_streaming(
         request_id=request_id,
@@ -329,7 +317,7 @@ async def handle_a2a_completion_streaming(
     litellm_params: dict[str, Any],
     api_base: str | None = None,
     agent_extra_headers: dict[str, str] | None = None,
-) -> AsyncIterator[dict[str, Any]]:
+) -> AsyncIterator[dict[str, object]]:
     """Convenience function for streaming A2A completion."""
     async for chunk in A2ACompletionBridgeHandler.handle_streaming(
         request_id=request_id,

litellm/assistants/utils.py

@@ -8,8 +8,8 @@ from ..types.llms.openai import *
 
 def get_optional_params_add_message(
     role: str | None,
-    content: str | List[MessageContentTextObject | MessageContentImageFileObject | MessageContentImageURLObject] | None,
-    attachments: List[Attachment] | None,
+    content: str | list[MessageContentTextObject | MessageContentImageFileObject | MessageContentImageURLObject] | None,
+    attachments: list[Attachment] | None,
     metadata: dict | None,
     custom_llm_provider: str,
     **kwargs,
@@ -57,7 +57,7 @@ def get_optional_params_add_message(
         optional_params = litellm.AzureOpenAIAssistantsAPIConfig().map_openai_params_create_message_params(
             non_default_params=non_default_params, optional_params=optional_params
         )
-    for k in passed_params.keys():
+    for k in passed_params:
         if k not in default_params:
             optional_params[k] = passed_params[k]
     return optional_params
@@ -128,7 +128,7 @@ def get_optional_params_image_gen(
         if n is not None:
             optional_params["sampleCount"] = int(n)
 
-    for k in passed_params.keys():
+    for k in passed_params:
         if k not in default_params:
             optional_params[k] = passed_params[k]
     return optional_params

litellm/caching/base_cache.py

@@ -9,12 +9,12 @@ Has 4 methods:
 """
 
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/caching/disk_cache.py

@@ -1,12 +1,12 @@
 import json
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from .base_cache import BaseCache
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/caching/dual_cache.py

@@ -13,7 +13,7 @@ import time
 import traceback
 from concurrent.futures import ThreadPoolExecutor
 from threading import Lock
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 if TYPE_CHECKING:
     from litellm.types.caching import RedisPipelineIncrementOperation
@@ -29,7 +29,7 @@ from .redis_cache import RedisCache
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/caching/evicted_client_closer.py

@@ -0,0 +1,276 @@
+"""
+Deferred close of HTTP/SDK clients that the LLM client cache has evicted.
+
+Eviction only drops the cache's reference to a client. Every OpenAI/Azure SDK
+client is a reference cycle (each resource namespace holds the client back), so
+an evicted client and its pooled TCP connections survive until a generational
+collection runs, which under load is thousands of requests later.
+
+Closing at eviction time is not an option: a request that was handed the client
+just before it was evicted is still using it, and closing it underneath that
+request raises ``RuntimeError: Cannot send a request, as the client has been
+closed.``
+
+So an evicted client is closed once two conditions hold. A grace window must
+have passed since its eviction, which covers a request that holds the client
+but is momentarily not on the wire, and the client must report no connection in
+flight. The second condition is what keeps the first honest: a request may run
+for ``litellm.request_timeout`` seconds, 6000 by default, and a streaming
+response is bounded only by how long the upstream keeps sending, so no deadline
+on its own can promise that a request has finished.
+
+Only clients litellm itself created are closed; a client the caller supplied is
+left alone because litellm does not own its lifecycle.
+
+A client that closes synchronously is closed from wherever the cache is next
+used. One whose close is a coroutine needs the event loop it was evicted on, so
+it waits for a call from that loop rather than having work scheduled onto a loop
+it does not belong to. Queued clients are therefore bucketed by what it takes to
+close them, and each bucket is ordered by deadline, so a reap walks the entries
+that are due rather than the whole queue.
+
+The queue holds its clients weakly, so waiting out a grace window never keeps
+alive anything the collector would have reclaimed first.
+"""
+
+import asyncio
+import contextlib
+import inspect
+import threading
+import time
+import weakref
+from collections import deque
+from collections.abc import Awaitable, Callable, Iterator
+from dataclasses import dataclass, replace
+from typing import Final
+
+from litellm.constants import (
+    EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS,
+    EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING,
+)
+
+_CLOSABLE_ANYWHERE: Final = "closable-anywhere"
+_CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop"
+
+_BucketKey = str | int
+
+
+@dataclass(frozen=True, slots=True)
+class _PendingClose:
+    """A queued close.
+
+    The client is held weakly, so queueing one never keeps alive anything the
+    collector would otherwise have reclaimed first.
+
+    ``needs_loop`` is set for a client whose close is a coroutine; those can only
+    be closed from the event loop they were evicted on, recorded in ``loop_id``.
+    A client that closes synchronously carries neither constraint.
+    """
+
+    client_ref: "weakref.ref[object]"
+    loop_id: int | None
+    needs_loop: bool
+    close_after: float
+
+
+def _bucket_key(pending: _PendingClose) -> _BucketKey:
+    """Which reaps can close this entry: any at all, any running a loop, or one loop's."""
+    if not pending.needs_loop:
+        return _CLOSABLE_ANYWHERE
+    if pending.loop_id is None:
+        return _CLOSABLE_ON_ANY_LOOP
+    return pending.loop_id
+
+
+def _running_loop_id() -> int | None:
+    try:
+        return id(asyncio.get_running_loop())
+    except RuntimeError:
+        return None
+
+
+def _close_function(client: object) -> Callable[[], object] | None:
+    close_fn: Final[Callable[[], object] | None] = getattr(client, "aclose", None) or getattr(client, "close", None)
+    return close_fn
+
+
+def _transport_of(client: object) -> object:
+    """The httpx transport behind an SDK wrapper, a litellm handler, or a bare client."""
+    for holder in (getattr(client, "_client", None), getattr(client, "client", None), client):
+        transport: object = getattr(holder, "_transport", None)
+        if transport is not None:
+            return transport
+    return None
+
+
+def _connection_is_idle(connection: object) -> bool:
+    """A pooled connection is idle unless it is servicing a request."""
+    is_idle: Final[object] = getattr(connection, "is_idle", None)
+    return bool(is_idle()) if callable(is_idle) else True
+
+
+def _pool_has_busy_connection(transport: object) -> bool | None:
+    """Whether the httpcore pool behind the transport is servicing a request.
+
+    ``None`` when there is no such pool, so the caller can ask the other backend.
+    """
+    pooled: Final[object] = getattr(getattr(transport, "_pool", None), "connections", None)
+    if not isinstance(pooled, (list, tuple)):
+        return None
+    return any(
+        not _connection_is_idle(connection)  # pyright: ignore[reportUnknownArgumentType]  # untyped pool list
+        for connection in pooled  # pyright: ignore[reportUnknownVariableType]  # untyped pool list
+    )
+
+
+def _has_connection_in_flight(client: object) -> bool:
+    """Whether the client is servicing a request right now.
+
+    Both connection backends litellm uses already account for the connections
+    they have handed out, so this reads the client's own lease accounting rather
+    than inferring it from elapsed time: httpcore reports a non-idle connection
+    for the whole of a response including a stream, and aiohttp holds the
+    connection in ``_acquired`` over the same span.
+
+    A client that cannot answer is reported as idle, which leaves the grace
+    window as the only guard, exactly as it was before this check existed.
+    """
+    try:
+        transport: Final = _transport_of(client)
+        pooled_busy: Final = _pool_has_busy_connection(transport)
+        if pooled_busy is not None:
+            return pooled_busy
+        session: Final[object] = getattr(transport, "client", None)
+        return bool(getattr(getattr(session, "connector", None), "_acquired", None))
+    except Exception:  # noqa: BLE001 - a client that cannot report its state is treated as idle
+        return False
+
+
+async def _close_quietly(closing: Awaitable[object]) -> None:
+    with contextlib.suppress(Exception):
+        await closing
+
+
+class EvictedClientCloser:
+    """Closes evicted, litellm-owned clients once they are idle and out of grace."""
+
+    def __init__(
+        self,
+        grace_seconds: float = EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS,
+        max_pending: int = EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING,
+        clock: Callable[[], float] = time.monotonic,
+    ) -> None:
+        self._grace_seconds = grace_seconds
+        self._max_pending = max_pending
+        self._clock = clock
+        self._owned: weakref.WeakSet[object] = weakref.WeakSet()
+        self._buckets: dict[_BucketKey, deque[_PendingClose]] = {}  # mutable-ok: deadline-ordered queues
+        self._pending_count = 0
+        self._queue_lock = threading.Lock()  # the cache is reachable from every worker thread's loop
+        self._close_tasks: set[asyncio.Task[None]] = set()  # mutable-ok: strong refs to running closes
+
+    def mark_owned(self, client: object) -> None:
+        """Record that litellm created this client, so it may be closed on eviction."""
+        try:
+            self._owned.add(client)
+        except TypeError:
+            pass  # values that cannot be weak-referenced are never litellm clients
+
+    def _is_owned(self, client: object) -> bool:
+        try:
+            return client in self._owned
+        except TypeError:
+            return False  # unhashable values are never litellm clients
+
+    def schedule(self, client: object) -> None:
+        """Queue an evicted client for closing once it is idle and out of grace.
+
+        Past ``max_pending`` the client is left to the collector instead, so a
+        workload that churns the cache cannot grow this queue without bound.
+        Every queued entry comes due within one grace window, so the capacity it
+        occupies is returned within that window rather than held.
+        """
+        if client is None or not self._is_owned(client):
+            return
+        close_fn: Final = _close_function(client)
+        if close_fn is None:
+            return
+        if self._pending_count >= self._max_pending:
+            return
+        self._enqueue(
+            _PendingClose(
+                client_ref=weakref.ref(client),
+                loop_id=_running_loop_id(),
+                needs_loop=inspect.iscoroutinefunction(close_fn),
+                close_after=self._clock() + self._grace_seconds,
+            )
+        )
+
+    def reap(self) -> None:
+        """Close every queued client that is due, idle, and closable from here.
+
+        Called from the cache's read path, so the empty-queue exit comes first and
+        the work done past it is proportional to what is due, not to the queue.
+        """
+        if not self._pending_count:
+            return
+        now: Final = self._clock()
+        for pending in self._take_due(_running_loop_id(), now):
+            client = pending.client_ref()
+            if client is None:
+                continue
+            if _has_connection_in_flight(client):
+                self._enqueue(replace(pending, close_after=now + self._grace_seconds))
+                continue
+            self._close(client)
+
+    @property
+    def pending_count(self) -> int:
+        return self._pending_count
+
+    def _enqueue(self, pending: _PendingClose) -> None:
+        """Append to the entry's bucket, dropping any dead entries it queues behind.
+
+        Deadlines only ever move forward, so appending keeps each bucket ordered
+        by deadline, and entries whose client the collector already took sit at
+        the front rather than having to be searched for.
+        """
+        with self._queue_lock:
+            bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque())  # mutable-ok: FIFO by design
+            while bucket and bucket[0].client_ref() is None:
+                bucket.popleft()
+                self._pending_count -= 1
+            bucket.append(pending)
+            self._pending_count += 1
+
+    def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]:
+        buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id)
+        with self._queue_lock:
+            return tuple(pending for key in buckets for pending in self._drain_locked(key, now))
+
+    def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]:
+        bucket: Final = self._buckets.get(key)
+        if bucket is None:
+            return
+        while bucket and bucket[0].close_after <= now:
+            self._pending_count -= 1
+            yield bucket.popleft()
+        if not bucket:
+            del self._buckets[key]
+
+    def _close(self, client: object) -> None:
+        close_fn: Final = _close_function(client)
+        if close_fn is None:
+            return
+        try:
+            closing: Final = close_fn()
+        except Exception:  # noqa: BLE001 - a discarded client's close must never surface to callers
+            return
+        if not inspect.isawaitable(closing):
+            return
+        task: Final = asyncio.get_running_loop().create_task(_close_quietly(closing))
+        self._close_tasks.add(task)
+        task.add_done_callback(self._close_tasks.discard)
+
+
+default_evicted_client_closer: Final = EvictedClientCloser()

litellm/caching/llm_caching_handler.py

@@ -5,21 +5,44 @@ Add the event loop to the cache key, to prevent event loop closed errors.
 import asyncio
 from typing import Final
 
+from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
 from .in_memory_cache import InMemoryCache
 
 
 class LLMClientCache(InMemoryCache):
     """Cache for LLM HTTP clients (OpenAI, Azure, httpx, etc.).
 
-    IMPORTANT: This cache intentionally does NOT close clients on eviction.
-    Evicted clients may still be in use by in-flight requests. Closing them
-    eagerly causes ``RuntimeError: Cannot send a request, as the client has
-    been closed.`` errors in production after the TTL (1 hour) expires.
+    An evicted client is never closed on the spot: a request handed the client
+    just before eviction is still using it, and closing it there raises
+    ``RuntimeError: Cannot send a request, as the client has been closed.``
 
-    Clients that are no longer referenced will be garbage-collected normally.
-    For explicit shutdown cleanup, use ``close_litellm_async_clients()``.
+    Nor can eviction be left to rely on garbage collection. The SDK clients are
+    reference cycles, so an evicted client and its open TCP connections survive
+    until a generational collection runs. Instead a client litellm created is
+    handed to ``EvictedClientCloser``, which closes it once a grace window has
+    passed. Clients the caller supplied are left untouched.
     """
 
+    def __init__(
+        self,
+        max_size_in_memory: int | None = 200,
+        default_ttl: int | None = 600,
+        max_size_per_item: int | None = 1024,
+        evicted_client_closer: EvictedClientCloser | None = None,
+    ) -> None:
+        super().__init__(
+            max_size_in_memory=max_size_in_memory,
+            default_ttl=default_ttl,
+            max_size_per_item=max_size_per_item,
+        )
+        self.evicted_client_closer = evicted_client_closer or default_evicted_client_closer
+
+    def _remove_key(self, key: str) -> None:
+        evicted: Final[object] = self.cache_dict.get(key)
+        super()._remove_key(key)
+        self.evicted_client_closer.schedule(evicted)
+        self.evicted_client_closer.reap()
+
     def update_cache_key_with_event_loop(self, key):
         """
         Add the event loop to the cache key, to prevent event loop closed errors.
@@ -32,16 +55,22 @@ class LLMClientCache(InMemoryCache):
         except RuntimeError:  # handle no current running event loop
             return key
 
-    def set_cache(self, key, value, **kwargs):
+    def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
+        """``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""
+        if litellm_owned_client:
+            self.evicted_client_closer.mark_owned(value)
         key = self.update_cache_key_with_event_loop(key)
         return super().set_cache(key, value, **kwargs)
 
-    async def async_set_cache(self, key, value, **kwargs):
+    async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
+        if litellm_owned_client:
+            self.evicted_client_closer.mark_owned(value)
         key = self.update_cache_key_with_event_loop(key)
         return await super().async_set_cache(key, value, **kwargs)
 
     def get_cache(self, key, **kwargs):
         key = self.update_cache_key_with_event_loop(key)
+        self.evicted_client_closer.reap()
 
         return super().get_cache(key, **kwargs)
 

litellm/caching/redis_cache.py

@@ -18,7 +18,7 @@ import time
 from collections.abc import Awaitable, Callable, Sequence
 from contextvars import ContextVar
 from datetime import timedelta
-from typing import TYPE_CHECKING, Any, Final, TypeVar, Union, cast
+from typing import TYPE_CHECKING, Any, Final, TypeVar, cast
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
@@ -49,7 +49,7 @@ if TYPE_CHECKING:
     cluster_pipeline = ClusterPipeline
     async_redis_client = Redis
     async_redis_cluster_client = RedisCluster
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     pipeline = Any
     cluster_pipeline = Any

litellm/caching/redis_cluster_cache.py

@@ -5,7 +5,7 @@ Key differences:
 - RedisClient NEEDs to be re-used across requests, adds 3000ms latency if it's re-created
 """
 
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm.caching.redis_cache import RedisCache
 
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
 
     pipeline = Pipeline
     async_redis_client = Redis
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     pipeline = Any
     async_redis_client = Any

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -367,7 +367,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                 stream_options = normalize_responses_api_stream_options(value)
                 if stream_options is not None:
                     responses_api_request["stream_options"] = stream_options
-            elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys():
+            elif key in ResponsesAPIOptionalRequestParams.__annotations__:
                 responses_api_request[key] = value
             elif key == "previous_response_id":
                 responses_api_request["previous_response_id"] = value

litellm/constants.py

@@ -91,6 +91,8 @@ DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD: Final = float(
 )
 MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH: Final = int(os.getenv("MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH", 150))
 
+DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS: Final = 2000
+
 # Semantic Guard Defaults
 DEFAULT_SEMANTIC_GUARD_EMBEDDING_MODEL: Final = str(
     os.getenv("DEFAULT_SEMANTIC_GUARD_EMBEDDING_MODEL", "text-embedding-3-small")
@@ -197,6 +199,16 @@ RUNWAYML_POLLING_TIMEOUT = int(os.getenv("RUNWAYML_POLLING_TIMEOUT", 600))  # 10
 ########## Networking constants ##############################################################
 _DEFAULT_TTL_FOR_HTTPX_CLIENTS: Final = 3600  # 1 hour, re-use the same httpx client for 1 hour
 
+# The earliest an evicted, litellm-created client may be closed. A request handed the
+# client just before eviction is still using it, so nothing is closed inside this window;
+# past it, the client is closed once it reports no connection in flight.
+EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS: Final = 900
+
+# How many evicted clients may be queued for closing at once. Past this, an evicted client
+# is left to the collector rather than letting a cache-churning workload grow the queue
+# without bound. Each queued entry is ~100 bytes and comes due within one grace window.
+EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING: Final = 10_000
+
 # Aiohttp connection pooling - prevents memory leaks from unbounded connection growth
 # Set to 0 for unlimited (not recommended for production)
 AIOHTTP_CONNECTOR_LIMIT: Final = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 1000))

litellm/containers/__init__.py

@@ -23,22 +23,20 @@ from .main import (
 )
 
 __all__ = [
-    # Core container operations
     "acreate_container",
     "adelete_container",
-    "alist_containers",
-    "aretrieve_container",
-    "create_container",
-    "delete_container",
-    "list_containers",
-    "retrieve_container",
-    # Container file operations (auto-generated from endpoints.json)
     "adelete_container_file",
     "alist_container_files",
+    "alist_containers",
+    "aretrieve_container",
     "aretrieve_container_file",
     "aretrieve_container_file_content",
+    "create_container",
+    "delete_container",
     "delete_container_file",
     "list_container_files",
+    "list_containers",
+    "retrieve_container",
     "retrieve_container_file",
     "retrieve_container_file_content",
 ]

litellm/fine_tuning/main.py

@@ -80,7 +80,7 @@ async def acreate_fine_tuning_job(
     hyperparameters: dict | None = {},
     suffix: str | None = None,
     validation_file: str | None = None,
-    integrations: List[str] | None = None,
+    integrations: list[str] | None = None,
     seed: int | None = None,
     custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
     extra_headers: dict[str, str] | None = None,
@@ -157,7 +157,7 @@ def create_fine_tuning_job(
     hyperparameters: dict | None = {},
     suffix: str | None = None,
     validation_file: str | None = None,
-    integrations: List[str] | None = None,
+    integrations: list[str] | None = None,
     seed: int | None = None,
     custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
     extra_headers: dict[str, str] | None = None,

litellm/google_genai/adapters/handler.py

@@ -1,7 +1,8 @@
-from collections.abc import AsyncIterator, Coroutine
-from typing import Any, Final, cast
+from collections.abc import AsyncIterator, Coroutine, Mapping
+from typing import Final, cast
 
 import litellm
+from litellm.types.google_genai.adapters import GenerateContentCompletionKwargs
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import ModelResponse
 
@@ -17,12 +18,12 @@ class GenerateContentToCompletionHandler:
     @staticmethod
     def _prepare_completion_kwargs(
         model: str,
-        contents: list[dict[str, Any]] | dict[str, Any],
-        config: dict[str, Any] | None = None,
+        contents: list[dict[str, object]] | dict[str, object],
+        config: dict[str, object] | None = None,
         stream: bool = False,
         litellm_params: GenericLiteLLMParams | None = None,
-        extra_kwargs: dict[str, Any] | None = None,
-    ) -> dict[str, Any]:
+        extra_kwargs: Mapping[str, object] | None = None,
+    ) -> GenerateContentCompletionKwargs:
         """Prepare kwargs for litellm.completion/acompletion"""
 
         # Transform generate_content request to completion format
@@ -34,7 +35,7 @@ class GenerateContentToCompletionHandler:
             **(extra_kwargs or {}),
         )
 
-        completion_kwargs: Final[dict[str, Any]] = dict(completion_request)
+        completion_kwargs: Final = dict(completion_request)
 
         # Forward extra_kwargs that should be passed to completion call
         if extra_kwargs is not None:
@@ -48,17 +49,17 @@ class GenerateContentToCompletionHandler:
         if stream:
             completion_kwargs["stream"] = stream
 
-        return completion_kwargs
+        return GenerateContentCompletionKwargs(**completion_kwargs)
 
     @staticmethod
     async def async_generate_content_handler(
         model: str,
-        contents: list[dict[str, Any]] | dict[str, Any],
+        contents: list[dict[str, object]] | dict[str, object],
         litellm_params: GenericLiteLLMParams,
-        config: dict[str, Any] | None = None,
+        config: dict[str, object] | None = None,
         stream: bool = False,
-        **kwargs,
-    ) -> dict[str, Any] | AsyncIterator[bytes]:
+        **kwargs: object,
+    ) -> dict[str, object] | AsyncIterator[bytes]:
         """Handle generate_content call asynchronously using completion adapter"""
 
         completion_kwargs: Final = GenerateContentToCompletionHandler._prepare_completion_kwargs(
@@ -103,13 +104,13 @@ class GenerateContentToCompletionHandler:
     @staticmethod
     def generate_content_handler(
         model: str,
-        contents: list[dict[str, Any]] | dict[str, Any],
+        contents: list[dict[str, object]] | dict[str, object],
         litellm_params: GenericLiteLLMParams,
-        config: dict[str, Any] | None = None,
+        config: dict[str, object] | None = None,
         stream: bool = False,
         _is_async: bool = False,
-        **kwargs,
-    ) -> dict[str, Any] | AsyncIterator[bytes] | Coroutine[Any, Any, dict[str, Any] | AsyncIterator[bytes]]:
+        **kwargs: object,
+    ) -> dict[str, object] | AsyncIterator[bytes] | Coroutine[None, None, dict[str, object] | AsyncIterator[bytes]]:
         """Handle generate_content call using completion adapter"""
 
         if _is_async:

litellm/integrations/arize/arize.py

@@ -6,7 +6,7 @@ this file has Arize ai specific helper functions
 
 import os
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm.integrations.arize import _utils
 from litellm.integrations.arize._utils import ArizeOTELAttributes
@@ -21,7 +21,7 @@ if TYPE_CHECKING:
     from litellm.types.integrations.arize import Protocol as _Protocol
 
     Protocol = _Protocol
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Protocol = Any
     Span = Any

litellm/integrations/arize/arize_phoenix.py

@@ -1,7 +1,7 @@
 import os
 import threading
 from collections import OrderedDict
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm._logging import verbose_logger
 from litellm.integrations.arize import _utils
@@ -22,7 +22,7 @@ if TYPE_CHECKING:
 
     Protocol = _Protocol
     OpenTelemetryConfig = _OpenTelemetryConfig
-    Span = Union[_Span, Any]
+    Span = _Span | Any
     OpenTelemetry = _OpenTelemetry
     LITELLM_TRACER_NAME: str
 else:

litellm/integrations/code_interpreter_interception/handler.py

@@ -9,13 +9,18 @@ captured stdout back through the typed agentic loop plan.
 import json
 import time
 import uuid
-from typing import Any, Final, Literal, TypedDict, cast
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypedDict, runtime_checkable
 
 from pydantic import ValidationError
 
 import litellm
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.llms.base_llm.sandbox.transformation import (
+    CodeExecutionResult,
+    ContainerHandle,
+)
 from litellm.types.integrations.code_interpreter_interception import (
     CodeInterpreterInterceptionConfig,
 )
@@ -37,6 +42,9 @@ from litellm.types.utils import (
     ModelResponse,
 )
 
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
 LITELLM_CODE_EXECUTION_TOOL_NAME: Final = "litellm_code_execution"
 _INTERCEPTION_ACTIVE_KEY: Final = "_code_interpreter_interception_active"
 _SANDBOX_KEY: Final = "_code_interpreter_interception_sandbox_key"
@@ -109,26 +117,94 @@ class ChatCompletionFunctionToolChoice(TypedDict):
 CodeExecutionFunctionToolChoice = ResponsesFunctionToolChoice | ChatCompletionFunctionToolChoice
 
 
-def _extract_session_id(kwargs: dict[str, Any]) -> str | None:
+class SandboxToolParams(TypedDict):
+    sandbox_provider: str
+    api_key: str | None
+    api_base: str | None
+
+
+class SandboxConfigProtocol(Protocol):
+    async def acreate_sandbox(self) -> ContainerHandle: ...
+
+    async def arun_code(self, *, container: ContainerHandle, code: str) -> CodeExecutionResult: ...
+
+    async def adelete_sandbox(self, *, container: ContainerHandle) -> object: ...
+
+
+@runtime_checkable
+class _SupportsOutput(Protocol):
+    output: object
+
+
+_CachedContainer: TypeAlias = tuple[ContainerHandle, SandboxToolParams | None, float, str | None]
+
+
+def _output_item_type(item: object) -> object:
+    if isinstance(item, dict):
+        item_mapping: Final[dict[str, object]] = item
+        return item_mapping.get("type")
+    return getattr(item, "type", None)
+
+
+def _response_output(response: object) -> object:
+    if isinstance(response, dict):
+        response_mapping: Final[Mapping[str, object]] = response
+        return response_mapping.get("output", [])
+    return getattr(response, "output", []) or []
+
+
+def _tool_call_arguments(arguments: object) -> str:
+    if isinstance(arguments, str):
+        return arguments
+    return "" if arguments is None else str(arguments)
+
+
+def _narrow_tool_call(tool_call: Mapping[str, object]) -> CodeExecutionToolCall:
+    tool_call_id: Final = tool_call.get("id")
+    call_id: Final = tool_call.get("call_id")
+    return {
+        "id": tool_call_id if isinstance(tool_call_id, str) else None,
+        "call_id": call_id if isinstance(call_id, str) else None,
+        "type": "function",
+        "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+        "arguments": _tool_call_arguments(tool_call.get("arguments")),
+    }
+
+
+def _extract_session_id(kwargs: dict[str, object]) -> str | None:
     for meta_key in ("metadata", "litellm_metadata"):
         meta = kwargs.get(meta_key)
         if isinstance(meta, dict):
-            sid = meta.get("session_id")
+            metadata: dict[str, object] = meta
+            sid = metadata.get("session_id")
             if sid and isinstance(sid, str):
                 return sid
     return None
 
 
-def _extract_identity(kwargs: dict[str, Any]) -> str:
-    return kwargs.get("user_api_key_hash") or ""
+def _extract_identity(kwargs: Mapping[str, object]) -> str:
+    identity: Final = kwargs.get("user_api_key_hash")
+    return identity if isinstance(identity, str) else ""
 
 
-def _resolve_sandbox_tool(sandbox_tool_name: str | None) -> dict[str, Any] | None:
+def _resolve_sandbox_tool(sandbox_tool_name: str | None) -> SandboxToolParams | None:
+    if sandbox_tool_name is None:
+        return None
     try:
         from litellm.sandbox.sandbox_tools import resolve_sandbox_tool
     except ImportError:
         return None
-    return resolve_sandbox_tool(sandbox_tool_name)
+    resolved: Final[dict[str, object] | None] = resolve_sandbox_tool(sandbox_tool_name)
+    if resolved is None:
+        return None
+    provider: Final = resolved.get("sandbox_provider")
+    api_key: Final = resolved.get("api_key")
+    api_base: Final = resolved.get("api_base")
+    return SandboxToolParams(
+        sandbox_provider=provider if isinstance(provider, str) else "",
+        api_key=api_key if isinstance(api_key, str) else None,
+        api_base=api_base if isinstance(api_base, str) else None,
+    )
 
 
 class CodeInterpreterInterceptionLogger(CustomLogger):
@@ -149,14 +225,14 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         enabled: bool = True,
         enabled_providers: list[str] | None = None,
         sandbox_tool_name: str | None = None,
-        sandbox_config: Any | None = None,
+        sandbox_config: SandboxConfigProtocol | None = None,
     ):
         super().__init__()
         self.enabled = enabled
         self.enabled_providers = enabled_providers
         self.sandbox_tool_name = sandbox_tool_name
         self.sandbox_config = sandbox_config
-        self._container_cache: dict[str, tuple[Any, dict[str, Any] | None, float, str | None]] = {}
+        self._container_cache: dict[str, _CachedContainer] = {}
 
     @classmethod
     def from_config_yaml(cls, config: CodeInterpreterInterceptionConfig) -> "CodeInterpreterInterceptionLogger":
@@ -171,19 +247,18 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         litellm_settings: dict[str, Any],
         callback_specific_params: dict[str, Any],
     ) -> "CodeInterpreterInterceptionLogger":
-        params: CodeInterpreterInterceptionConfig = {}
-        if "code_interpreter_interception_params" in litellm_settings:
-            params = litellm_settings["code_interpreter_interception_params"]
-        elif "code_interpreter_interception" in callback_specific_params and isinstance(
-            callback_specific_params["code_interpreter_interception"], dict
-        ):
-            params = cast(
-                CodeInterpreterInterceptionConfig,
-                callback_specific_params["code_interpreter_interception"],
-            )
+        params: Final[CodeInterpreterInterceptionConfig] = (
+            litellm_settings["code_interpreter_interception_params"]
+            if "code_interpreter_interception_params" in litellm_settings
+            else callback_specific_params["code_interpreter_interception"]
+            if isinstance(callback_specific_params.get("code_interpreter_interception"), dict)
+            else {}
+        )
         return CodeInterpreterInterceptionLogger.from_config_yaml(params)
 
-    async def async_pre_call_deployment_hook(self, kwargs: dict[str, Any], call_type: CallTypes | None) -> dict | None:
+    async def async_pre_call_deployment_hook(
+        self, kwargs: dict[str, object], call_type: CallTypes | None
+    ) -> dict | None:
         if not kwargs.get("_agentic_loop_depth"):
             kwargs.pop(_INTERCEPTION_ACTIVE_KEY, None)
             kwargs.pop(_SANDBOX_KEY, None)
@@ -229,13 +304,14 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         return kwargs
 
     @staticmethod
-    def _strip_interception_metadata(kwargs: dict[str, Any]) -> None:
+    def _strip_interception_metadata(kwargs: dict[str, object]) -> None:
         metadata: Final = kwargs.get(_LITELLM_METADATA_KEY)
         if not isinstance(metadata, dict):
             return
+        current_metadata: Final[dict[str, object]] = metadata
         filtered_metadata: Final = {
             key: value
-            for key, value in metadata.items()
+            for key, value in current_metadata.items()
             if not is_interception_internal_key(key)
             and not key.startswith("_agentic_loop")
             and key != "max_agentic_loops"
@@ -247,9 +323,9 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
             kwargs.pop(_LITELLM_METADATA_KEY, None)
 
     @staticmethod
-    def _write_interception_metadata(kwargs: dict[str, Any]) -> None:
-        metadata = kwargs.get(_LITELLM_METADATA_KEY)
-        metadata = dict(metadata) if isinstance(metadata, dict) else {}
+    def _write_interception_metadata(kwargs: dict[str, object]) -> None:
+        existing: Final = kwargs.get(_LITELLM_METADATA_KEY)
+        metadata: Final[dict[str, object]] = dict(existing) if isinstance(existing, dict) else {}
         for key in (_INTERCEPTION_ACTIVE_KEY, _SANDBOX_KEY, _SESSION_SCOPED_KEY, _CONVERTED_STREAM_KEY):
             if key in kwargs:
                 metadata[key] = kwargs[key]
@@ -296,20 +372,21 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         }
 
     @staticmethod
-    def _tool_choice_targets_code_interpreter(tool_choice: Any) -> bool:
+    def _tool_choice_targets_code_interpreter(tool_choice: object) -> bool:
         if not isinstance(tool_choice, dict):
             return False
-        function: Final = tool_choice.get("function")
+        choice: Final[dict[str, object]] = tool_choice
+        function: Final = choice.get("function")
         return (
-            tool_choice.get("type") == "code_interpreter"
-            or tool_choice.get("name") == "code_interpreter"
-            or tool_choice.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME
+            choice.get("type") == "code_interpreter"
+            or choice.get("name") == "code_interpreter"
+            or choice.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME
             or (isinstance(function, dict) and function.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME)
         )
 
-    def _resolve_provider(self, kwargs: dict[str, Any]) -> str | None:
+    def _resolve_provider(self, kwargs: dict[str, object]) -> str | None:
         provider: Final = kwargs.get("custom_llm_provider")
-        if provider:
+        if isinstance(provider, str) and provider:
             return provider
         model: Final = kwargs.get("model")
         if not isinstance(model, str):
@@ -321,7 +398,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
 
     async def async_should_run_agentic_loop(
         self,
-        response: Any,
+        response: object,
         model: str,
         messages: list[dict],
         tools: list[dict] | None,
@@ -351,12 +428,12 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         tools: dict,
         model: str,
         messages: list[dict],
-        response: Any,
-        anthropic_messages_provider_config: Any,
-        anthropic_messages_optional_request_params: dict,
-        logging_obj: Any,
+        response: object,
+        anthropic_messages_provider_config: object,
+        anthropic_messages_optional_request_params: dict[str, object],
+        logging_obj: "LiteLLMLoggingObj",
         stream: bool,
-        kwargs: dict,
+        kwargs: dict[str, object],
     ) -> AgenticLoopPlan:
         if kwargs.get("_agentic_loop_api_surface") == CHAT_COMPLETION_AGENTIC_SURFACE:
             return await self._build_chat_completion_agentic_loop_plan(
@@ -368,14 +445,14 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
             )
 
         await self._prune_expired_cache()
-        tool_calls: Final = cast(list[CodeExecutionToolCall], tools.get("tool_calls", []))
-        sandbox_key: Final = kwargs.get(_SANDBOX_KEY)
+        tool_calls: Final = self._agentic_tool_calls(tools)
+        sandbox_key: Final = self._extract_sandbox_key(kwargs)
         is_session: Final = bool(kwargs.get(_SESSION_SCOPED_KEY))
         identity: Final = _extract_identity(kwargs) if is_session else None
         container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity)
 
         try:
-            container_id: Final = cast(str | None, getattr(container, "id", None))
+            container_id: Final = self._container_id(container)
             input_list: Final = self._normalize_messages(messages)
             code_interpreter_calls: Final[list[CodeInterpreterCall]] = []
             for tool_call in tool_calls:
@@ -443,14 +520,14 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         kwargs: dict[str, object],
     ) -> AgenticLoopPlan:
         await self._prune_expired_cache()
-        tool_calls: Final = cast(list[CodeExecutionToolCall], tools.get("tool_calls", []))
-        sandbox_key: Final = cast(str | None, kwargs.get(_SANDBOX_KEY))
+        tool_calls: Final = self._agentic_tool_calls(tools)
+        sandbox_key: Final = self._extract_sandbox_key(kwargs)
         is_session: Final = bool(kwargs.get(_SESSION_SCOPED_KEY))
-        identity: Final = _extract_identity(cast(dict[str, Any], kwargs)) if is_session else None
+        identity: Final = _extract_identity(kwargs) if is_session else None
         container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity)
 
         try:
-            container_id: Final = cast(str | None, getattr(container, "id", None))
+            container_id: Final = self._container_id(container)
             tool_results: Final = [
                 await self._build_chat_completion_tool_result(
                     container=container,
@@ -489,10 +566,28 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
             },
         )
 
+    @staticmethod
+    def _container_id(container: ContainerHandle) -> str | None:
+        container_id: Final[object] = getattr(container, "id", None)
+        return container_id if isinstance(container_id, str) else None
+
+    @staticmethod
+    def _agentic_tool_calls(tools: dict[str, object]) -> list[CodeExecutionToolCall]:
+        tool_calls: Final = tools.get("tool_calls")
+        if not isinstance(tool_calls, list):
+            return []
+        items: Final[list[object]] = tool_calls
+        return [_narrow_tool_call(item) for item in items if isinstance(item, dict)]
+
+    @staticmethod
+    def _extract_sandbox_key(kwargs: dict[str, object]) -> str | None:
+        sandbox_key: Final = kwargs.get(_SANDBOX_KEY)
+        return sandbox_key if isinstance(sandbox_key, str) else None
+
     async def _build_chat_completion_tool_result(
         self,
-        container: object,
-        params: dict[str, Any] | None,
+        container: ContainerHandle,
+        params: SandboxToolParams | None,
         tool_call: CodeExecutionToolCall,
         container_id: str | None,
     ) -> tuple[ChatCompletionToolMessage, CodeInterpreterCall]:
@@ -517,10 +612,15 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         )
 
     async def async_agentic_loop_cleanup_hook(self, plan: AgenticLoopPlan, kwargs: dict) -> None:
-        metadata: Final = plan.metadata or {} if plan else {}
+        metadata: Final[dict[str, object]] = plan.metadata or {} if plan else {}
         if metadata.get("is_session_scoped"):
             return
-        await self._delete_container_for_cache_key(metadata.get("sandbox_key"))
+        await self._delete_container_for_cache_key(self._metadata_sandbox_key(metadata))
+
+    @staticmethod
+    def _metadata_sandbox_key(metadata: Mapping[str, object]) -> str | None:
+        sandbox_key: Final = metadata.get("sandbox_key")
+        return sandbox_key if isinstance(sandbox_key, str) else None
 
     @staticmethod
     def _filter_agentic_loop_kwargs(kwargs: dict[str, object]) -> dict[str, object]:
@@ -531,12 +631,12 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
             and not is_interception_internal_key(k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES)
         }
 
-    def _get_followup_tools(self, tools: object, call_type: CallTypes | None) -> list[dict[str, Any]] | None:
+    def _get_followup_tools(self, tools: object, call_type: CallTypes | None) -> list[dict[str, object]] | None:
         if not isinstance(tools, list):
             return None
         return [
             (
-                self._get_function_tool(call_type=call_type)
+                dict(self._get_function_tool(call_type=call_type))
                 if isinstance(tool, dict) and tool.get("type") == "code_interpreter"
                 else tool
             )
@@ -549,34 +649,42 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
             k: v for k, v in optional_params.items() if k != "tools" and not (k == "tool_choice" and drop_tool_choice)
         }
 
-    async def async_post_agentic_loop_response_hook(self, response: Any, plan: AgenticLoopPlan, kwargs: dict) -> Any:
-        metadata: Final = plan.metadata or {} if plan else {}
+    async def async_post_agentic_loop_response_hook(
+        self, response: object, plan: AgenticLoopPlan, kwargs: dict
+    ) -> object:
+        metadata: Final[dict[str, object]] = plan.metadata or {} if plan else {}
         if not metadata.get("is_session_scoped"):
-            await self._delete_container_for_cache_key(metadata.get("sandbox_key"))
+            await self._delete_container_for_cache_key(self._metadata_sandbox_key(metadata))
 
         calls: Final = metadata.get("code_interpreter_calls")
-        if not calls:
+        if not calls or not isinstance(calls, list):
             return response
 
-        is_dict: Final = isinstance(response, dict)
-        output: Final = response.get("output") if is_dict else getattr(response, "output", None)
-        if not isinstance(output, list):
+        if isinstance(response, dict):
+            response_mapping: Final[dict[str, object]] = response
+            merged_mapping_output: Final = self._merge_code_interpreter_calls(response_mapping.get("output"), calls)
+            if merged_mapping_output is not None:
+                response_mapping["output"] = merged_mapping_output
             return response
 
-        def _item_type(item: Any) -> Any:
-            return item.get("type") if isinstance(item, dict) else getattr(item, "type", None)
-
-        insert_at: Final = next(
-            (i for i, item in enumerate(output) if _item_type(item) == "message"),
-            len(output),
-        )
-        new_output: Final = output[:insert_at] + list(calls) + output[insert_at:]
-        if is_dict:
-            response["output"] = new_output
-        else:
-            response.output = new_output
+        if not isinstance(response, _SupportsOutput):
+            return response
+        merged_attr_output: Final = self._merge_code_interpreter_calls(response.output, calls)
+        if merged_attr_output is not None:
+            response.output = merged_attr_output
         return response
 
+    @staticmethod
+    def _merge_code_interpreter_calls(output: object, calls: Sequence[object]) -> list[object] | None:
+        if not isinstance(output, list):
+            return None
+        items: Final[list[object]] = output
+        insert_at: Final = next(
+            (i for i, item in enumerate(items) if _output_item_type(item) == "message"),
+            len(items),
+        )
+        return items[:insert_at] + list(calls) + items[insert_at:]
+
     @staticmethod
     def _parse_code(arguments: str) -> str:
         try:
@@ -584,7 +692,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         except (json.JSONDecodeError, TypeError, AttributeError):
             return ""
 
-    async def _run_tool_call(self, container: Any, params: dict[str, Any] | None, arguments: str) -> str:
+    async def _run_tool_call(self, container: ContainerHandle, params: SandboxToolParams | None, arguments: str) -> str:
         try:
             code: Final = json.loads(arguments).get("code", "") if arguments else ""
         except (json.JSONDecodeError, TypeError):
@@ -601,7 +709,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         self,
         cache_key: str | None,
         identity: str | None = None,
-    ) -> tuple[Any, dict[str, Any] | None]:
+    ) -> tuple[ContainerHandle, SandboxToolParams | None]:
         if cache_key:
             cached: Final = self._container_cache.get(cache_key)
             if cached is not None:
@@ -623,7 +731,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         self._container_cache.pop(lru_key, None)
         await self._delete_container(container=lru_entry[0], params=lru_entry[1])
 
-    async def _create_container(self) -> tuple[Any, dict[str, Any] | None]:
+    async def _create_container(self) -> tuple[ContainerHandle, SandboxToolParams | None]:
         if self.sandbox_config is not None:
             return await self.sandbox_config.acreate_sandbox(), None
 
@@ -641,7 +749,9 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         )
         return container, params
 
-    async def _run_code(self, container: Any, params: dict[str, Any] | None, code: str) -> Any:
+    async def _run_code(
+        self, container: ContainerHandle, params: SandboxToolParams | None, code: str
+    ) -> CodeExecutionResult:
         if self.sandbox_config is not None:
             return await self.sandbox_config.arun_code(container=container, code=code)
         if params is None:
@@ -653,7 +763,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
             api_key=params.get("api_key"),
         )
 
-    async def _delete_container(self, container: Any, params: dict[str, Any] | None) -> None:
+    async def _delete_container(self, container: ContainerHandle, params: SandboxToolParams | None) -> None:
         try:
             if self.sandbox_config is not None:
                 await self.sandbox_config.adelete_sandbox(container=container)
@@ -677,7 +787,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
             return
         await self._delete_container(container=cached[0], params=cached[1])
 
-    def _normalize_messages(self, messages: Any) -> list[dict[str, Any]]:
+    def _normalize_messages(self, messages: object) -> list[dict[str, object]]:
         if isinstance(messages, str):
             return [{"role": "user", "content": messages}]
         if isinstance(messages, list):
@@ -685,10 +795,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
         return []
 
     def _extract_code_execution_tool_calls(self, response: object) -> list[CodeExecutionToolCall]:
-        if isinstance(response, dict):
-            output = response.get("output", [])
-        else:
-            output = getattr(response, "output", []) or []
+        output: Final = _response_output(response)
         if not isinstance(output, list):
             return []
 
@@ -702,9 +809,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
             if self._is_code_execution_call(item)
         ]
 
-    def _extract_chat_completion_code_execution_tool_calls(
-        self, response: ModelResponse | dict[str, Any]
-    ) -> list[CodeExecutionToolCall]:
+    def _extract_chat_completion_code_execution_tool_calls(self, response: object) -> list[CodeExecutionToolCall]:
         model_response: Final = self._to_model_response(response)
         if model_response is None:
             return []
@@ -743,44 +848,46 @@ class CodeInterpreterInterceptionLogger(CustomLogger):
 
     @staticmethod
     def _build_chat_completion_assistant_message(
-        tool_calls: list[CodeExecutionToolCall],
+        tool_calls: Sequence[CodeExecutionToolCall],
     ) -> ChatCompletionAssistantMessage:
+        assistant_tool_calls: Final[list[ChatCompletionAssistantToolCall]] = [
+            {
+                "id": tool_call.get("id"),
+                "type": "function",
+                "function": {
+                    "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
+                    "arguments": tool_call.get("arguments", ""),
+                },
+            }
+            for tool_call in tool_calls
+        ]
         return {
             "role": "assistant",
-            "tool_calls": [
-                cast(
-                    ChatCompletionAssistantToolCall,
-                    {
-                        "id": tool_call.get("id"),
-                        "type": "function",
-                        "function": {
-                            "name": LITELLM_CODE_EXECUTION_TOOL_NAME,
-                            "arguments": tool_call.get("arguments", ""),
-                        },
-                    },
-                )
-                for tool_call in tool_calls
-            ],
+            "tool_calls": assistant_tool_calls,
         }
 
     @staticmethod
-    def _to_model_response(
-        response: ModelResponse | dict[str, Any],
-    ) -> ModelResponse | None:
+    def _to_model_response(response: object) -> ModelResponse | None:
         if isinstance(response, ModelResponse):
             return response
+        if not isinstance(response, dict):
+            return None
+        response_fields: Final[dict[str, object]] = response
         try:
-            return ModelResponse(**response)
+            return ModelResponse(**response_fields)
         except (TypeError, ValidationError):
             return None
 
-    def _is_code_execution_call(self, item: Any) -> bool:
+    def _is_code_execution_call(self, item: object) -> bool:
         if isinstance(item, dict):
-            return item.get("type") == "function_call" and item.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME
-        return (
-            getattr(item, "type", None) == "function_call"
-            and getattr(item, "name", None) == LITELLM_CODE_EXECUTION_TOOL_NAME
-        )
+            item_mapping: Final[dict[str, object]] = item
+            return (
+                item_mapping.get("type") == "function_call"
+                and item_mapping.get("name") == LITELLM_CODE_EXECUTION_TOOL_NAME
+            )
+        item_type: Final[object] = getattr(item, "type", None)
+        item_name: Final[object] = getattr(item, "name", None)
+        return item_type == "function_call" and item_name == LITELLM_CODE_EXECUTION_TOOL_NAME
 
     async def _prune_expired_cache(self) -> None:
         now: Final = time.time()

litellm/integrations/custom_logger.py

@@ -3,7 +3,7 @@
 import re
 import traceback
 from collections.abc import AsyncGenerator
-from typing import TYPE_CHECKING, Any, Final, Optional, Union
+from typing import TYPE_CHECKING, Any, Final, Optional
 
 from pydantic import BaseModel
 
@@ -39,7 +39,7 @@ if TYPE_CHECKING:
     )
     from litellm.types.router import PreRoutingHookResponse
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
     LiteLLMLoggingObj = Any

litellm/integrations/dotprompt/prompt_manager.py

@@ -31,7 +31,7 @@ class PromptTemplate:
         self.output_format = self.metadata.get("output", {}).get("format")
         self.output_schema = self.metadata.get("output", {}).get("schema", {})
         self.optional_params = {}
-        for key in self.metadata.keys():
+        for key in self.metadata:
             if key not in restricted_keys:
                 self.optional_params[key] = self.metadata[key]
 

litellm/integrations/langfuse/langfuse.py

@@ -128,8 +128,8 @@ class LangFuseLogger:
             self.langfuse_client = create_mock_langfuse_client()
             self.is_mock_mode = True
         else:
-            http_client: Final = _get_httpx_client()
-            self.langfuse_client = http_client.client
+            self._http_handler: Final = _get_httpx_client()
+            self.langfuse_client = self._http_handler.client
             self.is_mock_mode = False
 
         parameters: Final = {

litellm/integrations/langfuse/langfuse_otel.py

@@ -2,7 +2,7 @@ import base64
 import json
 import os
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final, Optional, Union
+from typing import TYPE_CHECKING, Any, Final, Optional
 
 from litellm._logging import verbose_logger
 from litellm.integrations.arize import _utils
@@ -18,7 +18,7 @@ from litellm.types.utils import StandardCallbackDynamicParams
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/integrations/langfuse/langfuse_prompt_management.py

@@ -4,7 +4,7 @@ Call Hook for LiteLLM Proxy which allows Langfuse prompt management.
 
 import os
 from functools import lru_cache
-from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
 
 from packaging.version import Version
 
@@ -30,7 +30,7 @@ if TYPE_CHECKING:
 
     LangfuseClass: TypeAlias = Langfuse
 
-    PROMPT_CLIENT = Union[TextPromptClient, ChatPromptClient]
+    PROMPT_CLIENT = TextPromptClient | ChatPromptClient
 else:
     PROMPT_CLIENT = Any
     LangfuseClass = Any

litellm/integrations/langtrace.py

@@ -1,12 +1,12 @@
 import json
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm.proxy._types import SpanAttributes
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/integrations/levo/levo.py

@@ -1,5 +1,5 @@
 import os
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm.integrations.opentelemetry import OpenTelemetry
 
@@ -13,7 +13,7 @@ if TYPE_CHECKING:
 
     Protocol = _Protocol
     OpenTelemetryConfig = _OpenTelemetryConfig
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Protocol = Any
     OpenTelemetryConfig = Any

litellm/integrations/opentelemetry.py

@@ -1,7 +1,7 @@
 import os
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final, Union, cast
+from typing import TYPE_CHECKING, Any, Final, cast
 
 import litellm
 from litellm._logging import verbose_logger
@@ -47,12 +47,12 @@ if TYPE_CHECKING:
     )
     from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth
 
-    Span = Union[_Span, Any]
-    Tracer = Union[_Tracer, Any]
-    Context = Union[_Context, Any]
-    SpanExporter = Union[_SpanExporter, Any]
-    UserAPIKeyAuth = Union[_UserAPIKeyAuth, Any]
-    ManagementEndpointLoggingPayload = Union[_ManagementEndpointLoggingPayload, Any]
+    Span = _Span | Any
+    Tracer = _Tracer | Any
+    Context = _Context | Any
+    SpanExporter = _SpanExporter | Any
+    UserAPIKeyAuth = _UserAPIKeyAuth | Any
+    ManagementEndpointLoggingPayload = _ManagementEndpointLoggingPayload | Any
 else:
     Span = Any
     Tracer = Any
@@ -186,16 +186,7 @@ def _normalize_team_metadata_keys(value: Any) -> list[str]:
 
 _FREEZE_MAX_DEPTH: Final = 16
 
-HashableScope = Union[
-    str,
-    int,
-    float,
-    bool,
-    bytes,
-    None,
-    tuple["HashableScope", ...],
-    frozenset["HashableScope"],
-]
+HashableScope = str | int | float | bool | bytes | None | tuple["HashableScope", ...] | frozenset["HashableScope"]
 
 
 def _freeze_for_dedupe(value: object, _depth: int = 0) -> HashableScope:

litellm/integrations/opentelemetry_utils/gen_ai_semconv.py

@@ -31,7 +31,7 @@ Events:
 
 from datetime import datetime
 from enum import Enum
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 
@@ -40,7 +40,7 @@ if TYPE_CHECKING:
 
     from litellm.integrations.opentelemetry import OpenTelemetryConfig
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/integrations/opik/opik_payload_builder/types.py

@@ -1,7 +1,7 @@
 """Type definitions for Opik payload building."""
 
 from dataclasses import dataclass
-from typing import Any, Final, Literal, Union
+from typing import Any, Final, Literal
 
 
 @dataclass
@@ -42,5 +42,5 @@ class SpanPayload:
     total_cost: float | None = None
 
 
-PayloadItem = Union[TracePayload, SpanPayload]
+PayloadItem = TracePayload | SpanPayload
 TraceSpanPayloadTuple: Final = tuple[TracePayload | None, SpanPayload]

litellm/integrations/otel/__init__.py

@@ -72,53 +72,49 @@ from litellm.integrations.otel.model.spans import (
 )
 
 __all__ = [
-    # config
-    "OTEL_V2_ENV",
-    "OpenTelemetryV2Config",
-    "is_otel_v2_enabled",
-    # semconv
     "BAGGAGE_PROMOTED_KEYS",
     "DB",
     "DEFAULT_BAGGAGE_METADATA_KEYS",
+    "HTTP",
+    "MCP",
+    "OTEL_V2_ENV",
+    "SPAN_REGISTRY",
     "Client",
     "Error",
     "GenAI",
     "GenAIOperation",
     "GenAIProvider",
-    "HTTP",
-    "JsonRpc",
-    "LiteLLM",
-    "LiteLLMError",
-    "MCP",
-    "MCPMethod",
-    "Metric",
-    "Network",
-    "NetworkTransport",
-    "Server",
-    "resolve_operation",
-    "resolve_provider",
-    # spans
-    "SPAN_REGISTRY",
-    "LiteLLMSpanKind",
-    "SpanRole",
-    "SpanSpec",
-    "db_system",
-    "span_role_for_service",
-    "validate_registry",
-    # payloads
     "GuardrailSpanData",
+    "JsonRpc",
     "LLMCallSpanData",
     "LLMRequestParams",
     "LLMUsage",
+    "LiteLLM",
+    "LiteLLMError",
+    "LiteLLMSpanKind",
     "MCPListToolsSpanData",
+    "MCPMethod",
     "MCPToolCallSpanData",
+    "Metric",
+    "Network",
+    "NetworkTransport",
+    "OpenTelemetryV2Config",
     "ProxyRequestSpanData",
     "RequestContext",
     "RequestIdentity",
+    "Server",
     "ServerInfo",
     "ServiceSpanData",
     "SpanError",
+    "SpanRole",
+    "SpanSpec",
+    "db_system",
     "is_mcp_list_tools",
     "is_mcp_tool_call",
+    "is_otel_v2_enabled",
     "promoted_baggage",
+    "resolve_operation",
+    "resolve_provider",
+    "span_role_for_service",
+    "validate_registry",
 ]

litellm/interactions/__init__.py

@@ -66,18 +66,13 @@ from litellm.interactions.main import (
 )
 
 __all__ = [
-    # Create
-    "create",
-    "acreate",
-    # Get
-    "get",
-    "aget",
-    # Delete
-    "delete",
-    "adelete",
-    # Cancel
-    "cancel",
     "acancel",
-    # Sub-modules
+    "acreate",
+    "adelete",
     "agents",
+    "aget",
+    "cancel",
+    "create",
+    "delete",
+    "get",
 ]

litellm/litellm_core_utils/audio_utils/utils.py

@@ -180,6 +180,7 @@ def get_audio_file_content_hash(file_obj: FileTypes) -> str:
     if isinstance(file_obj, tuple):
         if len(file_obj) < 2:
             fallback_filename = str(file_obj[0]) if len(file_obj) > 0 else None
+            file_content_obj = None
         else:
             fallback_filename = str(file_obj[0]) if file_obj[0] is not None else None
             file_content_obj = file_obj[1]
@@ -206,7 +207,7 @@ def get_audio_file_content_hash(file_obj: FileTypes) -> str:
             except OSError:
                 fallback_filename = str(file_content_obj)
                 file_content = None
-        elif hasattr(file_content_obj, "read"):
+        elif file_content_obj is not None and hasattr(file_content_obj, "read"):
             try:
                 current_position: Final = file_content_obj.tell() if hasattr(file_content_obj, "tell") else None
                 if hasattr(file_content_obj, "seek"):

litellm/litellm_core_utils/core_helpers.py

@@ -2,7 +2,7 @@
 ## Helper utilities
 import copy
 from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any, Final, Literal, Union
+from typing import TYPE_CHECKING, Any, Final, Literal
 
 import httpx
 
@@ -14,7 +14,7 @@ if TYPE_CHECKING:
 
     from litellm.types.utils import ModelResponseStream
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 
@@ -131,6 +131,9 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
     "content_filter": "content_filter",
     # Anthropic Sonnet 4
     "content_filtered": "content_filter",
+    # Generic error passthrough (OpenRouter and other OpenAI-compatible providers
+    # emit lowercase "error" when a provider fails mid-stream)
+    "error": "stop",
 }
 
 

litellm/litellm_core_utils/fallback_generalizations.py

@@ -48,7 +48,7 @@ O(number of rules); callers must only invoke them on a cache miss.
 
 import re
 from dataclasses import dataclass
-from typing import Final, Union
+from typing import Final
 
 from litellm._logging import verbose_logger
 
@@ -100,7 +100,7 @@ class _CapabilityRule:
     model_info: dict
 
 
-_CompiledRule = Union[_RoutingRule, _CapabilityRule]
+_CompiledRule = _RoutingRule | _CapabilityRule
 
 
 def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]:

litellm/litellm_core_utils/litellm_logging.py

@@ -4827,7 +4827,7 @@ class StandardLoggingPayloadSetup:
 
         # Populate well-known typed fields with int/str coercion where needed
         typed_keys: Final[dict] = {}
-        for key in StandardLoggingAdditionalHeaders.__annotations__.keys():
+        for key in StandardLoggingAdditionalHeaders.__annotations__:
             _key = key.lower().replace("_", "-")
             typed_keys[_key] = key
             if _key in additiona_headers:
@@ -4859,7 +4859,7 @@ class StandardLoggingPayloadSetup:
             usage_object=None,
         )
         if hidden_params is not None:
-            for key in StandardLoggingHiddenParams.__annotations__.keys():
+            for key in StandardLoggingHiddenParams.__annotations__:
                 if key in hidden_params:
                     if key == "additional_headers":
                         clean_hidden_params["additional_headers"] = StandardLoggingPayloadSetup.get_additional_headers(
@@ -5501,7 +5501,7 @@ def get_standard_logging_metadata(
     )
     if isinstance(metadata, dict):
         # Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields
-        for key in StandardLoggingMetadata.__annotations__.keys():
+        for key in StandardLoggingMetadata.__annotations__:
             if key in metadata:
                 clean_metadata[key] = metadata[key]
 

litellm/litellm_core_utils/logging_utils.py

@@ -4,7 +4,7 @@ import inspect
 import re
 import time
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm._logging import verbose_logger
 from litellm.constants import MAX_BASE64_LENGTH_FOR_LOGGING
@@ -23,7 +23,7 @@ if TYPE_CHECKING:
     )
 
     LiteLLMModelResponse = _ModelResponse
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     LiteLLMModelResponse = Any
     LiteLLMLoggingObject = Any

litellm/litellm_core_utils/model_response_utils.py

@@ -47,7 +47,7 @@ def is_model_response_stream_empty(model_response: ModelResponseStream) -> bool:
 
     # Check for any non-base fields that are set
     # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
-    for model_response_field in type(model_response).model_fields.keys():
+    for model_response_field in type(model_response).model_fields:
         # Skip base fields that are always set
         if model_response_field in BASE_FIELDS:
             continue

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -3684,7 +3684,7 @@ def _convert_to_bedrock_tool_call_invoke(
                             # cache_control applies to the whole original
                             # tool call; attach after the last split block.
                             if tool.get("cache_control", None) is not None:
-                                _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                                _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
                                     {"cache_control": tool["cache_control"]},
                                     block_type="content_block",
                                     model=model,
@@ -3701,7 +3701,7 @@ def _convert_to_bedrock_tool_call_invoke(
 
                 # Check for cache_control and add a separate cachePoint block
                 if tool.get("cache_control", None) is not None:
-                    cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                    cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
                         {"cache_control": tool["cache_control"]},
                         block_type="content_block",
                         model=model,
@@ -4360,7 +4360,7 @@ class BedrockConverseMessagesProcessor:
                             elif element["type"] == "document":
                                 _part = BedrockConverseMessagesProcessor._process_document_message(element)
                                 _parts.append(_part)
-                            _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                            _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
                                 message_block=cast(OpenAIMessageContentListBlock, element),
                                 block_type="content_block",
                                 model=model,
@@ -4370,7 +4370,7 @@ class BedrockConverseMessagesProcessor:
                     user_content.extend(_parts)
                 elif message_block["content"] and isinstance(message_block["content"], str):
                     _part = BedrockContentBlock(text=messages[msg_i]["content"])
-                    _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                    _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
                         message_block, block_type="content_block", model=model
                     )
                     user_content.append(_part)
@@ -4417,7 +4417,7 @@ class BedrockConverseMessagesProcessor:
 
                 # Add a separate cachePoint block if cache_control is present
                 if tool_msg_cache_control is not None:
-                    cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                    cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
                         {"cache_control": tool_msg_cache_control},
                         block_type="content_block",
                         model=model,
@@ -4496,7 +4496,7 @@ class BedrockConverseMessagesProcessor:
                                 assistants_part = await BedrockImageProcessor.process_image_async(image_url=image_url)
                                 assistants_parts.append(assistants_part)
                                 # Add cache point block for assistant content elements
-                        _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                        _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
                             message_block=cast(OpenAIMessageContentListBlock, element),
                             block_type="content_block",
                             model=model,
@@ -4510,7 +4510,7 @@ class BedrockConverseMessagesProcessor:
                         assistant_content.append(BedrockContentBlock(text=_assistant_content))
                     # If content is empty/whitespace, skip it (don't add a placeholder)
                     # Add cache point block for assistant string content
-                    _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                    _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
                         assistant_message_block, block_type="content_block", model=model
                     )
                     if _cache_point_block is not None:
@@ -4733,7 +4733,7 @@ def _bedrock_converse_messages_pt(
                         elif element["type"] == "document":
                             _part = BedrockConverseMessagesProcessor._process_document_message(element)
                             _parts.append(_part)
-                        _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                        _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
                             message_block=cast(OpenAIMessageContentListBlock, element),
                             block_type="content_block",
                             model=model,
@@ -4743,7 +4743,7 @@ def _bedrock_converse_messages_pt(
                 user_content.extend(_parts)
             elif message_block["content"] and isinstance(message_block["content"], str):
                 _part = BedrockContentBlock(text=messages[msg_i]["content"])
-                _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
                     message_block, block_type="content_block", model=model
                 )
                 user_content.append(_part)
@@ -4792,7 +4792,7 @@ def _bedrock_converse_messages_pt(
 
             # Add a separate cachePoint block if cache_control is present
             if tool_msg_cache_control is not None:
-                cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
                     {"cache_control": tool_msg_cache_control},
                     block_type="content_block",
                     model=model,
@@ -4874,7 +4874,7 @@ def _bedrock_converse_messages_pt(
                             assistants_part = BedrockImageProcessor.process_image_sync(image_url=image_url)
                             assistants_parts.append(assistants_part)
                         # Add cache point block for assistant content elements
-                        _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                        _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
                             message_block=cast(OpenAIMessageContentListBlock, element),
                             block_type="content_block",
                             model=model,
@@ -4887,7 +4887,7 @@ def _bedrock_converse_messages_pt(
                 if _assistant_content.strip():
                     assistant_content.append(BedrockContentBlock(text=_assistant_content))
                 # Add cache point block for assistant string content
-                _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block(
                     assistant_message_block, block_type="content_block", model=model
                 )
                 if _cache_point_block is not None:

litellm/litellm_core_utils/streaming_handler.py

@@ -8,7 +8,7 @@ import time
 import traceback
 from collections.abc import AsyncIterator, Callable, Iterator
 from dataclasses import dataclass
-from typing import Any, Final, NoReturn, TypeVar, Union, cast
+from typing import Any, Final, NoReturn, TypeVar, cast
 
 import anyio
 import httpx
@@ -99,7 +99,7 @@ class _ProviderChunkEarlyReturn:
     value: Any
 
 
-_ProviderChunkResult = Union[_ProviderChunkParsed, _ProviderChunkEarlyReturn]
+_ProviderChunkResult = _ProviderChunkParsed | _ProviderChunkEarlyReturn
 
 
 class CustomStreamWrapper:
@@ -256,9 +256,7 @@ class CustomStreamWrapper:
             chunk = chunk.strip()
             self.complete_response = self.complete_response.strip()
 
-            if chunk.startswith(self.complete_response):
-                # Remove last_sent_chunk only if it appears at the start of the new chunk
-                chunk = chunk[len(self.complete_response) :]
+            chunk = chunk.removeprefix(self.complete_response)
 
             self.complete_response += chunk
             return chunk

litellm/llms/anthropic/experimental_pass_through/adapters/handler.py

@@ -1,5 +1,10 @@
 from collections.abc import AsyncIterator, Coroutine, Iterator
-from typing import Any, Final, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Final,
+    cast,
+)
 
 import litellm
 from litellm._logging import verbose_logger
@@ -21,6 +26,10 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
 from litellm.types.utils import ModelResponse
 from litellm.utils import get_model_info
 
+if TYPE_CHECKING:
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.router import Router
+
 # Anthropic-only keys already mapped by the translator; strip on extra_kwargs re-merge.
 ANTHROPIC_ONLY_REQUEST_KEYS: Final[frozenset[str]] = frozenset({"output_config"})
 
@@ -37,6 +46,14 @@ def _messages_have_compaction_block(messages: list[dict]) -> bool:
     return False
 
 
+def _proxy_router_fallback() -> "Router | None":
+    try:
+        from litellm.proxy.proxy_server import llm_router as _proxy_router
+    except Exception:
+        return None
+    return _proxy_router
+
+
 def _extract_proxy_litellm_metadata(kwargs: dict[str, Any]) -> dict[str, Any] | None:
     """Return ``kwargs["litellm_metadata"]`` when it's a dict; ``None`` otherwise.
 
@@ -64,8 +81,8 @@ async def _prepare_context_managed_request(
     context_management_spec: Any,
     litellm_metadata: dict | None,
     additional_drop_params: list[str] | None,
-    llm_router: Any,
-    user_api_key_auth: Any = None,
+    llm_router: "Router | None",
+    user_api_key_auth: "UserAPIKeyAuth | None" = None,
 ) -> PolyfillResult | None:
     """Apply client compaction history, then optional context_management polyfill."""
     from litellm.llms.anthropic.experimental_pass_through.context_management.editors.compact import (
@@ -149,7 +166,7 @@ def _polyfill_will_run(
         COMPACT_EDIT_TYPE,
     )
 
-    return any(isinstance(edit, dict) and edit.get("type") == COMPACT_EDIT_TYPE for edit in edits)
+    return any(edit.get("type") == COMPACT_EDIT_TYPE for edit in edits)
 
 
 def _spec_has_non_compact_edits(
@@ -175,10 +192,7 @@ def _spec_has_non_compact_edits(
         COMPACT_EDIT_TYPE,
     )
 
-    return any(
-        isinstance(edit, dict) and isinstance(edit.get("type"), str) and edit.get("type") != COMPACT_EDIT_TYPE
-        for edit in edits
-    )
+    return any(isinstance(edit.get("type"), str) and edit.get("type") != COMPACT_EDIT_TYPE for edit in edits)
 
 
 def _context_management_explicitly_dropped(additional_drop_params: list[str] | None) -> bool:
@@ -228,8 +242,8 @@ async def _run_polyfill_if_enabled(
     context_management_spec: Any,
     litellm_metadata: dict | None,
     additional_drop_params: list[str] | None,
-    llm_router: Any,
-    user_api_key_auth: Any = None,
+    llm_router: "Router | None",
+    user_api_key_auth: "UserAPIKeyAuth | None" = None,
 ) -> PolyfillResult | None:
     """Run the async context_management polyfill if a spec is present.
 
@@ -339,7 +353,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         reasoning_effort: Final = completion_kwargs.get("reasoning_effort")
         summary: Final = thinking.get("summary")
         if isinstance(reasoning_effort, str) and reasoning_effort:
-            reasoning_dict: Final[dict[str, Any]] = {"effort": reasoning_effort}
+            reasoning_dict: Final[dict[str, object]] = {"effort": reasoning_effort}
             if summary:
                 reasoning_dict["summary"] = summary
             elif auto_summary:
@@ -528,21 +542,17 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         top_p: float | None = None,
         output_format: dict | None = None,
         **kwargs,
-    ) -> AnthropicMessagesResponse | AsyncIterator[Any] | Iterator[bytes]:
+    ) -> AnthropicMessagesResponse | AsyncIterator[bytes] | Iterator[bytes]:
         """Handle non-Anthropic models asynchronously using the adapter"""
         context_management: Final = kwargs.pop("context_management", None)
         additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None)
-        litellm_router = kwargs.pop("litellm_router", None)
-        if litellm_router is None:
-            try:
-                from litellm.proxy.proxy_server import llm_router as _proxy_router
-
-                litellm_router = _proxy_router
-            except Exception:
-                pass
+        requested_router: Final[Router | None] = kwargs.pop("litellm_router", None)
+        litellm_router: Final[Router | None] = (
+            requested_router if requested_router is not None else _proxy_router_fallback()
+        )
 
         proxy_litellm_metadata: Final = _extract_proxy_litellm_metadata(kwargs)
-        user_api_key_auth: Final = (
+        user_api_key_auth: Final[UserAPIKeyAuth | None] = (
             proxy_litellm_metadata.get("user_api_key_auth") if proxy_litellm_metadata is not None else None
         )
 
@@ -626,8 +636,8 @@ class LiteLLMMessagesToCompletionTransformationHandler:
     ) -> (
         AnthropicMessagesResponse
         | Iterator[bytes]
-        | AsyncIterator[Any]
-        | Coroutine[Any, Any, AnthropicMessagesResponse | AsyncIterator[Any] | Iterator[bytes]]
+        | AsyncIterator[bytes]
+        | Coroutine[None, None, AnthropicMessagesResponse | AsyncIterator[bytes] | Iterator[bytes]]
     ):
         """Handle non-Anthropic models using the adapter."""
         if _is_async is True:
@@ -667,7 +677,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         # ``llm_router`` is ``None``, which is safe to call from the bridged
         # loop. The async ``async_anthropic_messages_handler`` path is
         # unaffected because it ``await``s within the original event loop.
-        litellm_router: Final = kwargs.pop("litellm_router", None)
+        litellm_router: Final[Router | None] = kwargs.pop("litellm_router", None)
 
         # Skip the async bridge entirely when there is nothing for either the
         # polyfill or the client-history slice-only fallback to do. The vast
@@ -679,7 +689,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
             polyfill_result: PolyfillResult | None = None
         else:
             proxy_litellm_metadata: Final = _extract_proxy_litellm_metadata(kwargs)
-            user_api_key_auth: Final = (
+            user_api_key_auth: Final[UserAPIKeyAuth | None] = (
                 proxy_litellm_metadata.get("user_api_key_auth") if proxy_litellm_metadata is not None else None
             )
             polyfill_result = run_async_function(

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -4,8 +4,15 @@ import copy
 import json
 import traceback
 from collections import deque
-from collections.abc import AsyncIterator, Iterator
-from typing import TYPE_CHECKING, Any, Final, Literal, get_args
+from collections.abc import AsyncIterator, Iterator, Sequence
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Final,
+    Literal,
+    Protocol,
+    get_args,
+)
 
 from typing_extensions import assert_never
 
@@ -14,7 +21,9 @@ from litellm._uuid import uuid
 from litellm.types.llms.anthropic import (
     AppliedEdit,
     CompactionBlock,
+    ContentBlockDelta,
     ContextManagementResponse,
+    MessageBlockDelta,
     StreamingContentBlockDeltaType,
     UsageDelta,
     UsageIteration,
@@ -28,6 +37,25 @@ if TYPE_CHECKING:
 _STREAMING_DELTA_TYPES: Final = frozenset(get_args(StreamingContentBlockDeltaType))
 
 
+class _UsageDeltaWithIterations(UsageDelta, total=False):
+    iterations: list[UsageIteration]
+
+
+class _ChunkStream(Protocol):
+    def __iter__(self) -> "Iterator[ModelResponseStream]": ...
+
+    def __aiter__(self) -> "AsyncIterator[ModelResponseStream]": ...
+
+
+def _optional_attr(obj: object, name: str) -> object:
+    return getattr(obj, name, None)
+
+
+def _optional_attr_sequence(obj: object, name: str) -> Sequence[object]:
+    value: Final = getattr(obj, name, None)
+    return value if value else ()
+
+
 def _delta_payload_field(delta_type: StreamingContentBlockDeltaType) -> str:
     match delta_type:
         case "text_delta":
@@ -62,29 +90,29 @@ class _CombinedChunkSplitter:
     would advance them out of sync.
     """
 
-    def __init__(self, completion_stream: Any):
-        self._stream = completion_stream
-        self._sync_iter: Iterator[Any] | None = None
-        self._async_iter: AsyncIterator[Any] | None = None
-        self._buffer: deque = deque()
+    def __init__(self, completion_stream: _ChunkStream):
+        self._stream: _ChunkStream = completion_stream
+        self._sync_iter: Iterator[ModelResponseStream] | None = None
+        self._async_iter: AsyncIterator[ModelResponseStream] | None = None
+        self._buffer: deque[ModelResponseStream] = deque()
 
     @staticmethod
-    def _is_combined(chunk: Any) -> bool:
+    def _is_combined(chunk: "ModelResponseStream") -> bool:
         """True if ``chunk`` carries response content AND a finish_reason."""
-        choices: Final = getattr(chunk, "choices", None)
+        choices: Final = _optional_attr_sequence(chunk, "choices")
         if not choices:
             return False
         choice: Final = choices[0]
-        if getattr(choice, "finish_reason", None) is None:
+        if _optional_attr(choice, "finish_reason") is None:
             return False
-        delta: Final = getattr(choice, "delta", None)
+        delta: Final = _optional_attr(choice, "delta")
         if delta is None:
             return False
         return bool(
-            getattr(delta, "content", None)
-            or getattr(delta, "tool_calls", None)
-            or getattr(delta, "reasoning_content", None)
-            or getattr(delta, "thinking_blocks", None)
+            _optional_attr(delta, "content")
+            or _optional_attr(delta, "tool_calls")
+            or _optional_attr(delta, "reasoning_content")
+            or _optional_attr(delta, "thinking_blocks")
         )
 
     _PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = (
@@ -119,21 +147,21 @@ class _CombinedChunkSplitter:
         normalized to ``reasoning_content`` so the synthesized block start
         stays empty and the thinking text is emitted exactly once.
         """
-        choices: Final = getattr(chunk, "choices", None)
-        if not choices or len(choices) != 1:
+        choices: Final = _optional_attr_sequence(chunk, "choices")
+        if len(choices) != 1:
             return (chunk,)
-        delta: Final = getattr(choices[0], "delta", None)
+        delta: Final = _optional_attr(choices[0], "delta")
         if delta is None:
             return (chunk,)
-        tool_calls: Final = getattr(delta, "tool_calls", None)
+        tool_calls: Final = _optional_attr_sequence(delta, "tool_calls")
         if tool_calls and not any(
-            getattr(getattr(tool_call, "function", None), "name", None) for tool_call in tool_calls
+            _optional_attr(_optional_attr(tool_call, "function"), "name") for tool_call in tool_calls
         ):
             return (chunk,)
         present_groups: Final = tuple(
             group
             for group in _CombinedChunkSplitter._PAYLOAD_FIELD_GROUPS
-            if any(getattr(delta, field, None) for field in group)
+            if any(_optional_attr(delta, field) for field in group)
         )
         if len(present_groups) <= 1:
             return (chunk,)
@@ -172,7 +200,7 @@ class _CombinedChunkSplitter:
         return {"reasoning_content": thinking_text}
 
     @staticmethod
-    def _split(chunk: Any) -> list[Any]:
+    def _split(chunk: "ModelResponseStream") -> "list[ModelResponseStream]":
         """Return ``[chunk]``, or ``[content_chunk, finish_chunk]`` if combined."""
         if not _CombinedChunkSplitter._is_combined(chunk):
             return [chunk]
@@ -194,10 +222,10 @@ class _CombinedChunkSplitter:
             finish_delta.thinking_blocks = None
         return [content_chunk, finish_chunk]
 
-    def __iter__(self) -> "Iterator[Any]":
+    def __iter__(self) -> "Iterator[ModelResponseStream]":
         return self
 
-    def __next__(self) -> Any:
+    def __next__(self) -> "ModelResponseStream":
         if self._buffer:
             return self._buffer.popleft()
         if self._sync_iter is None:
@@ -210,10 +238,10 @@ class _CombinedChunkSplitter:
         )
         return self._buffer.popleft()
 
-    def __aiter__(self) -> "AsyncIterator[Any]":
+    def __aiter__(self) -> "AsyncIterator[ModelResponseStream]":
         return self
 
-    async def __anext__(self) -> Any:
+    async def __anext__(self) -> "ModelResponseStream":
         if self._buffer:
             return self._buffer.popleft()
         if self._async_iter is None:
@@ -246,14 +274,14 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     sent_content_block_finish: bool = False
     current_content_block_type: Literal["text", "tool_use", "thinking"] = "text"
     sent_last_message: bool = False
-    holding_chunk: Any | None = None
-    holding_stop_reason_chunk: Any | None = None
+    holding_chunk: ContentBlockDelta | None = None
+    holding_stop_reason_chunk: MessageBlockDelta | None = None
     queued_usage_chunk: bool = False
     current_content_block_index: int = 0
 
     def __init__(
         self,
-        completion_stream: Any,
+        completion_stream: _ChunkStream,
         model: str,
         tool_name_mapping: dict[str, str] | None = None,
         applied_edits: list[AppliedEdit] | None = None,
@@ -294,7 +322,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
             text="",
         )
 
-    def _merge_usage_into_held_stop_reason_chunk(self, chunk: Any) -> dict[str, Any]:
+    def _merge_usage_into_held_stop_reason_chunk(self, chunk: Any) -> MessageBlockDelta:
         """Merge usage data from ``chunk`` into the held ``message_delta`` chunk.
 
         Shared by both the sync ``__next__`` and async ``__anext__`` paths so
@@ -320,7 +348,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
             merged_chunk["context_management"] = ContextManagementResponse(applied_edits=list(self.applied_edits))
         return self._augment_message_delta_usage(merged_chunk)
 
-    def _ensure_context_management_attached(self, message_delta_chunk: dict[str, Any]) -> dict[str, Any]:
+    def _ensure_context_management_attached(self, message_delta_chunk: MessageBlockDelta) -> MessageBlockDelta:
         """Attach ``context_management`` to a ``message_delta`` chunk if
         ``self.applied_edits`` is non-empty and the chunk does not already
         carry it. Returns the (possibly new) chunk dict.
@@ -335,7 +363,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         augmented["context_management"] = ContextManagementResponse(applied_edits=list(self.applied_edits))
         return augmented
 
-    def _augment_message_delta_usage(self, message_delta_chunk: dict[str, Any]) -> dict[str, Any]:
+    def _augment_message_delta_usage(self, message_delta_chunk: MessageBlockDelta) -> MessageBlockDelta:
         """Attach polyfill compaction iteration usage to the final message_delta.
 
         Also defensively re-attaches ``context_management`` so the direct
@@ -352,7 +380,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         input_tokens: Final = usage.get("input_tokens", 0) or 0
         output_tokens: Final = usage.get("output_tokens", 0) or 0
         augmented: Final = message_delta_chunk.copy()
-        augmented_usage: Final = dict(usage)
+        augmented_usage: Final[_UsageDeltaWithIterations] = {**usage}
         iterations: Final[list[UsageIteration]] = list(self.iterations_usage)
         # Only emit a ``message`` iteration when we have real token data.
         # Without a separate usage chunk (e.g. provider sent finish_reason

litellm/llms/azure/common_utils.py

@@ -427,88 +427,95 @@ class BaseAzureLLM(BaseOpenAILLM):
             f"|azure_password={hashlib.sha256(_azure_password.encode()).hexdigest() if isinstance(_azure_password, str) else None}"
             f"|azure_scope={_lp.get('azure_scope')}"
         )
-        if client is None:
-            cached_client: Final = self.get_cached_openai_client(
-                client_initialization_params=client_initialization_params,
-                client_type="azure",
-            )
-            if cached_client:
-                if isinstance(cached_client, (AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI)):
-                    return cached_client
-
-            azure_client_params: Final = self.initialize_azure_sdk_client(
-                litellm_params=litellm_params or {},
-                api_key=api_key,
-                api_base=api_base,
-                model_name=model,
-                api_version=api_version,
-                is_async=_is_async,
-            )
-
-            # For Azure v1 API, use standard OpenAI client instead of AzureOpenAI
-            # See: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#api-specs
-            if self._is_azure_v1_api_version(api_version):
-                # Extract only params that OpenAI client accepts
-                # Always use /openai/v1/ regardless of whether user passed "v1", "latest", or "preview"
-                # The OpenAI client accepts a callable for `api_key` and re-invokes it
-                # on every request (via `_refresh_api_key`), so passing
-                # `azure_ad_token_provider` directly preserves Azure AD token refresh
-                # behavior that the regular AzureOpenAI client provides.
-                v1_api_key: str | Callable[[], Any] | None = (
-                    azure_client_params.get("api_key")
-                    or azure_client_params.get("azure_ad_token_provider")
-                    or azure_client_params.get("azure_ad_token")
-                )
-                if _is_async is True and callable(v1_api_key):
-                    # AsyncOpenAI expects an async provider; wrap the sync provider
-                    # returned by azure-identity. Offload to a thread so a token
-                    # refresh (blocking HTTP call to AAD on cache miss) does not
-                    # stall the event loop.
-                    _sync_provider: Final = v1_api_key
-
-                    async def _async_v1_api_key() -> str:
-                        return await asyncio.to_thread(_sync_provider)
-
-                    v1_api_key = _async_v1_api_key
-
-                v1_params: Final[dict[str, Any]] = {
-                    "api_key": v1_api_key,
-                    "base_url": f"{api_base}/openai/v1/",
-                }
-                if "timeout" in azure_client_params:
-                    v1_params["timeout"] = azure_client_params["timeout"]
-                if "max_retries" in azure_client_params:
-                    v1_params["max_retries"] = azure_client_params["max_retries"]
-                if "http_client" in azure_client_params:
-                    v1_params["http_client"] = azure_client_params["http_client"]
-
-                verbose_logger.debug("Using Azure v1 API with base_url: %s", v1_params["base_url"])
-
-                if _is_async is True:
-                    openai_client = AsyncOpenAI(**v1_params)
-                else:
-                    openai_client = OpenAI(**v1_params)
-            else:
-                # Traditional Azure API uses AzureOpenAI client
-                if _is_async is True:
-                    openai_client = AsyncAzureOpenAI(**azure_client_params)
-                else:
-                    openai_client = AzureOpenAI(**azure_client_params)
-        else:
-            openai_client = client
+        if client is not None:
             if (
                 api_version is not None
-                and isinstance(openai_client, (AzureOpenAI, AsyncAzureOpenAI))
-                and isinstance(openai_client._custom_query, dict)
+                and isinstance(client, (AzureOpenAI, AsyncAzureOpenAI))
+                and isinstance(client._custom_query, dict)
             ):
                 # set api_version to version passed by user
-                openai_client._custom_query.setdefault("api-version", api_version)
+                client._custom_query.setdefault("api-version", api_version)
+            self.set_cached_openai_client(
+                openai_client=client,
+                client_initialization_params=client_initialization_params,
+                client_type="azure",
+                litellm_owned_client=False,
+            )
+            return client
+
+        cached_client: Final = self.get_cached_openai_client(
+            client_initialization_params=client_initialization_params,
+            client_type="azure",
+        )
+        if cached_client:
+            if isinstance(cached_client, (AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI)):
+                return cached_client
+
+        azure_client_params: Final = self.initialize_azure_sdk_client(
+            litellm_params=litellm_params or {},
+            api_key=api_key,
+            api_base=api_base,
+            model_name=model,
+            api_version=api_version,
+            is_async=_is_async,
+        )
+
+        # For Azure v1 API, use standard OpenAI client instead of AzureOpenAI
+        # See: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#api-specs
+        if self._is_azure_v1_api_version(api_version):
+            # Extract only params that OpenAI client accepts
+            # Always use /openai/v1/ regardless of whether user passed "v1", "latest", or "preview"
+            # The OpenAI client accepts a callable for `api_key` and re-invokes it
+            # on every request (via `_refresh_api_key`), so passing
+            # `azure_ad_token_provider` directly preserves Azure AD token refresh
+            # behavior that the regular AzureOpenAI client provides.
+            v1_api_key: str | Callable[[], Any] | None = (
+                azure_client_params.get("api_key")
+                or azure_client_params.get("azure_ad_token_provider")
+                or azure_client_params.get("azure_ad_token")
+            )
+            if _is_async is True and callable(v1_api_key):
+                # AsyncOpenAI expects an async provider; wrap the sync provider
+                # returned by azure-identity. Offload to a thread so a token
+                # refresh (blocking HTTP call to AAD on cache miss) does not
+                # stall the event loop.
+                _sync_provider: Final = v1_api_key
+
+                async def _async_v1_api_key() -> str:
+                    return await asyncio.to_thread(_sync_provider)
+
+                v1_api_key = _async_v1_api_key
+
+            v1_params: Final[dict[str, Any]] = {
+                "api_key": v1_api_key,
+                "base_url": f"{api_base}/openai/v1/",
+            }
+            if "timeout" in azure_client_params:
+                v1_params["timeout"] = azure_client_params["timeout"]
+            if "max_retries" in azure_client_params:
+                v1_params["max_retries"] = azure_client_params["max_retries"]
+            if "http_client" in azure_client_params:
+                v1_params["http_client"] = azure_client_params["http_client"]
+
+            verbose_logger.debug("Using Azure v1 API with base_url: %s", v1_params["base_url"])
+
+            if _is_async is True:
+                openai_client = AsyncOpenAI(**v1_params)
+            else:
+                openai_client = OpenAI(**v1_params)
+        else:
+            # Traditional Azure API uses AzureOpenAI client
+            if _is_async is True:
+                openai_client = AsyncAzureOpenAI(**azure_client_params)
+            else:
+                openai_client = AzureOpenAI(**azure_client_params)
 
         # save client in-memory cache
         self.set_cached_openai_client(
             openai_client=openai_client,
             client_initialization_params=client_initialization_params,
             client_type="azure",
+            litellm_owned_client=self.owns_wrapped_http_client(azure_client_params.get("http_client")),
         )
         return openai_client
 

litellm/llms/base_llm/managed_resources/base_managed_resource.py

@@ -5,7 +5,7 @@
 import base64
 import json
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, Final, Generic, TypeVar, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Generic, TypeVar, cast
 
 from litellm import verbose_logger
 from litellm.llms.base_llm.managed_resources.isolation import (
@@ -23,7 +23,7 @@ if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient as _PrismaClient
     from litellm.router import Router as _Router
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
     InternalUsageCache = _InternalUsageCache
     PrismaClient = _PrismaClient
     Router = _Router

litellm/llms/bedrock/base_aws_llm.py

@@ -859,6 +859,7 @@ class BaseAWSLLM:
                     "Action": [
                         "bedrock:InvokeModel",
                         "bedrock:InvokeModelWithResponseStream",
+                        "bedrock:CountTokens",
                         "bedrock:ApplyGuardrail",
                         "bedrock:GetGuardrail",
                         "bedrock:ListGuardrails",

litellm/llms/bedrock/chat/converse_transformation.py

@@ -1081,7 +1081,7 @@ class AmazonConverseConfig(BaseConfig):
                 optional_params["maxTokens"] = thinking_token_budget + DEFAULT_MAX_TOKENS
 
     @overload
-    def _get_cache_point_block(
+    def get_cache_point_block(
         self,
         message_block: OpenAIMessageContentListBlock
         | ChatCompletionUserMessage
@@ -1093,7 +1093,7 @@ class AmazonConverseConfig(BaseConfig):
         pass
 
     @overload
-    def _get_cache_point_block(
+    def get_cache_point_block(
         self,
         message_block: OpenAIMessageContentListBlock
         | ChatCompletionUserMessage
@@ -1104,7 +1104,7 @@ class AmazonConverseConfig(BaseConfig):
     ) -> ContentBlock | None:
         pass
 
-    def _get_cache_point_block(
+    def get_cache_point_block(
         self,
         message_block: OpenAIMessageContentListBlock
         | ChatCompletionUserMessage
@@ -1149,14 +1149,14 @@ class AmazonConverseConfig(BaseConfig):
                 system_prompt_indices.append(idx)
                 if isinstance(message["content"], str) and message["content"]:
                     system_content_blocks.append(SystemContentBlock(text=message["content"]))
-                    cache_block = self._get_cache_point_block(message, block_type="system", model=model)
+                    cache_block = self.get_cache_point_block(message, block_type="system", model=model)
                     if cache_block:
                         system_content_blocks.append(cache_block)
                 elif isinstance(message["content"], list):
                     for m in message["content"]:
                         if m.get("type") == "text" and m.get("text"):
                             system_content_blocks.append(SystemContentBlock(text=m["text"]))
-                            cache_block = self._get_cache_point_block(m, block_type="system", model=model)
+                            cache_block = self.get_cache_point_block(m, block_type="system", model=model)
                             if cache_block:
                                 system_content_blocks.append(cache_block)
         if len(system_prompt_indices) > 0:

litellm/llms/bedrock/chat/invoke_handler.py

@@ -57,7 +57,7 @@ class AmazonCohereChatConfig:
     Reference - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-command-r-plus.html
     """
 
-    documents: List[Document] | None = None
+    documents: list[Document] | None = None
     search_queries_only: bool | None = None
     preamble: str | None = None
     max_tokens: int | None = None
@@ -69,12 +69,12 @@ class AmazonCohereChatConfig:
     presence_penalty: float | None = None
     seed: int | None = None
     return_prompt: bool | None = None
-    stop_sequences: List[str] | None = None
+    stop_sequences: list[str] | None = None
     raw_prompting: bool | None = None
 
     def __init__(
         self,
-        documents: List[Document] | None = None,
+        documents: list[Document] | None = None,
         search_queries_only: bool | None = None,
         preamble: str | None = None,
         max_tokens: int | None = None,
@@ -112,7 +112,7 @@ class AmazonCohereChatConfig:
             and v is not None
         }
 
-    def get_supported_openai_params(self) -> List[str]:
+    def get_supported_openai_params(self) -> list[str]:
         return [
             "max_tokens",
             "max_completion_tokens",
@@ -325,7 +325,7 @@ class AWSEventStreamDecoder:
 
         self.model = model
         self.parser = EventStreamJSONParser()
-        self.content_blocks: List[ContentBlockDeltaEvent] = []
+        self.content_blocks: list[ContentBlockDeltaEvent] = []
         self.tool_calls_index: int | None = None
         self.response_id: str | None = None
         self.json_mode = json_mode
@@ -362,13 +362,13 @@ class AWSEventStreamDecoder:
 
     def translate_thinking_blocks(
         self, thinking_block: BedrockConverseReasoningContentBlockDelta
-    ) -> List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] | None:
+    ) -> list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None:
         """
         Translate the thinking blocks to a string
         """
 
-        thinking_blocks_list: Final[List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]] = []
-        _thinking_block: Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] | None = None
+        thinking_blocks_list: Final[list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock]] = []
+        _thinking_block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock | None = None
 
         if "text" in thinking_block:
             _thinking_block = ChatCompletionThinkingBlock(type="thinking")
@@ -402,12 +402,12 @@ class AWSEventStreamDecoder:
     ) -> tuple[
         ChatCompletionToolCallChunk | None,
         dict,
-        List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] | None,
+        list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None,
     ]:
         """Handle 'start' event in converse chunk parsing."""
         tool_use: ChatCompletionToolCallChunk | None = None
         provider_specific_fields: dict = {}
-        thinking_blocks: List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] | None = None
+        thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None = None
 
         self.content_blocks = []  # reset
         if start_obj is not None:
@@ -450,14 +450,14 @@ class AWSEventStreamDecoder:
         ChatCompletionToolCallChunk | None,
         dict,
         str | None,
-        List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] | None,
+        list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None,
     ]:
         """Handle 'delta' event in converse chunk parsing."""
         text = ""
         tool_use: ChatCompletionToolCallChunk | None = None
         provider_specific_fields: dict = {}
         reasoning_content: str | None = None
-        thinking_blocks: List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] | None = None
+        thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None = None
 
         self.content_blocks.append(delta_obj)
         if "text" in delta_obj:
@@ -535,7 +535,7 @@ class AWSEventStreamDecoder:
             usage: Usage | None = None
             provider_specific_fields: dict = {}
             reasoning_content: str | None = None
-            thinking_blocks: List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] | None = None
+            thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None = None
 
             content_block_index: Final = int(chunk_data.get("contentBlockIndex", 0))
             if "start" in chunk_data:
@@ -590,7 +590,7 @@ class AWSEventStreamDecoder:
         except Exception as e:
             raise Exception(f"Received streaming error - {e}")
 
-    def _chunk_parser(self, chunk_data: dict) -> Union[GChunk, ModelResponseStream, dict]:
+    def _chunk_parser(self, chunk_data: dict) -> GChunk | ModelResponseStream | dict:
         text = ""
         is_finished = False
         finish_reason = ""
@@ -645,7 +645,7 @@ class AWSEventStreamDecoder:
             tool_use=None,
         )
 
-    def iter_bytes(self, iterator: Iterator[bytes]) -> Iterator[Union[GChunk, ModelResponseStream, dict]]:
+    def iter_bytes(self, iterator: Iterator[bytes]) -> Iterator[GChunk | ModelResponseStream | dict]:
         """Given an iterator that yields lines, iterate over it & yield every event encountered"""
         from botocore.eventstream import EventStreamBuffer
 
@@ -659,9 +659,7 @@ class AWSEventStreamDecoder:
                     _data = json.loads(message)
                     yield self._chunk_parser(chunk_data=_data)
 
-    async def aiter_bytes(
-        self, iterator: AsyncIterator[bytes]
-    ) -> AsyncIterator[Union[GChunk, ModelResponseStream, dict]]:
+    async def aiter_bytes(self, iterator: AsyncIterator[bytes]) -> AsyncIterator[GChunk | ModelResponseStream | dict]:
         """Given an async iterator that yields lines, iterate over it & yield every event encountered"""
         from botocore.eventstream import EventStreamBuffer
 
@@ -741,7 +739,7 @@ class AmazonDeepSeekR1StreamDecoder(AWSEventStreamDecoder):
             sync_stream=sync_stream,
         )
 
-    def _chunk_parser(self, chunk_data: dict) -> Union[GChunk, ModelResponseStream, dict]:
+    def _chunk_parser(self, chunk_data: dict) -> GChunk | ModelResponseStream | dict:
         return self.deepseek_model_response_iterator.chunk_parser(chunk=chunk_data)
 
 
@@ -756,7 +754,7 @@ class MockResponseIterator:  # for returning ai21 streaming responses
         return self
 
     def _handle_json_mode_chunk(
-        self, text: str, tool_calls: List[ChatCompletionToolCallChunk] | None
+        self, text: str, tool_calls: list[ChatCompletionToolCallChunk] | None
     ) -> tuple[str, ChatCompletionToolCallChunk | None]:
         """
         If JSON mode is enabled, convert the tool call to a message.
@@ -789,7 +787,7 @@ class MockResponseIterator:  # for returning ai21 streaming responses
             text = chunk_data.choices[0].message.content or ""
             tool_use = None
             _model_response_tool_call: Final = cast(
-                List[ChatCompletionMessageToolCall] | None,
+                list[ChatCompletionMessageToolCall] | None,
                 cast(Choices, chunk_data.choices[0]).message.tool_calls,
             )
             if self.json_mode is True:

litellm/llms/bedrock/embed/cohere_transformation.py

@@ -34,7 +34,7 @@ class BedrockCohereEmbeddingConfig:
         new_transformed_request: Final = CohereEmbeddingRequest(
             input_type=transformed_request["input_type"],
         )
-        for k in CohereEmbeddingRequest.__annotations__.keys():
+        for k in CohereEmbeddingRequest.__annotations__:
             if k in transformed_request:
                 new_transformed_request[k] = transformed_request[k]
 

litellm/llms/bedrock/files/transformation.py

@@ -759,7 +759,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
             import hashlib
 
             import requests
-            from botocore.auth import SigV4Auth
+            from botocore.auth import S3SigV4Auth
             from botocore.awsrequest import AWSRequest
         except ImportError:
             raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
@@ -804,7 +804,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         # Get region name for non-LLM API calls (same as s3_v2.py)
         signing_region: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=aws_region_name)
 
-        SigV4Auth(credentials, "s3", signing_region).add_auth(aws_request)
+        S3SigV4Auth(credentials, "s3", signing_region).add_auth(aws_request)
 
         # Return signed headers and body
         signed_body = aws_request.body
@@ -1015,7 +1015,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         try:
             import hashlib
 
-            from botocore.auth import SigV4Auth
+            from botocore.auth import S3SigV4Auth
             from botocore.awsrequest import AWSRequest
         except ImportError:
             raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
@@ -1038,7 +1038,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
             url=api_base,
             headers={"x-amz-content-sha256": empty_body_hash},
         )
-        auth: Final = SigV4Auth(credentials, "s3", aws_region_name)  # any-ok: botocore untyped
+        auth: Final = S3SigV4Auth(credentials, "s3", aws_region_name)  # any-ok: botocore untyped
         auth.add_auth(aws_request)  # any-ok: botocore request mutation is untyped
         return dict(aws_request.headers)  # any-ok: botocore headers are untyped
 

litellm/llms/bedrock/image_generation/image_handler.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import json
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 import httpx
 from pydantic import BaseModel
@@ -49,12 +49,12 @@ class BedrockImagePreparedRequest(BaseModel):
     data: dict
 
 
-BedrockImageConfigClass = Union[
-    type[AmazonTitanImageGenerationConfig],
-    type[AmazonNovaCanvasConfig],
-    type[AmazonStability3Config],
-    type[AmazonStabilityConfig],
-]
+BedrockImageConfigClass = (
+    type[AmazonTitanImageGenerationConfig]
+    | type[AmazonNovaCanvasConfig]
+    | type[AmazonStability3Config]
+    | type[AmazonStabilityConfig]
+)
 
 
 class BedrockImageGeneration(BaseAWSLLM):

litellm/llms/bedrock/vector_stores/transformation.py

@@ -160,10 +160,10 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
                 aws_filters: dict | None = None
 
                 if isinstance(value, dict):
-                    if "operator" in value.keys():
+                    if "operator" in value:
                         # Single operator - map directly (no wrapping needed)
                         aws_filters = self._map_operator_filter(value)
-                    elif "and" in value.keys() or "or" in value.keys():
+                    elif "and" in value or "or" in value:
                         aws_filters = self._map_and_or_filters(value)
                     else:
                         # Assume it's already in AWS KB format

litellm/llms/custom_httpx/http_handler.py

@@ -129,6 +129,21 @@ def _default_cached_client_timeout() -> httpx.Timeout:
     return httpx.Timeout(timeout=configured, connect=HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS)
 
 
+_CLIENT_REFCOUNT_WHEN_HANDLER_IS_SOLE_REFERRER: Final = 2
+
+
+def _handler_may_close_client(client_refcount: int, owns_client: bool) -> bool:
+    """
+    Whether a handler being finalized may close its client.
+
+    Only when the handler built the client and is still its sole referrer. Finalization
+    proves that nothing references the *handler*; it proves nothing about the client, which
+    a cached handler may have handed to consumers that outlive it. Callers must read the
+    refcount at the call site, since binding the client to a parameter would inflate it.
+    """
+    return owns_client and client_refcount <= _CLIENT_REFCOUNT_WHEN_HANDLER_IS_SOLE_REFERRER
+
+
 _STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS: Final = 5.0
 _STREAMING_ERROR_BODY_READ_EXECUTOR: Final = concurrent.futures.ThreadPoolExecutor(
     max_workers=50,
@@ -577,14 +592,15 @@ class AsyncHTTPHandler:
 
     async def close(self):
         # Close the client when you're done with it
-        await self._client.aclose()
+        if self._owns_client:
+            await self._client.aclose()
 
     async def __aenter__(self):
         return self.client
 
     async def __aexit__(self):
         # close the client when exiting
-        await self._client.aclose()
+        await self.close()
 
     async def get(
         self,
@@ -893,7 +909,9 @@ class AsyncHTTPHandler:
 
     def __del__(self) -> None:
         try:
-            asyncio.get_running_loop().create_task(self.close())
+            if not _handler_may_close_client(sys.getrefcount(self._client), self._owns_client):
+                return
+            asyncio.get_running_loop().create_task(self._client.aclose())
         except Exception:
             pass
 
@@ -1132,7 +1150,8 @@ class HTTPHandler:
 
     def close(self):
         # Close the client when you're done with it
-        self._client.close()
+        if self._owns_client:
+            self._client.close()
 
     def get(
         self,
@@ -1375,7 +1394,8 @@ class HTTPHandler:
 
     def __del__(self) -> None:
         try:
-            self.close()
+            if _handler_may_close_client(sys.getrefcount(self._client), self._owns_client):
+                self._client.close()
         except Exception:
             pass
 
@@ -1441,6 +1461,7 @@ def get_async_httpx_client(
         key=_cache_key_name,
         value=_new_client,
         ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
+        litellm_owned_client=True,
     )
     return _new_client
 
@@ -1486,5 +1507,6 @@ def _get_httpx_client(params: dict | None = None) -> HTTPHandler:
         key=_cache_key_name,
         value=_new_client,
         ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
+        litellm_owned_client=True,
     )
     return _new_client

litellm/llms/jina_ai/embedding/transformation.py

@@ -82,10 +82,7 @@ class JinaAIEmbeddingConfig(BaseEmbeddingConfig):
         """
         api_base = api_base or get_secret_str("JINA_AI_API_BASE") or "https://api.jina.ai/v1"
         dynamic_api_key: Final = api_key or (
-            get_secret_str("JINA_AI_API_KEY")
-            or get_secret_str("JINA_AI_API_KEY")
-            or get_secret_str("JINA_AI_API_KEY")
-            or get_secret_str("JINA_AI_TOKEN")
+            get_secret_str("JINA_AI_API_KEY") or get_secret_str("JINA_API_KEY") or get_secret_str("JINA_AI_TOKEN")
         )
         return LlmProviders.JINA_AI.value, api_base, dynamic_api_key
 

litellm/llms/openai/common_utils.py

@@ -128,13 +128,33 @@ class BaseOpenAILLM:
         _cached_client: Final = litellm.in_memory_llm_clients_cache.get_cache(_cache_key)
         return _cached_client
 
+    @staticmethod
+    def owns_wrapped_http_client(http_client: httpx.Client | httpx.AsyncClient | None) -> bool:
+        """Whether litellm may close an SDK client built around ``http_client``.
+
+        ``_get_async_http_client`` / ``_get_sync_http_client`` hand back
+        ``litellm.aclient_session`` / ``litellm.client_session`` when the caller
+        configured one. The SDK's ``close()`` closes whatever http client it was
+        given, so an SDK client wrapping one of those shared sessions must never be
+        closed on eviction; the caller goes on using the session. ``None`` means the
+        SDK built its own http client, which litellm does own.
+        """
+        if http_client is None:
+            return True
+        return http_client is not litellm.aclient_session and http_client is not litellm.client_session
+
     @staticmethod
     def set_cached_openai_client(
         openai_client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI,
         client_type: Literal["openai", "azure"],
         client_initialization_params: dict,
+        litellm_owned_client: bool = False,
     ):
-        """Stores the OpenAI client in the in-memory cache for _DEFAULT_TTL_FOR_HTTPX_CLIENTS SECONDS"""
+        """Stores the OpenAI client in the in-memory cache for _DEFAULT_TTL_FOR_HTTPX_CLIENTS SECONDS
+
+        ``litellm_owned_client`` says litellm built this client, so the cache may close it once it
+        is evicted. A client the caller supplied stays open, since litellm does not own it.
+        """
         _cache_key: Final = BaseOpenAILLM.get_openai_client_cache_key(
             client_initialization_params=client_initialization_params,
             client_type=client_type,
@@ -143,6 +163,7 @@ class BaseOpenAILLM:
             key=_cache_key,
             value=openai_client,
             ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
+            litellm_owned_client=litellm_owned_client,
         )
 
     @staticmethod

litellm/llms/openai/openai.py

@@ -345,7 +345,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         client: OpenAI | AsyncOpenAI | None = None,
         shared_session: Optional["ClientSession"] = None,
     ) -> OpenAI | AsyncOpenAI | None:
-        client_initialization_params: Final[Dict] = locals()
+        client_initialization_params: Final[dict] = locals()
         if client is None:
             if not isinstance(max_retries, int):
                 raise OpenAIError(
@@ -360,11 +360,16 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             if cached_client:
                 if isinstance(cached_client, OpenAI) or isinstance(cached_client, AsyncOpenAI):
                     return cached_client
+            http_client: Final[httpx.Client | httpx.AsyncClient | None] = (
+                OpenAIChatCompletion._get_async_http_client(shared_session=shared_session)
+                if is_async
+                else OpenAIChatCompletion._get_sync_http_client()
+            )
             if is_async:
                 _new_client: OpenAI | AsyncOpenAI = AsyncOpenAI(
                     api_key=api_key,
                     base_url=api_base,
-                    http_client=OpenAIChatCompletion._get_async_http_client(shared_session=shared_session),
+                    http_client=http_client,
                     timeout=timeout,
                     max_retries=max_retries,
                     organization=organization,
@@ -373,7 +378,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                 _new_client = OpenAI(
                     api_key=api_key,
                     base_url=api_base,
-                    http_client=OpenAIChatCompletion._get_sync_http_client(),
+                    http_client=http_client,
                     timeout=timeout,
                     max_retries=max_retries,
                     organization=organization,
@@ -384,6 +389,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                 openai_client=_new_client,
                 client_initialization_params=client_initialization_params,
                 client_type="openai",
+                litellm_owned_client=self.owns_wrapped_http_client(http_client),
             )
             return _new_client
 
@@ -402,7 +408,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         data: dict,
         timeout: float | httpx.Timeout,
         logging_obj: LiteLLMLoggingObj,
-    ) -> Tuple[dict, BaseModel]:
+    ) -> tuple[dict, BaseModel]:
         """
         Helper to:
         - call chat.completions.create.with_raw_response when litellm.return_response_headers is True
@@ -439,7 +445,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         data: dict,
         timeout: float | httpx.Timeout,
         logging_obj: LiteLLMLoggingObj,
-    ) -> Tuple[dict, BaseModel]:
+    ) -> tuple[dict, BaseModel]:
         """
         Helper to:
         - call chat.completions.create.with_raw_response when litellm.return_response_headers is True
@@ -474,11 +480,11 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         self,
         response: Any,
         model: str,
-        messages: list[Dict],
-        optional_params: Dict,
+        messages: list[dict],
+        optional_params: dict,
         logging_obj: LiteLLMLoggingObj,
         stream: bool,
-        litellm_params: Dict,
+        litellm_params: dict,
     ) -> Any | None:
         """
         Call agentic completion hooks for all custom loggers (OpenAI Chat Completions API).
@@ -1288,7 +1294,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             )
 
             ## embedding CALL
-            headers: Dict | None = None
+            headers: dict | None = None
             headers, sync_embedding_response = self.make_sync_openai_embedding_request(
                 openai_client=openai_client,
                 data=data,
@@ -2842,7 +2848,7 @@ class OpenAIAssistantsAPI(BaseLLM):
         assistant_id: str,
         additional_instructions: str | None,
         instructions: str | None,
-        metadata: Dict | None,
+        metadata: dict | None,
         model: str | None,
         stream: bool | None,
         tools: Iterable[AssistantToolParam] | None,
@@ -2881,12 +2887,12 @@ class OpenAIAssistantsAPI(BaseLLM):
         assistant_id: str,
         additional_instructions: str | None,
         instructions: str | None,
-        metadata: Dict | None,
+        metadata: dict | None,
         model: str | None,
         tools: Iterable[AssistantToolParam] | None,
         event_handler: AssistantEventHandler | None,
     ) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]:
-        data: Final[Dict[str, Any]] = {
+        data: Final[dict[str, Any]] = {
             "thread_id": thread_id,
             "assistant_id": assistant_id,
             "additional_instructions": additional_instructions,
@@ -2906,12 +2912,12 @@ class OpenAIAssistantsAPI(BaseLLM):
         assistant_id: str,
         additional_instructions: str | None,
         instructions: str | None,
-        metadata: Dict | None,
+        metadata: dict | None,
         model: str | None,
         tools: Iterable[AssistantToolParam] | None,
         event_handler: AssistantEventHandler | None,
     ) -> AssistantStreamManager[AssistantEventHandler]:
-        data: Final[Dict[str, Any]] = {
+        data: Final[dict[str, Any]] = {
             "thread_id": thread_id,
             "assistant_id": assistant_id,
             "additional_instructions": additional_instructions,
@@ -2933,7 +2939,7 @@ class OpenAIAssistantsAPI(BaseLLM):
         assistant_id: str,
         additional_instructions: str | None,
         instructions: str | None,
-        metadata: Dict | None,
+        metadata: dict | None,
         model: str | None,
         stream: bool | None,
         tools: Iterable[AssistantToolParam] | None,
@@ -2955,7 +2961,7 @@ class OpenAIAssistantsAPI(BaseLLM):
         assistant_id: str,
         additional_instructions: str | None,
         instructions: str | None,
-        metadata: Dict | None,
+        metadata: dict | None,
         model: str | None,
         stream: bool | None,
         tools: Iterable[AssistantToolParam] | None,
@@ -2978,7 +2984,7 @@ class OpenAIAssistantsAPI(BaseLLM):
         assistant_id: str,
         additional_instructions: str | None,
         instructions: str | None,
-        metadata: Dict | None,
+        metadata: dict | None,
         model: str | None,
         stream: bool | None,
         tools: Iterable[AssistantToolParam] | None,